From 047644b888b121fa3feb10a5f33bdef60b1072ce Mon Sep 17 00:00:00 2001 From: Hyejin Yoon <0327jane@gmail.com> Date: Tue, 24 Dec 2024 10:06:35 +0900 Subject: [PATCH 01/27] feat: update mlflow-related metadata models (#12174) Co-authored-by: John Joyce Co-authored-by: John Joyce --- .../src/main/resources/entity.graphql | 196 +++++++++++++++++- .../dataprocess/DataProcessInstanceOutput.pdl | 2 +- .../DataProcessInstanceProperties.pdl | 2 +- .../ml/metadata/MLModelGroupProperties.pdl | 35 ++++ .../ml/metadata/MLModelProperties.pdl | 28 ++- .../ml/metadata/MLTrainingRunProperties.pdl | 36 ++++ .../src/main/resources/entity-registry.yml | 4 + .../com.linkedin.entity.aspects.snapshot.json | 54 +++-- ...com.linkedin.entity.entities.snapshot.json | 99 +++++++-- .../com.linkedin.entity.runs.snapshot.json | 54 +++-- ...nkedin.operations.operations.snapshot.json | 54 +++-- ...m.linkedin.platform.platform.snapshot.json | 99 +++++++-- 12 files changed, 568 insertions(+), 95 deletions(-) create mode 100644 metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLTrainingRunProperties.pdl diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index e086273068ee5..9abf4e16f12dd 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -262,8 +262,16 @@ type Query { Fetch all Business Attributes """ listBusinessAttributes(input: ListBusinessAttributesInput!): ListBusinessAttributesResult + + """ + Fetch a Data Process Instance by primary key (urn) + """ + dataProcessInstance(urn: String!): DataProcessInstance + + } + """ An ERModelRelationship is a high-level abstraction that dictates what datasets fields are erModelRelationshiped. """ @@ -9832,15 +9840,45 @@ type MLModelGroup implements EntityWithRelationships & Entity & BrowsableEntity privileges: EntityPrivileges } +""" +Properties describing a group of related ML models +""" type MLModelGroupProperties { + """ + Display name of the model group + """ + name: String + """ + Detailed description of the model group's purpose and contents + """ description: String - createdAt: Long + """ + When this model group was created + """ + created: AuditStamp + """ + When this model group was last modified + """ + lastModified: AuditStamp + + """ + Version identifier for this model group + """ version: VersionTag + """ + Custom key-value properties for the model group + """ customProperties: [CustomPropertiesEntry!] + + """ + Deprecated creation timestamp + @deprecated Use the 'created' field instead + """ + createdAt: Long @deprecated(reason: "Use `created` instead") } """ @@ -9990,40 +10028,103 @@ description: String } type MLMetric { + """ + Name of the metric (e.g. accuracy, precision, recall) + """ name: String + """ + Description of what this metric measures + """ description: String + """ + The computed value of the metric + """ value: String + """ + Timestamp when this metric was recorded + """ createdAt: Long } type MLModelProperties { + """ + The display name of the model used in the UI + """ + name: String! + """ + Detailed description of the model's purpose and characteristics + """ description: String - date: Long + """ + When the model was last modified + """ + lastModified: AuditStamp + """ + Version identifier for this model + """ version: String + """ + The type/category of ML model (e.g. classification, regression) + """ type: String + """ + Mapping of hyperparameter configurations + """ hyperParameters: HyperParameterMap - hyperParams: [MLHyperParam] + """ + List of hyperparameter settings used to train this model + """ + hyperParams: [MLHyperParam] + """ + Performance metrics from model training + """ trainingMetrics: [MLMetric] + """ + Names of ML features used by this model + """ mlFeatures: [String!] + """ + Tags for categorizing and searching models + """ tags: [String!] + """ + Model groups this model belongs to + """ groups: [MLModelGroup] + """ + Additional custom properties specific to this model + """ customProperties: [CustomPropertiesEntry!] + """ + URL to view this model in external system + """ externalUrl: String + + """ + When this model was created + """ + created: AuditStamp + + """ + Deprecated timestamp for model creation + @deprecated Use 'created' field instead + """ + date: Long @deprecated(reason: "Use `created` instead") } type MLFeatureProperties { @@ -12804,3 +12905,92 @@ type CronSchedule { """ timezone: String! } + + +""" +Properties describing a data process instance's execution metadata +""" +type DataProcessInstanceProperties { + """ + The display name of this process instance + """ + name: String! + + """ + URL to view this process instance in the external system + """ + externalUrl: String + + """ + When this process instance was created + """ + created: AuditStamp + + """ + Additional custom properties specific to this process instance + """ + customProperties: [CustomPropertiesEntry!] +} + +""" +Properties specific to an ML model training run instance +""" +type MLTrainingRunProperties { + """ + Unique identifier for this training run + """ + id: String + + """ + List of URLs to access training run outputs (e.g. model artifacts, logs) + """ + outputUrls: [String] + + """ + Hyperparameters used in this training run + """ + hyperParams: [MLHyperParam] + + """ + Performance metrics recorded during this training run + """ + trainingMetrics: [MLMetric] +} + +extend type DataProcessInstance { + + """ + Additional read only properties associated with the Data Job + """ + properties: DataProcessInstanceProperties + + """ + The specific instance of the data platform that this entity belongs to + """ + dataPlatformInstance: DataPlatformInstance + + """ + Sub Types that this entity implements + """ + subTypes: SubTypes + + """ + The parent container in which the entity resides + """ + container: Container + + """ + Standardized platform urn where the data process instance is defined + """ + platform: DataPlatform! + + """ + Recursively get the lineage of containers for this entity + """ + parentContainers: ParentContainersResult + + """ + Additional properties when subtype is Training Run + """ + mlTrainingRunProperties: MLTrainingRunProperties +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceOutput.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceOutput.pdl index f33c41e63efed..fe782dbe01ca9 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceOutput.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceOutput.pdl @@ -15,7 +15,7 @@ record DataProcessInstanceOutput { @Relationship = { "/*": { "name": "Produces", - "entityTypes": [ "dataset" ] + "entityTypes": [ "dataset", "mlModel" ] } } @Searchable = { diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceProperties.pdl index c63cb1a97c017..5c6bfaecf1ef4 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceProperties.pdl @@ -52,4 +52,4 @@ record DataProcessInstanceProperties includes CustomProperties, ExternalReferenc } created: AuditStamp -} +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelGroupProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelGroupProperties.pdl index b54e430038082..81c5e7a240f61 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelGroupProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelGroupProperties.pdl @@ -4,6 +4,7 @@ import com.linkedin.common.Urn import com.linkedin.common.Time import com.linkedin.common.VersionTag import com.linkedin.common.CustomProperties +import com.linkedin.common.TimeStamp /** * Properties associated with an ML Model Group @@ -13,6 +14,17 @@ import com.linkedin.common.CustomProperties } record MLModelGroupProperties includes CustomProperties { + /** + * Display name of the MLModelGroup + */ + @Searchable = { + "fieldType": "WORD_GRAM", + "enableAutocomplete": true, + "boostScore": 10.0, + "queryByDefault": true, + } + name: optional string + /** * Documentation of the MLModelGroup */ @@ -25,8 +37,31 @@ record MLModelGroupProperties includes CustomProperties { /** * Date when the MLModelGroup was developed */ + @deprecated createdAt: optional Time + /** + * Time and Actor who created the MLModelGroup + */ + created: optional TimeStamp + + /** + * Date when the MLModelGroup was last modified + */ + lastModified: optional TimeStamp + + /** + * List of jobs (if any) used to train the model group. Visible in Lineage. + */ + @Relationship = { + "/*": { + "name": "TrainedBy", + "entityTypes": [ "dataJob" ], + "isLineage": true + } + } + trainingJobs: optional array[Urn] + /** * Version of the MLModelGroup */ diff --git a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelProperties.pdl index 621a3e1747b50..d89d07384bba1 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLModelProperties.pdl @@ -6,6 +6,7 @@ import com.linkedin.common.Time import com.linkedin.common.VersionTag import com.linkedin.common.CustomProperties import com.linkedin.common.ExternalReference +import com.linkedin.common.TimeStamp /** * Properties associated with a ML Model @@ -15,6 +16,18 @@ import com.linkedin.common.ExternalReference } record MLModelProperties includes CustomProperties, ExternalReference { + /** + * Display name of the MLModel + */ + @Searchable = { + "fieldType": "WORD_GRAM", + "enableAutocomplete": true, + "boostScore": 10.0, + "queryByDefault": true, + } + name: optional string + + /** * Documentation of the MLModel */ @@ -27,8 +40,19 @@ record MLModelProperties includes CustomProperties, ExternalReference { /** * Date when the MLModel was developed */ + @deprecated date: optional Time + /** + * Audit stamp containing who created this and when + */ + created: optional TimeStamp + + /** + * Date when the MLModel was last modified + */ + lastModified: optional TimeStamp + /** * Version of the MLModel */ @@ -93,12 +117,12 @@ record MLModelProperties includes CustomProperties, ExternalReference { deployments: optional array[Urn] /** - * List of jobs (if any) used to train the model + * List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect. */ @Relationship = { "/*": { "name": "TrainedBy", - "entityTypes": [ "dataJob" ], + "entityTypes": [ "dataJob", "dataProcessInstance" ], "isLineage": true } } diff --git a/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLTrainingRunProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLTrainingRunProperties.pdl new file mode 100644 index 0000000000000..f8b8eeafe908b --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/ml/metadata/MLTrainingRunProperties.pdl @@ -0,0 +1,36 @@ +namespace com.linkedin.ml.metadata + +import com.linkedin.common.AuditStamp +import com.linkedin.common.CustomProperties +import com.linkedin.common.ExternalReference +import com.linkedin.common.Urn +import com.linkedin.common.JobFlowUrn +import com.linkedin.common.DataJobUrn +/** + * The inputs and outputs of this training run + */ +@Aspect = { + "name": "mlTrainingRunProperties", +} +record MLTrainingRunProperties includes CustomProperties, ExternalReference { + + /** + * Run Id of the ML Training Run + */ + id: optional string + + /** + * List of URLs for the Outputs of the ML Training Run + */ + outputUrls: optional array[string] + + /** + * Hyperparameters of the ML Training Run + */ + hyperParams: optional array[MLHyperParam] + + /** + * Metrics of the ML Training Run + */ + trainingMetrics: optional array[MLMetric] +} \ No newline at end of file diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 1c3eb5b574e20..4fe170ced69f3 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -116,6 +116,10 @@ entities: - dataProcessInstanceRunEvent - status - testResults + - dataPlatformInstance + - subTypes + - container + - mlTrainingRunProperties - name: chart category: core keyAspect: chartKey diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index 827789130d8bb..1c713fd33884b 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -3826,12 +3826,23 @@ "type" : "record", "name" : "MLModelProperties", "namespace" : "com.linkedin.ml.metadata", - "doc" : "Properties associated with a ML Model", + "doc" : "Properties associated with a ML Model\r", "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ], "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Display name of the MLModel\r", + "optional" : true, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "WORD_GRAM", + "queryByDefault" : true + } + }, { "name" : "description", "type" : "string", - "doc" : "Documentation of the MLModel", + "doc" : "Documentation of the MLModel\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT", @@ -3840,17 +3851,28 @@ }, { "name" : "date", "type" : "com.linkedin.common.Time", - "doc" : "Date when the MLModel was developed", + "doc" : "Date when the MLModel was developed\r", + "optional" : true, + "deprecated" : true + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Audit stamp containing who created this and when\r", + "optional" : true + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Date when the MLModel was last modified\r", "optional" : true }, { "name" : "version", "type" : "com.linkedin.common.VersionTag", - "doc" : "Version of the MLModel", + "doc" : "Version of the MLModel\r", "optional" : true }, { "name" : "type", "type" : "string", - "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc", + "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT_PARTIAL" @@ -3866,7 +3888,7 @@ "ref" : [ "string", "int", "float", "double", "boolean" ] } }, - "doc" : "Hyper Parameters of the MLModel\n\nNOTE: these are deprecated in favor of hyperParams", + "doc" : "Hyper Parameters of the MLModel\r\n\r\nNOTE: these are deprecated in favor of hyperParams\r", "optional" : true }, { "name" : "hyperParams", @@ -3901,7 +3923,7 @@ } } }, - "doc" : "Hyperparameters of the MLModel", + "doc" : "Hyperparameters of the MLModel\r", "optional" : true }, { "name" : "trainingMetrics", @@ -3936,7 +3958,7 @@ } } }, - "doc" : "Metrics of the MLModel used in training", + "doc" : "Metrics of the MLModel used in training\r", "optional" : true }, { "name" : "onlineMetrics", @@ -3944,7 +3966,7 @@ "type" : "array", "items" : "MLMetric" }, - "doc" : "Metrics of the MLModel used in production", + "doc" : "Metrics of the MLModel used in production\r", "optional" : true }, { "name" : "mlFeatures", @@ -3952,7 +3974,7 @@ "type" : "array", "items" : "com.linkedin.common.MLFeatureUrn" }, - "doc" : "List of features used for MLModel training", + "doc" : "List of features used for MLModel training\r", "optional" : true, "Relationship" : { "/*" : { @@ -3967,7 +3989,7 @@ "type" : "array", "items" : "string" }, - "doc" : "Tags for the MLModel", + "doc" : "Tags for the MLModel\r", "default" : [ ] }, { "name" : "deployments", @@ -3975,7 +3997,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "Deployments for the MLModel", + "doc" : "Deployments for the MLModel\r", "optional" : true, "Relationship" : { "/*" : { @@ -3989,11 +4011,11 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "List of jobs (if any) used to train the model", + "doc" : "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.\r", "optional" : true, "Relationship" : { "/*" : { - "entityTypes" : [ "dataJob" ], + "entityTypes" : [ "dataJob", "dataProcessInstance" ], "isLineage" : true, "name" : "TrainedBy" } @@ -4004,7 +4026,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "List of jobs (if any) that use the model", + "doc" : "List of jobs (if any) that use the model\r", "optional" : true, "Relationship" : { "/*" : { @@ -4020,7 +4042,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "Groups the model belongs to", + "doc" : "Groups the model belongs to\r", "optional" : true, "Relationship" : { "/*" : { diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index b549cef0af84b..77d4644f3c121 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -3984,12 +3984,23 @@ "type" : "record", "name" : "MLModelProperties", "namespace" : "com.linkedin.ml.metadata", - "doc" : "Properties associated with a ML Model", + "doc" : "Properties associated with a ML Model\r", "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ], "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Display name of the MLModel\r", + "optional" : true, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "WORD_GRAM", + "queryByDefault" : true + } + }, { "name" : "description", "type" : "string", - "doc" : "Documentation of the MLModel", + "doc" : "Documentation of the MLModel\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT", @@ -3998,17 +4009,28 @@ }, { "name" : "date", "type" : "com.linkedin.common.Time", - "doc" : "Date when the MLModel was developed", + "doc" : "Date when the MLModel was developed\r", + "optional" : true, + "deprecated" : true + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Audit stamp containing who created this and when\r", + "optional" : true + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Date when the MLModel was last modified\r", "optional" : true }, { "name" : "version", "type" : "com.linkedin.common.VersionTag", - "doc" : "Version of the MLModel", + "doc" : "Version of the MLModel\r", "optional" : true }, { "name" : "type", "type" : "string", - "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc", + "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT_PARTIAL" @@ -4024,7 +4046,7 @@ "ref" : [ "string", "int", "float", "double", "boolean" ] } }, - "doc" : "Hyper Parameters of the MLModel\n\nNOTE: these are deprecated in favor of hyperParams", + "doc" : "Hyper Parameters of the MLModel\r\n\r\nNOTE: these are deprecated in favor of hyperParams\r", "optional" : true }, { "name" : "hyperParams", @@ -4059,7 +4081,7 @@ } } }, - "doc" : "Hyperparameters of the MLModel", + "doc" : "Hyperparameters of the MLModel\r", "optional" : true }, { "name" : "trainingMetrics", @@ -4094,7 +4116,7 @@ } } }, - "doc" : "Metrics of the MLModel used in training", + "doc" : "Metrics of the MLModel used in training\r", "optional" : true }, { "name" : "onlineMetrics", @@ -4102,7 +4124,7 @@ "type" : "array", "items" : "MLMetric" }, - "doc" : "Metrics of the MLModel used in production", + "doc" : "Metrics of the MLModel used in production\r", "optional" : true }, { "name" : "mlFeatures", @@ -4110,7 +4132,7 @@ "type" : "array", "items" : "com.linkedin.common.MLFeatureUrn" }, - "doc" : "List of features used for MLModel training", + "doc" : "List of features used for MLModel training\r", "optional" : true, "Relationship" : { "/*" : { @@ -4125,7 +4147,7 @@ "type" : "array", "items" : "string" }, - "doc" : "Tags for the MLModel", + "doc" : "Tags for the MLModel\r", "default" : [ ] }, { "name" : "deployments", @@ -4133,7 +4155,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "Deployments for the MLModel", + "doc" : "Deployments for the MLModel\r", "optional" : true, "Relationship" : { "/*" : { @@ -4147,11 +4169,11 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "List of jobs (if any) used to train the model", + "doc" : "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.\r", "optional" : true, "Relationship" : { "/*" : { - "entityTypes" : [ "dataJob" ], + "entityTypes" : [ "dataJob", "dataProcessInstance" ], "isLineage" : true, "name" : "TrainedBy" } @@ -4162,7 +4184,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "List of jobs (if any) that use the model", + "doc" : "List of jobs (if any) that use the model\r", "optional" : true, "Relationship" : { "/*" : { @@ -4178,7 +4200,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "Groups the model belongs to", + "doc" : "Groups the model belongs to\r", "optional" : true, "Relationship" : { "/*" : { @@ -4981,12 +5003,23 @@ "type" : "record", "name" : "MLModelGroupProperties", "namespace" : "com.linkedin.ml.metadata", - "doc" : "Properties associated with an ML Model Group", + "doc" : "Properties associated with an ML Model Group\r", "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Display name of the MLModelGroup\r", + "optional" : true, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "WORD_GRAM", + "queryByDefault" : true + } + }, { "name" : "description", "type" : "string", - "doc" : "Documentation of the MLModelGroup", + "doc" : "Documentation of the MLModelGroup\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT", @@ -4995,12 +5028,38 @@ }, { "name" : "createdAt", "type" : "com.linkedin.common.Time", - "doc" : "Date when the MLModelGroup was developed", + "doc" : "Date when the MLModelGroup was developed\r", + "optional" : true, + "deprecated" : true + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Time and Actor who created the MLModelGroup\r", + "optional" : true + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Date when the MLModelGroup was last modified\r", "optional" : true + }, { + "name" : "trainingJobs", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "List of jobs (if any) used to train the model group. Visible in Lineage.\r", + "optional" : true, + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataJob" ], + "isLineage" : true, + "name" : "TrainedBy" + } + } }, { "name" : "version", "type" : "com.linkedin.common.VersionTag", - "doc" : "Version of the MLModelGroup", + "doc" : "Version of the MLModelGroup\r", "optional" : true } ], "Aspect" : { diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json index c8be9d063eaea..8b6def75f7a66 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json @@ -3550,12 +3550,23 @@ "type" : "record", "name" : "MLModelProperties", "namespace" : "com.linkedin.ml.metadata", - "doc" : "Properties associated with a ML Model", + "doc" : "Properties associated with a ML Model\r", "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ], "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Display name of the MLModel\r", + "optional" : true, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "WORD_GRAM", + "queryByDefault" : true + } + }, { "name" : "description", "type" : "string", - "doc" : "Documentation of the MLModel", + "doc" : "Documentation of the MLModel\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT", @@ -3564,17 +3575,28 @@ }, { "name" : "date", "type" : "com.linkedin.common.Time", - "doc" : "Date when the MLModel was developed", + "doc" : "Date when the MLModel was developed\r", + "optional" : true, + "deprecated" : true + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Audit stamp containing who created this and when\r", + "optional" : true + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Date when the MLModel was last modified\r", "optional" : true }, { "name" : "version", "type" : "com.linkedin.common.VersionTag", - "doc" : "Version of the MLModel", + "doc" : "Version of the MLModel\r", "optional" : true }, { "name" : "type", "type" : "string", - "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc", + "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT_PARTIAL" @@ -3590,7 +3612,7 @@ "ref" : [ "string", "int", "float", "double", "boolean" ] } }, - "doc" : "Hyper Parameters of the MLModel\n\nNOTE: these are deprecated in favor of hyperParams", + "doc" : "Hyper Parameters of the MLModel\r\n\r\nNOTE: these are deprecated in favor of hyperParams\r", "optional" : true }, { "name" : "hyperParams", @@ -3625,7 +3647,7 @@ } } }, - "doc" : "Hyperparameters of the MLModel", + "doc" : "Hyperparameters of the MLModel\r", "optional" : true }, { "name" : "trainingMetrics", @@ -3660,7 +3682,7 @@ } } }, - "doc" : "Metrics of the MLModel used in training", + "doc" : "Metrics of the MLModel used in training\r", "optional" : true }, { "name" : "onlineMetrics", @@ -3668,7 +3690,7 @@ "type" : "array", "items" : "MLMetric" }, - "doc" : "Metrics of the MLModel used in production", + "doc" : "Metrics of the MLModel used in production\r", "optional" : true }, { "name" : "mlFeatures", @@ -3676,7 +3698,7 @@ "type" : "array", "items" : "com.linkedin.common.MLFeatureUrn" }, - "doc" : "List of features used for MLModel training", + "doc" : "List of features used for MLModel training\r", "optional" : true, "Relationship" : { "/*" : { @@ -3691,7 +3713,7 @@ "type" : "array", "items" : "string" }, - "doc" : "Tags for the MLModel", + "doc" : "Tags for the MLModel\r", "default" : [ ] }, { "name" : "deployments", @@ -3699,7 +3721,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "Deployments for the MLModel", + "doc" : "Deployments for the MLModel\r", "optional" : true, "Relationship" : { "/*" : { @@ -3713,11 +3735,11 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "List of jobs (if any) used to train the model", + "doc" : "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.\r", "optional" : true, "Relationship" : { "/*" : { - "entityTypes" : [ "dataJob" ], + "entityTypes" : [ "dataJob", "dataProcessInstance" ], "isLineage" : true, "name" : "TrainedBy" } @@ -3728,7 +3750,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "List of jobs (if any) that use the model", + "doc" : "List of jobs (if any) that use the model\r", "optional" : true, "Relationship" : { "/*" : { @@ -3744,7 +3766,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "Groups the model belongs to", + "doc" : "Groups the model belongs to\r", "optional" : true, "Relationship" : { "/*" : { diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json index 8c7595c5e505d..e4cc5c42303ee 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json @@ -3544,12 +3544,23 @@ "type" : "record", "name" : "MLModelProperties", "namespace" : "com.linkedin.ml.metadata", - "doc" : "Properties associated with a ML Model", + "doc" : "Properties associated with a ML Model\r", "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ], "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Display name of the MLModel\r", + "optional" : true, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "WORD_GRAM", + "queryByDefault" : true + } + }, { "name" : "description", "type" : "string", - "doc" : "Documentation of the MLModel", + "doc" : "Documentation of the MLModel\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT", @@ -3558,17 +3569,28 @@ }, { "name" : "date", "type" : "com.linkedin.common.Time", - "doc" : "Date when the MLModel was developed", + "doc" : "Date when the MLModel was developed\r", + "optional" : true, + "deprecated" : true + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Audit stamp containing who created this and when\r", + "optional" : true + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Date when the MLModel was last modified\r", "optional" : true }, { "name" : "version", "type" : "com.linkedin.common.VersionTag", - "doc" : "Version of the MLModel", + "doc" : "Version of the MLModel\r", "optional" : true }, { "name" : "type", "type" : "string", - "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc", + "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT_PARTIAL" @@ -3584,7 +3606,7 @@ "ref" : [ "string", "int", "float", "double", "boolean" ] } }, - "doc" : "Hyper Parameters of the MLModel\n\nNOTE: these are deprecated in favor of hyperParams", + "doc" : "Hyper Parameters of the MLModel\r\n\r\nNOTE: these are deprecated in favor of hyperParams\r", "optional" : true }, { "name" : "hyperParams", @@ -3619,7 +3641,7 @@ } } }, - "doc" : "Hyperparameters of the MLModel", + "doc" : "Hyperparameters of the MLModel\r", "optional" : true }, { "name" : "trainingMetrics", @@ -3654,7 +3676,7 @@ } } }, - "doc" : "Metrics of the MLModel used in training", + "doc" : "Metrics of the MLModel used in training\r", "optional" : true }, { "name" : "onlineMetrics", @@ -3662,7 +3684,7 @@ "type" : "array", "items" : "MLMetric" }, - "doc" : "Metrics of the MLModel used in production", + "doc" : "Metrics of the MLModel used in production\r", "optional" : true }, { "name" : "mlFeatures", @@ -3670,7 +3692,7 @@ "type" : "array", "items" : "com.linkedin.common.MLFeatureUrn" }, - "doc" : "List of features used for MLModel training", + "doc" : "List of features used for MLModel training\r", "optional" : true, "Relationship" : { "/*" : { @@ -3685,7 +3707,7 @@ "type" : "array", "items" : "string" }, - "doc" : "Tags for the MLModel", + "doc" : "Tags for the MLModel\r", "default" : [ ] }, { "name" : "deployments", @@ -3693,7 +3715,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "Deployments for the MLModel", + "doc" : "Deployments for the MLModel\r", "optional" : true, "Relationship" : { "/*" : { @@ -3707,11 +3729,11 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "List of jobs (if any) used to train the model", + "doc" : "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.\r", "optional" : true, "Relationship" : { "/*" : { - "entityTypes" : [ "dataJob" ], + "entityTypes" : [ "dataJob", "dataProcessInstance" ], "isLineage" : true, "name" : "TrainedBy" } @@ -3722,7 +3744,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "List of jobs (if any) that use the model", + "doc" : "List of jobs (if any) that use the model\r", "optional" : true, "Relationship" : { "/*" : { @@ -3738,7 +3760,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "Groups the model belongs to", + "doc" : "Groups the model belongs to\r", "optional" : true, "Relationship" : { "/*" : { diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index 75e5c9a559076..e375ac698ab51 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -3978,12 +3978,23 @@ "type" : "record", "name" : "MLModelProperties", "namespace" : "com.linkedin.ml.metadata", - "doc" : "Properties associated with a ML Model", + "doc" : "Properties associated with a ML Model\r", "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ], "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Display name of the MLModel\r", + "optional" : true, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "WORD_GRAM", + "queryByDefault" : true + } + }, { "name" : "description", "type" : "string", - "doc" : "Documentation of the MLModel", + "doc" : "Documentation of the MLModel\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT", @@ -3992,17 +4003,28 @@ }, { "name" : "date", "type" : "com.linkedin.common.Time", - "doc" : "Date when the MLModel was developed", + "doc" : "Date when the MLModel was developed\r", + "optional" : true, + "deprecated" : true + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Audit stamp containing who created this and when\r", + "optional" : true + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Date when the MLModel was last modified\r", "optional" : true }, { "name" : "version", "type" : "com.linkedin.common.VersionTag", - "doc" : "Version of the MLModel", + "doc" : "Version of the MLModel\r", "optional" : true }, { "name" : "type", "type" : "string", - "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc", + "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT_PARTIAL" @@ -4018,7 +4040,7 @@ "ref" : [ "string", "int", "float", "double", "boolean" ] } }, - "doc" : "Hyper Parameters of the MLModel\n\nNOTE: these are deprecated in favor of hyperParams", + "doc" : "Hyper Parameters of the MLModel\r\n\r\nNOTE: these are deprecated in favor of hyperParams\r", "optional" : true }, { "name" : "hyperParams", @@ -4053,7 +4075,7 @@ } } }, - "doc" : "Hyperparameters of the MLModel", + "doc" : "Hyperparameters of the MLModel\r", "optional" : true }, { "name" : "trainingMetrics", @@ -4088,7 +4110,7 @@ } } }, - "doc" : "Metrics of the MLModel used in training", + "doc" : "Metrics of the MLModel used in training\r", "optional" : true }, { "name" : "onlineMetrics", @@ -4096,7 +4118,7 @@ "type" : "array", "items" : "MLMetric" }, - "doc" : "Metrics of the MLModel used in production", + "doc" : "Metrics of the MLModel used in production\r", "optional" : true }, { "name" : "mlFeatures", @@ -4104,7 +4126,7 @@ "type" : "array", "items" : "com.linkedin.common.MLFeatureUrn" }, - "doc" : "List of features used for MLModel training", + "doc" : "List of features used for MLModel training\r", "optional" : true, "Relationship" : { "/*" : { @@ -4119,7 +4141,7 @@ "type" : "array", "items" : "string" }, - "doc" : "Tags for the MLModel", + "doc" : "Tags for the MLModel\r", "default" : [ ] }, { "name" : "deployments", @@ -4127,7 +4149,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "Deployments for the MLModel", + "doc" : "Deployments for the MLModel\r", "optional" : true, "Relationship" : { "/*" : { @@ -4141,11 +4163,11 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "List of jobs (if any) used to train the model", + "doc" : "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.\r", "optional" : true, "Relationship" : { "/*" : { - "entityTypes" : [ "dataJob" ], + "entityTypes" : [ "dataJob", "dataProcessInstance" ], "isLineage" : true, "name" : "TrainedBy" } @@ -4156,7 +4178,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "List of jobs (if any) that use the model", + "doc" : "List of jobs (if any) that use the model\r", "optional" : true, "Relationship" : { "/*" : { @@ -4172,7 +4194,7 @@ "type" : "array", "items" : "com.linkedin.common.Urn" }, - "doc" : "Groups the model belongs to", + "doc" : "Groups the model belongs to\r", "optional" : true, "Relationship" : { "/*" : { @@ -4975,12 +4997,23 @@ "type" : "record", "name" : "MLModelGroupProperties", "namespace" : "com.linkedin.ml.metadata", - "doc" : "Properties associated with an ML Model Group", + "doc" : "Properties associated with an ML Model Group\r", "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Display name of the MLModelGroup\r", + "optional" : true, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "WORD_GRAM", + "queryByDefault" : true + } + }, { "name" : "description", "type" : "string", - "doc" : "Documentation of the MLModelGroup", + "doc" : "Documentation of the MLModelGroup\r", "optional" : true, "Searchable" : { "fieldType" : "TEXT", @@ -4989,12 +5022,38 @@ }, { "name" : "createdAt", "type" : "com.linkedin.common.Time", - "doc" : "Date when the MLModelGroup was developed", + "doc" : "Date when the MLModelGroup was developed\r", + "optional" : true, + "deprecated" : true + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Time and Actor who created the MLModelGroup\r", + "optional" : true + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "Date when the MLModelGroup was last modified\r", "optional" : true + }, { + "name" : "trainingJobs", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "List of jobs (if any) used to train the model group. Visible in Lineage.\r", + "optional" : true, + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataJob" ], + "isLineage" : true, + "name" : "TrainedBy" + } + } }, { "name" : "version", "type" : "com.linkedin.common.VersionTag", - "doc" : "Version of the MLModelGroup", + "doc" : "Version of the MLModelGroup\r", "optional" : true } ], "Aspect" : { From 09a9b6eef912d8f855a2cc6fdc03032f5ec7a652 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Mon, 23 Dec 2024 22:39:57 -0800 Subject: [PATCH 02/27] feat(ingest/looker): Do not emit usage for non-ingested dashboards and charts (#11647) --- .../ingestion/source/looker/looker_common.py | 9 + .../ingestion/source/looker/looker_source.py | 22 +- .../ingestion/source/looker/looker_usage.py | 40 +- .../looker/looker_mces_usage_history.json | 364 +++++++++++++++++- .../tests/integration/looker/test_looker.py | 87 ++++- 5 files changed, 482 insertions(+), 40 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py index a66962f962255..1183916e9b3fe 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py @@ -1408,6 +1408,15 @@ class LookerDashboardSourceReport(StaleEntityRemovalSourceReport): dashboards_with_activity: LossySet[str] = dataclasses_field( default_factory=LossySet ) + + # Entities that don't seem to exist, so we don't emit usage aspects for them despite having usage data + dashboards_skipped_for_usage: LossySet[str] = dataclasses_field( + default_factory=LossySet + ) + charts_skipped_for_usage: LossySet[str] = dataclasses_field( + default_factory=LossySet + ) + stage_latency: List[StageLatency] = dataclasses_field(default_factory=list) _looker_explore_registry: Optional[LookerExploreRegistry] = None total_explores: int = 0 diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index 815c5dfb1c014..8487d5113bc1d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -68,6 +68,7 @@ ViewField, ViewFieldType, gen_model_key, + get_urn_looker_element_id, ) from datahub.ingestion.source.looker.looker_config import LookerDashboardSourceConfig from datahub.ingestion.source.looker.looker_lib_wrapper import LookerAPI @@ -165,6 +166,9 @@ def __init__(self, config: LookerDashboardSourceConfig, ctx: PipelineContext): # Required, as we do not ingest all folders but only those that have dashboards/looks self.processed_folders: List[str] = [] + # Keep track of ingested chart urns, to omit usage for non-ingested entities + self.chart_urns: Set[str] = set() + @staticmethod def test_connection(config_dict: dict) -> TestConnectionReport: test_report = TestConnectionReport() @@ -642,6 +646,7 @@ def _make_chart_metadata_events( chart_urn = self._make_chart_urn( element_id=dashboard_element.get_urn_element_id() ) + self.chart_urns.add(chart_urn) chart_snapshot = ChartSnapshot( urn=chart_urn, aspects=[Status(removed=False)], @@ -1380,7 +1385,9 @@ def _get_folder_and_ancestors_workunits( yield from self._emit_folder_as_container(folder) def extract_usage_stat( - self, looker_dashboards: List[looker_usage.LookerDashboardForUsage] + self, + looker_dashboards: List[looker_usage.LookerDashboardForUsage], + ingested_chart_urns: Set[str], ) -> List[MetadataChangeProposalWrapper]: looks: List[looker_usage.LookerChartForUsage] = [] # filter out look from all dashboard @@ -1391,6 +1398,15 @@ def extract_usage_stat( # dedup looks looks = list({str(look.id): look for look in looks}.values()) + filtered_looks = [] + for look in looks: + if not look.id: + continue + chart_urn = self._make_chart_urn(get_urn_looker_element_id(look.id)) + if chart_urn in ingested_chart_urns: + filtered_looks.append(look) + else: + self.reporter.charts_skipped_for_usage.add(look.id) # Keep stat generators to generate entity stat aspect later stat_generator_config: looker_usage.StatGeneratorConfig = ( @@ -1414,7 +1430,7 @@ def extract_usage_stat( stat_generator_config, self.reporter, self._make_chart_urn, - looks, + filtered_looks, ) mcps: List[MetadataChangeProposalWrapper] = [] @@ -1669,7 +1685,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: if self.source_config.extract_usage_history: self.reporter.report_stage_start("usage_extraction") usage_mcps: List[MetadataChangeProposalWrapper] = self.extract_usage_stat( - looker_dashboards_for_usage + looker_dashboards_for_usage, self.chart_urns ) for usage_mcp in usage_mcps: yield usage_mcp.as_workunit() diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py index ef7d64e4f42d4..098d7d73a3da8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py @@ -42,6 +42,7 @@ TimeWindowSizeClass, _Aspect as AspectAbstract, ) +from datahub.utilities.lossy_collections import LossySet logger = logging.getLogger(__name__) @@ -170,7 +171,7 @@ def __init__( self.config = config self.looker_models = looker_models # Later it will help to find out for what are the looker entities from query result - self.id_vs_model: Dict[str, ModelForUsage] = { + self.id_to_model: Dict[str, ModelForUsage] = { self.get_id(looker_object): looker_object for looker_object in looker_models } self.post_filter = len(self.looker_models) > 100 @@ -225,6 +226,10 @@ def get_id(self, looker_object: ModelForUsage) -> str: def get_id_from_row(self, row: dict) -> str: pass + @abstractmethod + def report_skip_set(self) -> LossySet[str]: + pass + def create_mcp( self, model: ModelForUsage, aspect: Aspect ) -> MetadataChangeProposalWrapper: @@ -258,20 +263,11 @@ def _process_entity_timeseries_rows( return entity_stat_aspect - def _process_absolute_aspect(self) -> List[Tuple[ModelForUsage, AspectAbstract]]: - aspects: List[Tuple[ModelForUsage, AspectAbstract]] = [] - for looker_object in self.looker_models: - aspects.append( - (looker_object, self.to_entity_absolute_stat_aspect(looker_object)) - ) - - return aspects - def _fill_user_stat_aspect( self, entity_usage_stat: Dict[Tuple[str, str], Aspect], user_wise_rows: List[Dict], - ) -> Iterable[Tuple[ModelForUsage, Aspect]]: + ) -> Iterable[Tuple[str, Aspect]]: logger.debug("Entering fill user stat aspect") # We first resolve all the users using a threadpool to warm up the cache @@ -300,7 +296,7 @@ def _fill_user_stat_aspect( for row in user_wise_rows: # Confirm looker object was given for stat generation - looker_object = self.id_vs_model.get(self.get_id_from_row(row)) + looker_object = self.id_to_model.get(self.get_id_from_row(row)) if looker_object is None: logger.warning( "Looker object with id({}) was not register with stat generator".format( @@ -338,7 +334,7 @@ def _fill_user_stat_aspect( logger.debug("Starting to yield answers for user-wise counts") for (id, _), aspect in entity_usage_stat.items(): - yield self.id_vs_model[id], aspect + yield id, aspect def _execute_query(self, query: LookerQuery, query_name: str) -> List[Dict]: rows = [] @@ -357,7 +353,7 @@ def _execute_query(self, query: LookerQuery, query_name: str) -> List[Dict]: ) if self.post_filter: logger.debug("post filtering") - rows = [r for r in rows if self.get_id_from_row(r) in self.id_vs_model] + rows = [r for r in rows if self.get_id_from_row(r) in self.id_to_model] logger.debug("Filtered down to %d rows", len(rows)) except Exception as e: logger.warning(f"Failed to execute {query_name} query: {e}") @@ -378,7 +374,8 @@ def generate_usage_stat_mcps(self) -> Iterable[MetadataChangeProposalWrapper]: return # yield absolute stat for looker entities - for looker_object, aspect in self._process_absolute_aspect(): # type: ignore + for looker_object in self.looker_models: + aspect = self.to_entity_absolute_stat_aspect(looker_object) yield self.create_mcp(looker_object, aspect) # Execute query and process the raw json which contains stat information @@ -399,10 +396,13 @@ def generate_usage_stat_mcps(self) -> Iterable[MetadataChangeProposalWrapper]: ) user_wise_rows = self._execute_query(user_wise_query_with_filters, "user_query") # yield absolute stat for entity - for looker_object, aspect in self._fill_user_stat_aspect( + for object_id, aspect in self._fill_user_stat_aspect( entity_usage_stat, user_wise_rows ): - yield self.create_mcp(looker_object, aspect) + if object_id in self.id_to_model: + yield self.create_mcp(self.id_to_model[object_id], aspect) + else: + self.report_skip_set().add(object_id) class DashboardStatGenerator(BaseStatGenerator): @@ -425,6 +425,9 @@ def __init__( def get_stats_generator_name(self) -> str: return "DashboardStats" + def report_skip_set(self) -> LossySet[str]: + return self.report.dashboards_skipped_for_usage + def get_filter(self) -> Dict[ViewField, str]: return { HistoryViewField.HISTORY_DASHBOARD_ID: ",".join( @@ -541,6 +544,9 @@ def __init__( def get_stats_generator_name(self) -> str: return "ChartStats" + def report_skip_set(self) -> LossySet[str]: + return self.report.charts_skipped_for_usage + def get_filter(self) -> Dict[ViewField, str]: return { LookViewField.LOOK_ID: ",".join( diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json index 594983c8fb0f2..ed0c5401c9029 100644 --- a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json +++ b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json @@ -1,4 +1,66 @@ [ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,dashboard_elements.3)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "upstream_fields": "" + }, + "title": "", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "https://looker.company.com/x/", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.3)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Look" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9,7 +71,9 @@ "customProperties": {}, "title": "foo", "description": "lorem ipsum", - "charts": [], + "charts": [ + "urn:li:chart:(looker,dashboard_elements.3)" + ], "datasets": [], "dashboards": [], "lastModified": { @@ -89,6 +153,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.3)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(looker,dashboards.1)", @@ -215,6 +295,98 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "looker", + "env": "PROD", + "model_name": "look_data" + }, + "name": "look_data", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:looker" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "LookML Model" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Explore" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -389,6 +561,180 @@ "lastRunId": "no-run-id-provided" } }, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Explore/look_data" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "project": "lkml_samples", + "model": "look_data", + "looker.explore.label": "My Explore View", + "looker.explore.name": "look_view", + "looker.explore.file": "test_source_file.lkml" + }, + "externalUrl": "https://looker.company.com/explore/look_data/look_view", + "name": "My Explore View", + "description": "lorem ipsum", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.underlying_view,PROD)", + "type": "VIEW" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "look_view", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + ], + "primaryKeys": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Explore" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)", + "changeType": "UPSERT", + "aspectName": "embed", + "aspect": { + "json": { + "renderUrl": "https://looker.company.com/embed/explore/look_data/look_view" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,look_data.explore.look_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Explore" + }, + { + "id": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb", + "urn": "urn:li:container:a2a7aa63752695f9a1705faed9d03ffb" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "lastRunId": "no-run-id-provided" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { @@ -747,22 +1093,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.3)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "looker-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", diff --git a/metadata-ingestion/tests/integration/looker/test_looker.py b/metadata-ingestion/tests/integration/looker/test_looker.py index a39de8384efb2..c96bcc729a95d 100644 --- a/metadata-ingestion/tests/integration/looker/test_looker.py +++ b/metadata-ingestion/tests/integration/looker/test_looker.py @@ -31,7 +31,10 @@ from datahub.ingestion.api.source import SourceReport from datahub.ingestion.run.pipeline import Pipeline, PipelineInitError from datahub.ingestion.source.looker import looker_common, looker_usage -from datahub.ingestion.source.looker.looker_common import LookerExplore +from datahub.ingestion.source.looker.looker_common import ( + LookerDashboardSourceReport, + LookerExplore, +) from datahub.ingestion.source.looker.looker_config import LookerCommonConfig from datahub.ingestion.source.looker.looker_lib_wrapper import ( LookerAPI, @@ -414,7 +417,9 @@ def setup_mock_dashboard_multiple_charts(mocked_client): ) -def setup_mock_dashboard_with_usage(mocked_client): +def setup_mock_dashboard_with_usage( + mocked_client: mock.MagicMock, skip_look: bool = False +) -> None: mocked_client.all_dashboards.return_value = [Dashboard(id="1")] mocked_client.dashboard.return_value = Dashboard( id="1", @@ -437,7 +442,13 @@ def setup_mock_dashboard_with_usage(mocked_client): ), ), DashboardElement( - id="3", type="", look=LookWithQuery(id="3", view_count=30) + id="3", + type="" if skip_look else "vis", # Looks only ingested if type == `vis` + look=LookWithQuery( + id="3", + view_count=30, + query=Query(model="look_data", view="look_view"), + ), ), ], ) @@ -611,6 +622,12 @@ def side_effect_query_inline( HistoryViewField.HISTORY_DASHBOARD_USER: 1, HistoryViewField.HISTORY_DASHBOARD_RUN_COUNT: 5, }, + { + HistoryViewField.HISTORY_DASHBOARD_ID: "5", + HistoryViewField.HISTORY_CREATED_DATE: "2022-07-07", + HistoryViewField.HISTORY_DASHBOARD_USER: 1, + HistoryViewField.HISTORY_DASHBOARD_RUN_COUNT: 5, + }, ] ), looker_usage.QueryId.DASHBOARD_PER_USER_PER_DAY_USAGE_STAT: json.dumps( @@ -790,6 +807,70 @@ def test_looker_ingest_usage_history(pytestconfig, tmp_path, mock_time): ) +@freeze_time(FROZEN_TIME) +def test_looker_filter_usage_history(pytestconfig, tmp_path, mock_time): + mocked_client = mock.MagicMock() + with mock.patch("looker_sdk.init40") as mock_sdk: + mock_sdk.return_value = mocked_client + setup_mock_dashboard_with_usage(mocked_client, skip_look=True) + mocked_client.run_inline_query.side_effect = side_effect_query_inline + setup_mock_explore(mocked_client) + setup_mock_user(mocked_client) + + temp_output_file = f"{tmp_path}/looker_mces.json" + pipeline = Pipeline.create( + { + "run_id": "looker-test", + "source": { + "type": "looker", + "config": { + "base_url": "https://looker.company.com", + "client_id": "foo", + "client_secret": "bar", + "extract_usage_history": True, + "max_threads": 1, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": temp_output_file, + }, + }, + } + ) + pipeline.run() + pipeline.pretty_print_summary() + pipeline.raise_from_status() + + # There should be 4 dashboardUsageStatistics aspects (one absolute and 3 timeseries) + dashboard_usage_aspect_count = 0 + # There should be 0 chartUsageStatistics -- filtered by set of ingested charts + chart_usage_aspect_count = 0 + with open(temp_output_file) as f: + temp_output_dict = json.load(f) + for element in temp_output_dict: + if ( + element.get("entityType") == "dashboard" + and element.get("aspectName") == "dashboardUsageStatistics" + ): + dashboard_usage_aspect_count = dashboard_usage_aspect_count + 1 + if ( + element.get("entityType") == "chart" + and element.get("aspectName") == "chartUsageStatistics" + ): + chart_usage_aspect_count = chart_usage_aspect_count + 1 + + assert dashboard_usage_aspect_count == 4 + assert chart_usage_aspect_count == 0 + + source_report = cast(LookerDashboardSourceReport, pipeline.source.get_report()) + # From timeseries query + assert str(source_report.dashboards_skipped_for_usage) == str(["5"]) + # From dashboard element + assert str(source_report.charts_skipped_for_usage) == str(["3"]) + + @freeze_time(FROZEN_TIME) def test_looker_ingest_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): output_file_name: str = "looker_mces.json" From 87e7b58ac699005ca5757e6ef47fb853d89a6583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Tue, 24 Dec 2024 10:46:19 +0100 Subject: [PATCH 03/27] fix(tableau): retry on InternalServerError 504 (#12213) --- .../ingestion/source/tableau/tableau.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 984cf9357199d..2b7aac2bea1d0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -35,7 +35,10 @@ SiteItem, TableauAuth, ) -from tableauserverclient.server.endpoint.exceptions import NonXMLResponseError +from tableauserverclient.server.endpoint.exceptions import ( + InternalServerError, + NonXMLResponseError, +) from urllib3 import Retry import datahub.emitter.mce_builder as builder @@ -1196,6 +1199,24 @@ def get_connection_object_page( retry_on_auth_error=False, retries_remaining=retries_remaining - 1, ) + + except InternalServerError as ise: + # In some cases Tableau Server returns 504 error, which is a timeout error, so it worths to retry. + if ise.code == 504: + if retries_remaining <= 0: + raise ise + return self.get_connection_object_page( + query=query, + connection_type=connection_type, + query_filter=query_filter, + fetch_size=fetch_size, + current_cursor=current_cursor, + retry_on_auth_error=False, + retries_remaining=retries_remaining - 1, + ) + else: + raise ise + except OSError: # In tableauseverclient 0.26 (which was yanked and released in 0.28 on 2023-10-04), # the request logic was changed to use threads. From 4d990b06bd0df4f51443893e2efb39e09d9818b6 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Tue, 24 Dec 2024 18:14:51 +0530 Subject: [PATCH 04/27] fix(ingest/snowflake): always ingest view and external table ddl lineage (#12191) --- docs/how/updating-datahub.md | 2 +- .../source/snowflake/snowflake_config.py | 28 ++----------------- .../source/snowflake/snowflake_lineage_v2.py | 13 ++------- .../source/snowflake/snowflake_query.py | 9 ------ .../source/snowflake/snowflake_schema_gen.py | 6 +--- .../source/snowflake/snowflake_shares.py | 2 +- .../source/snowflake/snowflake_v2.py | 20 +++++++++---- .../source_report/ingestion_stage.py | 1 + .../tests/integration/snowflake/common.py | 2 -- .../integration/snowflake/test_snowflake.py | 2 -- .../test_snowflake_classification.py | 1 - .../snowflake/test_snowflake_failures.py | 2 -- .../snowflake/test_snowflake_tag.py | 2 -- .../performance/snowflake/test_snowflake.py | 1 - .../unit/snowflake/test_snowflake_source.py | 23 +++++++-------- 15 files changed, 36 insertions(+), 78 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 5bc0e66fa2ff1..a742ebe0cd896 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -17,7 +17,7 @@ This file documents any backwards-incompatible changes in DataHub and assists people when migrating to a new version. ## Next - +- #12191 - Configs `include_view_lineage` and `include_view_column_lineage` are removed from snowflake ingestion source. View and External Table DDL lineage will always be ingested when definitions are available. - #11560 - The PowerBI ingestion source configuration option include_workspace_name_in_dataset_urn determines whether the workspace name is included in the PowerBI dataset's URN.
PowerBI allows to have identical name of semantic model and their tables across the workspace, It will overwrite the semantic model in-case of multi-workspace ingestion.
Entity urn with `include_workspace_name_in_dataset_urn: false` diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index 1d1cc3c2af4f0..2b2dcf860cdb0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -163,26 +163,13 @@ class SnowflakeConfig( default=True, description="If enabled, populates the snowflake table-to-table and s3-to-snowflake table lineage. Requires appropriate grants given to the role and Snowflake Enterprise Edition or above.", ) - include_view_lineage: bool = pydantic.Field( - default=True, - description="If enabled, populates the snowflake view->table and table->view lineages. Requires appropriate grants given to the role, and include_table_lineage to be True. view->table lineage requires Snowflake Enterprise Edition or above.", - ) + + _include_view_lineage = pydantic_removed_field("include_view_lineage") + _include_view_column_lineage = pydantic_removed_field("include_view_column_lineage") ignore_start_time_lineage: bool = False upstream_lineage_in_report: bool = False - @pydantic.root_validator(skip_on_failure=True) - def validate_include_view_lineage(cls, values): - if ( - "include_table_lineage" in values - and not values.get("include_table_lineage") - and values.get("include_view_lineage") - ): - raise ValueError( - "include_table_lineage must be True for include_view_lineage to be set." - ) - return values - class SnowflakeV2Config( SnowflakeConfig, @@ -222,11 +209,6 @@ class SnowflakeV2Config( description="Populates table->table and view->table column lineage. Requires appropriate grants given to the role and the Snowflake Enterprise Edition or above.", ) - include_view_column_lineage: bool = Field( - default=True, - description="Populates view->view and table->view column lineage using DataHub's sql parser.", - ) - use_queries_v2: bool = Field( default=False, description="If enabled, uses the new queries extractor to extract queries from snowflake.", @@ -355,10 +337,6 @@ def get_sql_alchemy_url( self, database=database, username=username, password=password, role=role ) - @property - def parse_view_ddl(self) -> bool: - return self.include_view_column_lineage - @validator("shares") def validate_shares( cls, shares: Optional[Dict[str, SnowflakeShareConfig]], values: Dict diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py index b815a6584379a..6b200590d7ab6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py @@ -8,7 +8,6 @@ from datahub.configuration.datetimes import parse_absolute_time from datahub.ingestion.api.closeable import Closeable -from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.aws.s3_util import make_s3_urn_for_lineage from datahub.ingestion.source.snowflake.constants import ( LINEAGE_PERMISSION_ERROR, @@ -163,11 +162,11 @@ def get_time_window(self) -> Tuple[datetime, datetime]: self.config.end_time, ) - def get_workunits( + def add_time_based_lineage_to_aggregator( self, discovered_tables: List[str], discovered_views: List[str], - ) -> Iterable[MetadataWorkUnit]: + ) -> None: if not self._should_ingest_lineage(): return @@ -177,9 +176,7 @@ def get_workunits( # snowflake view/table -> snowflake table self.populate_table_upstreams(discovered_tables) - for mcp in self.sql_aggregator.gen_metadata(): - yield mcp.as_workunit() - + def update_state(self): if self.redundant_run_skip_handler: # Update the checkpoint state for this run. self.redundant_run_skip_handler.update_state( @@ -337,10 +334,6 @@ def _fetch_upstream_lineages_for_tables(self) -> Iterable[UpstreamLineageEdge]: start_time_millis=int(self.start_time.timestamp() * 1000), end_time_millis=int(self.end_time.timestamp() * 1000), upstreams_deny_pattern=self.config.temporary_tables_pattern, - # The self.config.include_view_lineage setting is about fetching upstreams of views. - # We always generate lineage pointing at views from tables, even if self.config.include_view_lineage is False. - # TODO: Remove this `include_view_lineage` flag, since it's effectively dead code. - include_view_lineage=True, include_column_lineage=self.config.include_column_lineage, ) try: diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index 97c398c1962d6..a94b39476b2c2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -376,7 +376,6 @@ def view_dependencies() -> str: def table_to_table_lineage_history_v2( start_time_millis: int, end_time_millis: int, - include_view_lineage: bool = True, include_column_lineage: bool = True, upstreams_deny_pattern: List[str] = DEFAULT_TEMP_TABLES_PATTERNS, ) -> str: @@ -385,14 +384,12 @@ def table_to_table_lineage_history_v2( start_time_millis, end_time_millis, upstreams_deny_pattern, - include_view_lineage, ) else: return SnowflakeQuery.table_upstreams_only( start_time_millis, end_time_millis, upstreams_deny_pattern, - include_view_lineage, ) @staticmethod @@ -677,12 +674,9 @@ def table_upstreams_with_column_lineage( start_time_millis: int, end_time_millis: int, upstreams_deny_pattern: List[str], - include_view_lineage: bool = True, ) -> str: allowed_upstream_table_domains = ( SnowflakeQuery.ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER - if include_view_lineage - else SnowflakeQuery.ACCESS_HISTORY_TABLE_DOMAINS_FILTER ) upstream_sql_filter = create_deny_regex_sql_filter( @@ -847,12 +841,9 @@ def table_upstreams_only( start_time_millis: int, end_time_millis: int, upstreams_deny_pattern: List[str], - include_view_lineage: bool = True, ) -> str: allowed_upstream_table_domains = ( SnowflakeQuery.ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER - if include_view_lineage - else SnowflakeQuery.ACCESS_HISTORY_TABLE_DOMAINS_FILTER ) upstream_sql_filter = create_deny_regex_sql_filter( diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 4b72b09fafe2d..8a1bf15b7a7bc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -435,11 +435,7 @@ def _process_schema( ) if self.config.include_views: - if ( - self.aggregator - and self.config.include_view_lineage - and self.config.parse_view_ddl - ): + if self.aggregator: for view in views: view_identifier = self.identifiers.get_dataset_identifier( view.name, schema_name, db_name diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_shares.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_shares.py index 794a6f4a59f46..606acd53dc332 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_shares.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_shares.py @@ -72,7 +72,7 @@ def get_shares_workunits( assert len(sibling_dbs) == 1 # SnowflakeLineageExtractor is unaware of database->schema->table hierarchy # hence this lineage code is not written in SnowflakeLineageExtractor - # also this is not governed by configs include_table_lineage and include_view_lineage + # also this is not governed by configs include_table_lineage yield self.get_upstream_lineage_with_primary_sibling( db.name, schema.name, table_name, sibling_dbs[0] ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 884e6c49f5b62..954e8a29c1a1b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -82,6 +82,7 @@ LINEAGE_EXTRACTION, METADATA_EXTRACTION, QUERIES_EXTRACTION, + VIEW_PARSING, ) from datahub.sql_parsing.sql_parsing_aggregator import SqlParsingAggregator from datahub.utilities.registries.domain_registry import DomainRegistry @@ -103,7 +104,7 @@ @capability(SourceCapability.DESCRIPTIONS, "Enabled by default") @capability( SourceCapability.LINEAGE_COARSE, - "Enabled by default, can be disabled via configuration `include_table_lineage` and `include_view_lineage`", + "Enabled by default, can be disabled via configuration `include_table_lineage`", ) @capability( SourceCapability.LINEAGE_FINE, @@ -512,15 +513,14 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: discovered_datasets = discovered_tables + discovered_views if self.config.use_queries_v2: - self.report.set_ingestion_stage("*", "View Parsing") - assert self.aggregator is not None + self.report.set_ingestion_stage("*", VIEW_PARSING) yield from auto_workunit(self.aggregator.gen_metadata()) self.report.set_ingestion_stage("*", QUERIES_EXTRACTION) schema_resolver = self.aggregator._schema_resolver - queries_extractor: SnowflakeQueriesExtractor = SnowflakeQueriesExtractor( + queries_extractor = SnowflakeQueriesExtractor( connection=self.connection, config=SnowflakeQueriesExtractorConfig( window=self.config, @@ -546,13 +546,21 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: queries_extractor.close() else: - if self.config.include_table_lineage and self.lineage_extractor: + if self.lineage_extractor: self.report.set_ingestion_stage("*", LINEAGE_EXTRACTION) - yield from self.lineage_extractor.get_workunits( + self.lineage_extractor.add_time_based_lineage_to_aggregator( discovered_tables=discovered_tables, discovered_views=discovered_views, ) + # This would emit view and external table ddl lineage + # as well as query lineage via lineage_extractor + for mcp in self.aggregator.gen_metadata(): + yield mcp.as_workunit() + + if self.lineage_extractor: + self.lineage_extractor.update_state() + if ( self.config.include_usage_stats or self.config.include_operational_stats ) and self.usage_extractor: diff --git a/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py b/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py index 92407eaae6e90..42b3b648bd298 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py +++ b/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py @@ -15,6 +15,7 @@ USAGE_EXTRACTION_OPERATIONAL_STATS = "Usage Extraction Operational Stats" USAGE_EXTRACTION_USAGE_AGGREGATION = "Usage Extraction Usage Aggregation" EXTERNAL_TABLE_DDL_LINEAGE = "External table DDL Lineage" +VIEW_PARSING = "View Parsing" QUERIES_EXTRACTION = "Queries Extraction" PROFILING = "Profiling" diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py index 862d27186703a..7b4f5abe1cd46 100644 --- a/metadata-ingestion/tests/integration/snowflake/common.py +++ b/metadata-ingestion/tests/integration/snowflake/common.py @@ -458,7 +458,6 @@ def default_query_results( # noqa: C901 snowflake_query.SnowflakeQuery.table_to_table_lineage_history_v2( start_time_millis=1654473600000, end_time_millis=1654586220000, - include_view_lineage=True, include_column_lineage=True, ), ): @@ -548,7 +547,6 @@ def default_query_results( # noqa: C901 snowflake_query.SnowflakeQuery.table_to_table_lineage_history_v2( start_time_millis=1654473600000, end_time_millis=1654586220000, - include_view_lineage=True, include_column_lineage=False, ), ): diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py index 1d7470d24f768..ef4918a20e640 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py @@ -117,7 +117,6 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): schema_pattern=AllowDenyPattern(allow=["test_db.test_schema"]), include_technical_schema=True, include_table_lineage=True, - include_view_lineage=True, include_usage_stats=True, format_sql_queries=True, validate_upstreams_against_patterns=False, @@ -216,7 +215,6 @@ def test_snowflake_private_link_and_incremental_mcps( include_table_lineage=True, include_column_lineage=False, include_views=True, - include_view_lineage=True, include_usage_stats=False, format_sql_queries=True, incremental_lineage=False, diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_classification.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_classification.py index 75a9df4f28051..52453b30f740a 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_classification.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_classification.py @@ -66,7 +66,6 @@ def test_snowflake_classification_perf(num_workers, num_cols_per_table, num_tabl schema_pattern=AllowDenyPattern(allow=["test_db.test_schema"]), include_technical_schema=True, include_table_lineage=False, - include_view_lineage=False, include_column_lineage=False, include_usage_stats=False, include_operational_stats=False, diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py index 0b838b0bb59c3..de6e996a52642 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py @@ -49,7 +49,6 @@ def snowflake_pipeline_config(tmp_path): include_technical_schema=True, match_fully_qualified_names=True, schema_pattern=AllowDenyPattern(allow=["test_db.test_schema"]), - include_view_lineage=False, include_usage_stats=False, start_time=datetime(2022, 6, 6, 0, 0, 0, 0).replace( tzinfo=timezone.utc @@ -227,7 +226,6 @@ def test_snowflake_missing_snowflake_lineage_permission_causes_pipeline_failure( snowflake_query.SnowflakeQuery.table_to_table_lineage_history_v2( start_time_millis=1654473600000, end_time_millis=1654586220000, - include_view_lineage=True, include_column_lineage=True, ) ], diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_tag.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_tag.py index d5e265e783882..9bb598cb0c1c7 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_tag.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_tag.py @@ -30,7 +30,6 @@ def test_snowflake_tag_pattern(): ), include_technical_schema=True, include_table_lineage=False, - include_view_lineage=False, include_column_lineage=False, include_usage_stats=False, include_operational_stats=False, @@ -74,7 +73,6 @@ def test_snowflake_tag_pattern_deny(): ), include_technical_schema=True, include_table_lineage=False, - include_view_lineage=False, include_column_lineage=False, include_usage_stats=False, include_operational_stats=False, diff --git a/metadata-ingestion/tests/performance/snowflake/test_snowflake.py b/metadata-ingestion/tests/performance/snowflake/test_snowflake.py index 5042c78c2e7b9..984d9e4295745 100644 --- a/metadata-ingestion/tests/performance/snowflake/test_snowflake.py +++ b/metadata-ingestion/tests/performance/snowflake/test_snowflake.py @@ -37,7 +37,6 @@ def run_test(): password="TST_PWD", include_technical_schema=False, include_table_lineage=True, - include_view_lineage=True, include_usage_stats=True, include_operational_stats=True, start_time=datetime(2022, 6, 6, 0, 0, 0, 0).replace(tzinfo=timezone.utc), diff --git a/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py b/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py index 2ff85a08f052f..75f32b535eb2e 100644 --- a/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py +++ b/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py @@ -257,17 +257,6 @@ def test_options_contain_connect_args(): assert connect_args is not None -def test_snowflake_config_with_view_lineage_no_table_lineage_throws_error(): - config_dict = default_config_dict.copy() - config_dict["include_view_lineage"] = True - config_dict["include_table_lineage"] = False - with pytest.raises( - ValidationError, - match="include_table_lineage must be True for include_view_lineage to be set", - ): - SnowflakeV2Config.parse_obj(config_dict) - - def test_snowflake_config_with_column_lineage_no_table_lineage_throws_error(): config_dict = default_config_dict.copy() config_dict["include_column_lineage"] = True @@ -667,6 +656,18 @@ def test_snowflake_utils() -> None: assert_doctest(datahub.ingestion.source.snowflake.snowflake_utils) +def test_using_removed_fields_causes_no_error() -> None: + assert SnowflakeV2Config.parse_obj( + { + "account_id": "test", + "username": "snowflake", + "password": "snowflake", + "include_view_lineage": "true", + "include_view_column_lineage": "true", + } + ) + + def test_snowflake_query_result_parsing(): db_row = { "DOWNSTREAM_TABLE_NAME": "db.schema.downstream_table", From d88e6c997713509d8ecdb463c42d072c5c857853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Tue, 24 Dec 2024 16:03:36 +0100 Subject: [PATCH 05/27] fix(tableau): fixes wrong argument when reauthenticating (#12216) --- .../ingestion/source/tableau/tableau.py | 48 +++++++++++-------- .../tableau/test_tableau_ingest.py | 10 ++-- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 2b7aac2bea1d0..508500ffe489b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -621,6 +621,12 @@ def update_table( self.parsed_columns = parsed_columns +@dataclass +class SiteIdContentUrl: + site_id: str + site_content_url: str + + class TableauSourceReport(StaleEntityRemovalSourceReport): get_all_datasources_query_failed: bool = False num_get_datasource_query_failures: int = 0 @@ -773,7 +779,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: config=self.config, ctx=self.ctx, site=site, - site_id=site.id, report=self.report, server=self.server, platform=self.platform, @@ -792,8 +797,11 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: site_source = TableauSiteSource( config=self.config, ctx=self.ctx, - site=site, - site_id=self.server.site_id, + site=site + if site + else SiteIdContentUrl( + site_id=self.server.site_id, site_content_url=self.config.site + ), report=self.report, server=self.server, platform=self.platform, @@ -826,8 +834,7 @@ def __init__( self, config: TableauConfig, ctx: PipelineContext, - site: Optional[SiteItem], - site_id: Optional[str], + site: Union[SiteItem, SiteIdContentUrl], report: TableauSourceReport, server: Server, platform: str, @@ -838,13 +845,18 @@ def __init__( self.ctx: PipelineContext = ctx self.platform = platform - self.site: Optional[SiteItem] = site - if site_id is not None: - self.site_id: str = site_id + self.site: Optional[SiteItem] = None + if isinstance(site, SiteItem): + self.site = site + assert site.id is not None, "Site ID is required" + self.site_id = site.id + self.site_content_url = site.content_url + elif isinstance(site, SiteIdContentUrl): + self.site = None + self.site_id = site.site_id + self.site_content_url = site.site_content_url else: - assert self.site is not None, "site or site_id is required" - assert self.site.id is not None, "site_id is required when site is provided" - self.site_id = self.site.id + raise AssertionError("site or site id+content_url pair is required") self.database_tables: Dict[str, DatabaseTable] = {} self.tableau_stat_registry: Dict[str, UsageStat] = {} @@ -898,16 +910,14 @@ def dataset_browse_prefix(self) -> str: # datasets also have the env in the browse path return f"/{self.config.env.lower()}{self.no_env_browse_prefix}" - def _re_authenticate(self): + def _re_authenticate(self) -> None: + self.report.info( + message="Re-authenticating to Tableau", + context=f"site='{self.site_content_url}'", + ) # Sign-in again may not be enough because Tableau sometimes caches invalid sessions # so we need to recreate the Tableau Server object - self.server = self.config.make_tableau_client(self.site_id) - - @property - def site_content_url(self) -> Optional[str]: - if self.site and self.site.content_url: - return self.site.content_url - return None + self.server = self.config.make_tableau_client(self.site_content_url) def _populate_usage_stat_registry(self) -> None: if self.server is None: diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index c3a8880bf20a0..902ff243c802a 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -26,6 +26,7 @@ from datahub.ingestion.run.pipeline import Pipeline, PipelineContext from datahub.ingestion.source.tableau import tableau_constant as c from datahub.ingestion.source.tableau.tableau import ( + SiteIdContentUrl, TableauConfig, TableauProject, TableauSiteSource, @@ -1008,8 +1009,7 @@ def check_lineage_metadata( config=config, ctx=context, platform="tableau", - site=SiteItem(name="Site 1", content_url="site1"), - site_id="site1", + site=SiteIdContentUrl(site_id="id1", site_content_url="site1"), report=TableauSourceReport(), server=Server("https://test-tableau-server.com"), ) @@ -1313,8 +1313,7 @@ def test_permission_warning(pytestconfig, tmp_path, mock_datahub_graph): platform="tableau", config=mock.MagicMock(), ctx=mock.MagicMock(), - site=mock.MagicMock(), - site_id=None, + site=mock.MagicMock(spec=SiteItem, id="Site1", content_url="site1"), server=mock_sdk.return_value, report=reporter, ) @@ -1371,8 +1370,7 @@ def test_extract_project_hierarchy(extract_project_hierarchy, allowed_projects): config=config, ctx=context, platform="tableau", - site=SiteItem(name="Site 1", content_url="site1"), - site_id="site1", + site=mock.MagicMock(spec=SiteItem, id="Site1", content_url="site1"), report=TableauSourceReport(), server=Server("https://test-tableau-server.com"), ) From 48736a03dd56c70a3894efcbef6e95a23d8cbfdd Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware <159135491+sagar-salvi-apptware@users.noreply.github.com> Date: Wed, 25 Dec 2024 00:57:27 +0530 Subject: [PATCH 06/27] fix(ingest/looker): Add flag for Looker metadata extraction (#12205) Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> --- .../src/datahub/sql_parsing/tool_meta_extractor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py b/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py index 5af9d9d4f0fff..d2682252e0fbf 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py +++ b/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py @@ -40,6 +40,7 @@ def _get_last_line(query: str) -> str: class ToolMetaExtractorReport(Report): num_queries_meta_extracted: Dict[str, int] = field(default_factory=int_top_k_dict) failures: List[str] = field(default_factory=list) + looker_user_mapping_missing: Optional[bool] = None class ToolMetaExtractor: @@ -108,7 +109,9 @@ def extract_looker_user_mapping_from_graph( PlatformResource.search_by_filters(query=query, graph_client=graph) ) - if len(platform_resources) > 1: + if len(platform_resources) == 0: + report.looker_user_mapping_missing = True + elif len(platform_resources) > 1: report.failures.append( "Looker user metadata extraction failed. Found more than one looker user id mappings." ) From f4b33b59d1726dd962db4d3300f085cc60626a81 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Tue, 24 Dec 2024 11:33:06 -0800 Subject: [PATCH 07/27] fix(ingest/mode): Handle 204 response and invalid json (#12156) Co-authored-by: Aseem Bansal --- .../src/datahub/ingestion/source/mode.py | 46 +++--- .../tests/integration/mode/test_mode.py | 141 ++++++++++++++++-- 2 files changed, 151 insertions(+), 36 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py index ef0b499129f97..68ecc5d8694ac 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mode.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mode.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from functools import lru_cache +from json import JSONDecodeError from typing import Dict, Iterable, List, Optional, Set, Tuple, Union import dateutil.parser as dp @@ -193,6 +194,9 @@ class HTTPError429(HTTPError): pass +ModeRequestError = (HTTPError, JSONDecodeError) + + @dataclass class ModeSourceReport(StaleEntityRemovalSourceReport): filtered_spaces: LossyList[str] = dataclasses.field(default_factory=LossyList) @@ -328,11 +332,11 @@ def __init__(self, ctx: PipelineContext, config: ModeConfig): # Test the connection try: self._get_request_json(f"{self.config.connect_uri}/api/verify") - except HTTPError as http_error: + except ModeRequestError as e: self.report.report_failure( title="Failed to Connect", message="Unable to verify connection to mode.", - context=f"Error: {str(http_error)}", + context=f"Error: {str(e)}", ) self.workspace_uri = f"{self.config.connect_uri}/api/{self.config.workspace}" @@ -521,11 +525,11 @@ def _get_creator(self, href: str) -> Optional[str]: if self.config.owner_username_instead_of_email else user_json.get("email") ) - except HTTPError as http_error: + except ModeRequestError as e: self.report.report_warning( title="Failed to retrieve Mode creator", message=f"Unable to retrieve user for {href}", - context=f"Reason: {str(http_error)}", + context=f"Reason: {str(e)}", ) return user @@ -571,11 +575,11 @@ def _get_space_name_and_tokens(self) -> dict: logging.debug(f"Skipping space {space_name} due to space pattern") continue space_info[s.get("token", "")] = s.get("name", "") - except HTTPError as http_error: + except ModeRequestError as e: self.report.report_failure( title="Failed to Retrieve Spaces", message="Unable to retrieve spaces / collections for workspace.", - context=f"Workspace: {self.workspace_uri}, Error: {str(http_error)}", + context=f"Workspace: {self.workspace_uri}, Error: {str(e)}", ) return space_info @@ -721,11 +725,11 @@ def _get_data_sources(self) -> List[dict]: try: ds_json = self._get_request_json(f"{self.workspace_uri}/data_sources") data_sources = ds_json.get("_embedded", {}).get("data_sources", []) - except HTTPError as http_error: + except ModeRequestError as e: self.report.report_failure( title="Failed to retrieve Data Sources", message="Unable to retrieve data sources from Mode.", - context=f"Error: {str(http_error)}", + context=f"Error: {str(e)}", ) return data_sources @@ -812,11 +816,11 @@ def _get_definition(self, definition_name): if definition.get("name", "") == definition_name: return definition.get("source", "") - except HTTPError as http_error: + except ModeRequestError as e: self.report.report_failure( title="Failed to Retrieve Definition", message="Unable to retrieve definition from Mode.", - context=f"Definition Name: {definition_name}, Error: {str(http_error)}", + context=f"Definition Name: {definition_name}, Error: {str(e)}", ) return None @@ -1382,11 +1386,11 @@ def _get_reports(self, space_token: str) -> List[dict]: f"{self.workspace_uri}/spaces/{space_token}/reports" ) reports = reports_json.get("_embedded", {}).get("reports", {}) - except HTTPError as http_error: + except ModeRequestError as e: self.report.report_failure( title="Failed to Retrieve Reports for Space", message="Unable to retrieve reports for space token.", - context=f"Space Token: {space_token}, Error: {str(http_error)}", + context=f"Space Token: {space_token}, Error: {str(e)}", ) return reports @@ -1400,11 +1404,11 @@ def _get_datasets(self, space_token: str) -> List[dict]: url = f"{self.workspace_uri}/spaces/{space_token}/datasets" datasets_json = self._get_request_json(url) datasets = datasets_json.get("_embedded", {}).get("reports", []) - except HTTPError as http_error: + except ModeRequestError as e: self.report.report_failure( title="Failed to Retrieve Datasets for Space", message=f"Unable to retrieve datasets for space token {space_token}.", - context=f"Error: {str(http_error)}", + context=f"Error: {str(e)}", ) return datasets @@ -1416,11 +1420,11 @@ def _get_queries(self, report_token: str) -> list: f"{self.workspace_uri}/reports/{report_token}/queries" ) queries = queries_json.get("_embedded", {}).get("queries", {}) - except HTTPError as http_error: + except ModeRequestError as e: self.report.report_failure( title="Failed to Retrieve Queries", message="Unable to retrieve queries for report token.", - context=f"Report Token: {report_token}, Error: {str(http_error)}", + context=f"Report Token: {report_token}, Error: {str(e)}", ) return queries @@ -1433,11 +1437,11 @@ def _get_last_query_run( f"{self.workspace_uri}/reports/{report_token}/runs/{report_run_id}/query_runs{query_run_id}" ) queries = queries_json.get("_embedded", {}).get("queries", {}) - except HTTPError as http_error: + except ModeRequestError as e: self.report.report_failure( title="Failed to Retrieve Queries for Report", message="Unable to retrieve queries for report token.", - context=f"Report Token:{report_token}, Error: {str(http_error)}", + context=f"Report Token:{report_token}, Error: {str(e)}", ) return {} return queries @@ -1451,13 +1455,13 @@ def _get_charts(self, report_token: str, query_token: str) -> list: f"/queries/{query_token}/charts" ) charts = charts_json.get("_embedded", {}).get("charts", {}) - except HTTPError as http_error: + except ModeRequestError as e: self.report.report_failure( title="Failed to Retrieve Charts", message="Unable to retrieve charts from Mode.", context=f"Report Token: {report_token}, " f"Query token: {query_token}, " - f"Error: {str(http_error)}", + f"Error: {str(e)}", ) return charts @@ -1477,6 +1481,8 @@ def get_request(): response = self.session.get( url, timeout=self.config.api_options.timeout ) + if response.status_code == 204: # No content, don't parse json + return {} return response.json() except HTTPError as http_error: error_response = http_error.response diff --git a/metadata-ingestion/tests/integration/mode/test_mode.py b/metadata-ingestion/tests/integration/mode/test_mode.py index ce7533d5611e4..7f1e3935aa0fa 100644 --- a/metadata-ingestion/tests/integration/mode/test_mode.py +++ b/metadata-ingestion/tests/integration/mode/test_mode.py @@ -1,11 +1,14 @@ import json import pathlib +from typing import Sequence from unittest.mock import patch +import pytest from freezegun import freeze_time from requests.models import HTTPError from datahub.configuration.common import PipelineExecutionError +from datahub.ingestion.api.source import StructuredLogEntry from datahub.ingestion.run.pipeline import Pipeline from tests.test_helpers import mce_helpers @@ -28,7 +31,7 @@ "https://app.mode.com/api/acryl/reports/24f66e1701b6/queries": "dataset_queries_24f66e1701b6.json", } -RESPONSE_ERROR_LIST = ["https://app.mode.com/api/acryl/spaces/75737b70402e/reports"] +ERROR_URL = "https://app.mode.com/api/acryl/spaces/75737b70402e/reports" test_resources_dir = pathlib.Path(__file__).parent @@ -49,6 +52,14 @@ def mount(self, prefix, adaptor): return self def get(self, url, timeout=40): + if self.error_list is not None and self.url in self.error_list: + http_error_msg = "{} Client Error: {} for url: {}".format( + 400, + "Simulate error", + self.url, + ) + raise HTTPError(http_error_msg, response=self) + self.url = url self.timeout = timeout response_json_path = f"{test_resources_dir}/setup/{JSON_RESPONSE_MAP.get(url)}" @@ -57,29 +68,46 @@ def get(self, url, timeout=40): self.json_data = data return self - def raise_for_status(self): - if self.error_list is not None and self.url in self.error_list: - http_error_msg = "{} Client Error: {} for url: {}".format( - 400, - "Simulate error", - self.url, - ) - raise HTTPError(http_error_msg, response=self) + +class MockResponseJson(MockResponse): + def __init__( + self, + status_code: int = 200, + *, + json_empty_list: Sequence[str] = (), + json_error_list: Sequence[str] = (), + ): + super().__init__(None, status_code) + self.json_empty_list = json_empty_list + self.json_error_list = json_error_list + + def json(self): + if self.url in self.json_empty_list: + return json.loads("") # Shouldn't be called + if self.url in self.json_error_list: + return json.loads("{") + return super().json() + + def get(self, url, timeout=40): + response = super().get(url, timeout) + if self.url in self.json_empty_list: + response.status_code = 204 + return response -def mocked_requests_sucess(*args, **kwargs): +def mocked_requests_success(*args, **kwargs): return MockResponse(None, 200) def mocked_requests_failure(*args, **kwargs): - return MockResponse(RESPONSE_ERROR_LIST, 200) + return MockResponse([ERROR_URL], 200) @freeze_time(FROZEN_TIME) def test_mode_ingest_success(pytestconfig, tmp_path): with patch( "datahub.ingestion.source.mode.requests.Session", - side_effect=mocked_requests_sucess, + side_effect=mocked_requests_success, ): pipeline = Pipeline.create( { @@ -142,8 +170,89 @@ def test_mode_ingest_failure(pytestconfig, tmp_path): } ) pipeline.run() - try: + with pytest.raises(PipelineExecutionError) as exec_error: pipeline.raise_from_status() - except PipelineExecutionError as exec_error: - assert exec_error.args[0] == "Source reported errors" - assert len(exec_error.args[1].failures) == 1 + assert exec_error.value.args[0] == "Source reported errors" + assert len(exec_error.value.args[1].failures) == 1 + error_dict: StructuredLogEntry + _level, error_dict = exec_error.value.args[1].failures[0] + error = next(iter(error_dict.context)) + assert "Simulate error" in error + assert ERROR_URL in error + + +@freeze_time(FROZEN_TIME) +def test_mode_ingest_json_empty(pytestconfig, tmp_path): + with patch( + "datahub.ingestion.source.mode.requests.Session", + side_effect=lambda *args, **kwargs: MockResponseJson( + json_empty_list=["https://app.mode.com/api/modeuser"] + ), + ): + global test_resources_dir + test_resources_dir = pytestconfig.rootpath / "tests/integration/mode" + + pipeline = Pipeline.create( + { + "run_id": "mode-test", + "source": { + "type": "mode", + "config": { + "token": "xxxx", + "password": "xxxx", + "connect_uri": "https://app.mode.com/", + "workspace": "acryl", + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/mode_mces.json", + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status(raise_warnings=True) + + +@freeze_time(FROZEN_TIME) +def test_mode_ingest_json_failure(pytestconfig, tmp_path): + with patch( + "datahub.ingestion.source.mode.requests.Session", + side_effect=lambda *args, **kwargs: MockResponseJson( + json_error_list=["https://app.mode.com/api/modeuser"] + ), + ): + global test_resources_dir + test_resources_dir = pytestconfig.rootpath / "tests/integration/mode" + + pipeline = Pipeline.create( + { + "run_id": "mode-test", + "source": { + "type": "mode", + "config": { + "token": "xxxx", + "password": "xxxx", + "connect_uri": "https://app.mode.com/", + "workspace": "acryl", + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/mode_mces.json", + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status(raise_warnings=False) + with pytest.raises(PipelineExecutionError) as exec_error: + pipeline.raise_from_status(raise_warnings=True) + assert len(exec_error.value.args[1].warnings) > 0 + error_dict: StructuredLogEntry + _level, error_dict = exec_error.value.args[1].warnings[0] + error = next(iter(error_dict.context)) + assert "Expecting property name enclosed in double quotes" in error From 756b199506d57449c60a3a28901f7d22fe89f9f1 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Tue, 24 Dec 2024 14:56:35 -0800 Subject: [PATCH 08/27] fix(ingest/glue): Add additional checks and logging when specifying catalog_id (#12168) --- .../src/datahub/ingestion/source/aws/glue.py | 14 +++++- .../tests/unit/glue/glue_mces_golden.json | 2 +- .../glue/glue_mces_golden_table_lineage.json | 2 +- .../glue_mces_platform_instance_golden.json | 2 +- .../tests/unit/glue/test_glue_source.py | 43 +++++++++++++++++-- .../tests/unit/glue/test_glue_source_stubs.py | 8 ++-- 6 files changed, 59 insertions(+), 12 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index 37c146218e263..7a5ed154d40bc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -52,6 +52,7 @@ platform_name, support_status, ) +from datahub.ingestion.api.report import EntityFilterReport from datahub.ingestion.api.source import MetadataWorkUnitProcessor from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.aws import s3_util @@ -115,7 +116,6 @@ logger = logging.getLogger(__name__) - DEFAULT_PLATFORM = "glue" VALID_PLATFORMS = [DEFAULT_PLATFORM, "athena"] @@ -220,6 +220,7 @@ def platform_validator(cls, v: str) -> str: class GlueSourceReport(StaleEntityRemovalSourceReport): tables_scanned = 0 filtered: List[str] = dataclass_field(default_factory=list) + databases: EntityFilterReport = EntityFilterReport.field(type="database") num_job_script_location_missing: int = 0 num_job_script_location_invalid: int = 0 @@ -668,6 +669,7 @@ def get_datajob_wu(self, node: Dict[str, Any], job_name: str) -> MetadataWorkUni return MetadataWorkUnit(id=f'{job_name}-{node["Id"]}', mce=mce) def get_all_databases(self) -> Iterable[Mapping[str, Any]]: + logger.debug("Getting all databases") # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/glue/paginator/GetDatabases.html paginator = self.glue_client.get_paginator("get_databases") @@ -684,10 +686,18 @@ def get_all_databases(self) -> Iterable[Mapping[str, Any]]: pattern += "[?!TargetDatabase]" for database in paginator_response.search(pattern): - if self.source_config.database_pattern.allowed(database["Name"]): + if (not self.source_config.database_pattern.allowed(database["Name"])) or ( + self.source_config.catalog_id + and database.get("CatalogId") + and database.get("CatalogId") != self.source_config.catalog_id + ): + self.report.databases.dropped(database["Name"]) + else: + self.report.databases.processed(database["Name"]) yield database def get_tables_from_database(self, database: Mapping[str, Any]) -> Iterable[Dict]: + logger.debug(f"Getting tables from database {database['Name']}") # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/glue/paginator/GetTables.html paginator = self.glue_client.get_paginator("get_tables") database_name = database["Name"] diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index 87971de12fbb3..71d7c31b222bd 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -124,7 +124,7 @@ "CreateTime": "June 01, 2021 at 14:55:13" }, "name": "empty-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database", + "qualifiedName": "arn:aws:glue:us-west-2:000000000000:database/empty-database", "env": "PROD" } } diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_lineage.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_lineage.json index e2dd4cec97c2e..22bb4b53b91ef 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_lineage.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_lineage.json @@ -124,7 +124,7 @@ "CreateTime": "June 01, 2021 at 14:55:13" }, "name": "empty-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database", + "qualifiedName": "arn:aws:glue:us-west-2:000000000000:database/empty-database", "env": "PROD" } } diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index 0b883062763f4..b700335c26e5a 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -129,7 +129,7 @@ "CreateTime": "June 01, 2021 at 14:55:13" }, "name": "empty-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database", + "qualifiedName": "arn:aws:glue:us-west-2:000000000000:database/empty-database", "env": "PROD" } } diff --git a/metadata-ingestion/tests/unit/glue/test_glue_source.py b/metadata-ingestion/tests/unit/glue/test_glue_source.py index 693fd6bc336fd..9e3f260a23f1c 100644 --- a/metadata-ingestion/tests/unit/glue/test_glue_source.py +++ b/metadata-ingestion/tests/unit/glue/test_glue_source.py @@ -35,8 +35,8 @@ validate_all_providers_have_committed_successfully, ) from tests.unit.glue.test_glue_source_stubs import ( - databases_1, - databases_2, + empty_database, + flights_database, get_bucket_tagging, get_databases_delta_response, get_databases_response, @@ -64,6 +64,7 @@ tables_2, tables_profiling_1, target_database_tables, + test_database, ) FROZEN_TIME = "2020-04-14 07:00:00" @@ -310,6 +311,40 @@ def test_config_without_platform(): assert source.platform == "glue" +def test_get_databases_filters_by_catalog(): + def format_databases(databases): + return set(d["Name"] for d in databases) + + all_catalogs_source: GlueSource = GlueSource( + config=GlueSourceConfig(aws_region="us-west-2"), + ctx=PipelineContext(run_id="glue-source-test"), + ) + with Stubber(all_catalogs_source.glue_client) as glue_stubber: + glue_stubber.add_response("get_databases", get_databases_response, {}) + + expected = [flights_database, test_database, empty_database] + actual = all_catalogs_source.get_all_databases() + assert format_databases(actual) == format_databases(expected) + assert all_catalogs_source.report.databases.dropped_entities.as_obj() == [] + + catalog_id = "123412341234" + single_catalog_source: GlueSource = GlueSource( + config=GlueSourceConfig(catalog_id=catalog_id, aws_region="us-west-2"), + ctx=PipelineContext(run_id="glue-source-test"), + ) + with Stubber(single_catalog_source.glue_client) as glue_stubber: + glue_stubber.add_response( + "get_databases", get_databases_response, {"CatalogId": catalog_id} + ) + + expected = [flights_database, test_database] + actual = single_catalog_source.get_all_databases() + assert format_databases(actual) == format_databases(expected) + assert single_catalog_source.report.databases.dropped_entities.as_obj() == [ + "empty-database" + ] + + @freeze_time(FROZEN_TIME) def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): deleted_actor_golden_mcs = "{}/glue_deleted_actor_mces_golden.json".format( @@ -357,8 +392,8 @@ def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): tables_on_first_call = tables_1 tables_on_second_call = tables_2 mock_get_all_databases_and_tables.side_effect = [ - (databases_1, tables_on_first_call), - (databases_2, tables_on_second_call), + ([flights_database], tables_on_first_call), + ([test_database], tables_on_second_call), ] pipeline_run1 = run_and_get_pipeline(pipeline_config_dict) diff --git a/metadata-ingestion/tests/unit/glue/test_glue_source_stubs.py b/metadata-ingestion/tests/unit/glue/test_glue_source_stubs.py index dba1eea3010c2..43bf62fd4e3b8 100644 --- a/metadata-ingestion/tests/unit/glue/test_glue_source_stubs.py +++ b/metadata-ingestion/tests/unit/glue/test_glue_source_stubs.py @@ -88,12 +88,14 @@ "Permissions": ["ALL"], } ], - "CatalogId": "123412341234", + "CatalogId": "000000000000", }, ] } -databases_1 = [{"Name": "flights-database", "CatalogId": "123412341234"}] -databases_2 = [{"Name": "test-database", "CatalogId": "123412341234"}] +flights_database = {"Name": "flights-database", "CatalogId": "123412341234"} +test_database = {"Name": "test-database", "CatalogId": "123412341234"} +empty_database = {"Name": "empty-database", "CatalogId": "000000000000"} + tables_1 = [ { "Name": "avro", From 16698da509ec5c9f86188db7a16c38cea19d61bf Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Thu, 26 Dec 2024 18:10:09 +0530 Subject: [PATCH 09/27] fix(ingest/gc): misc fixes in gc source (#12226) --- .../datahub/ingestion/source/gc/datahub_gc.py | 23 +++++-- .../source/gc/execution_request_cleanup.py | 61 +++++++++++++++---- .../source_report/ingestion_stage.py | 1 + 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py index 4eecbb4d9d717..168b787b85e8b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py @@ -34,6 +34,7 @@ SoftDeletedEntitiesCleanupConfig, SoftDeletedEntitiesReport, ) +from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport logger = logging.getLogger(__name__) @@ -86,6 +87,7 @@ class DataHubGcSourceReport( DataProcessCleanupReport, SoftDeletedEntitiesReport, DatahubExecutionRequestCleanupReport, + IngestionStageReport, ): expired_tokens_revoked: int = 0 @@ -139,31 +141,40 @@ def get_workunits_internal( ) -> Iterable[MetadataWorkUnit]: if self.config.cleanup_expired_tokens: try: + self.report.report_ingestion_stage_start("Expired Token Cleanup") self.revoke_expired_tokens() except Exception as e: self.report.failure("While trying to cleanup expired token ", exc=e) if self.config.truncate_indices: try: + self.report.report_ingestion_stage_start("Truncate Indices") self.truncate_indices() except Exception as e: self.report.failure("While trying to truncate indices ", exc=e) if self.config.soft_deleted_entities_cleanup.enabled: try: + self.report.report_ingestion_stage_start( + "Soft Deleted Entities Cleanup" + ) self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities() except Exception as e: self.report.failure( "While trying to cleanup soft deleted entities ", exc=e ) - if self.config.execution_request_cleanup.enabled: - try: - self.execution_request_cleanup.run() - except Exception as e: - self.report.failure("While trying to cleanup execution request ", exc=e) if self.config.dataprocess_cleanup.enabled: try: + self.report.report_ingestion_stage_start("Data Process Cleanup") yield from self.dataprocess_cleanup.get_workunits_internal() except Exception as e: self.report.failure("While trying to cleanup data process ", exc=e) + if self.config.execution_request_cleanup.enabled: + try: + self.report.report_ingestion_stage_start("Execution request Cleanup") + self.execution_request_cleanup.run() + except Exception as e: + self.report.failure("While trying to cleanup execution request ", exc=e) + # Otherwise last stage's duration does not get calculated. + self.report.report_ingestion_stage_start("End") yield from [] def truncate_indices(self) -> None: @@ -281,6 +292,8 @@ def revoke_expired_tokens(self) -> None: list_access_tokens = expired_tokens_res.get("listAccessTokens", {}) tokens = list_access_tokens.get("tokens", []) total = list_access_tokens.get("total", 0) + if tokens == []: + break for token in tokens: self.report.expired_tokens_revoked += 1 token_id = token["id"] diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py index 3baf858e44cdc..170a6ada3e336 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py @@ -1,3 +1,4 @@ +import datetime import logging import time from typing import Any, Dict, Iterator, Optional @@ -42,16 +43,28 @@ class DatahubExecutionRequestCleanupConfig(ConfigModel): description="Global switch for this cleanup task", ) + runtime_limit_seconds: int = Field( + default=3600, + description="Maximum runtime in seconds for the cleanup task", + ) + + max_read_errors: int = Field( + default=10, + description="Maximum number of read errors before aborting", + ) + def keep_history_max_milliseconds(self): return self.keep_history_max_days * 24 * 3600 * 1000 class DatahubExecutionRequestCleanupReport(SourceReport): - execution_request_cleanup_records_read: int = 0 - execution_request_cleanup_records_preserved: int = 0 - execution_request_cleanup_records_deleted: int = 0 - execution_request_cleanup_read_errors: int = 0 - execution_request_cleanup_delete_errors: int = 0 + ergc_records_read: int = 0 + ergc_records_preserved: int = 0 + ergc_records_deleted: int = 0 + ergc_read_errors: int = 0 + ergc_delete_errors: int = 0 + ergc_start_time: Optional[datetime.datetime] = None + ergc_end_time: Optional[datetime.datetime] = None class CleanupRecord(BaseModel): @@ -124,6 +137,13 @@ def _scroll_execution_requests( params.update(overrides) while True: + if self._reached_runtime_limit(): + break + if self.report.ergc_read_errors >= self.config.max_read_errors: + self.report.failure( + f"ergc({self.instance_id}): too many read errors, aborting." + ) + break try: url = f"{self.graph.config.server}/openapi/v2/entity/{DATAHUB_EXECUTION_REQUEST_ENTITY_NAME}" response = self.graph._session.get(url, headers=headers, params=params) @@ -141,7 +161,7 @@ def _scroll_execution_requests( logger.error( f"ergc({self.instance_id}): failed to fetch next batch of execution requests: {e}" ) - self.report.execution_request_cleanup_read_errors += 1 + self.report.ergc_read_errors += 1 def _scroll_garbage_records(self): state: Dict[str, Dict] = {} @@ -150,7 +170,7 @@ def _scroll_garbage_records(self): running_guard_timeout = now_ms - 30 * 24 * 3600 * 1000 for entry in self._scroll_execution_requests(): - self.report.execution_request_cleanup_records_read += 1 + self.report.ergc_records_read += 1 key = entry.ingestion_source # Always delete corrupted records @@ -171,7 +191,7 @@ def _scroll_garbage_records(self): # Do not delete if number of requests is below minimum if state[key]["count"] < self.config.keep_history_min_count: - self.report.execution_request_cleanup_records_preserved += 1 + self.report.ergc_records_preserved += 1 continue # Do not delete if number of requests do not exceed allowed maximum, @@ -179,7 +199,7 @@ def _scroll_garbage_records(self): if (state[key]["count"] < self.config.keep_history_max_count) and ( entry.requested_at > state[key]["cutoffTimestamp"] ): - self.report.execution_request_cleanup_records_preserved += 1 + self.report.ergc_records_preserved += 1 continue # Do not delete if status is RUNNING or PENDING and created within last month. If the record is >month old and it did not @@ -188,7 +208,7 @@ def _scroll_garbage_records(self): "RUNNING", "PENDING", ]: - self.report.execution_request_cleanup_records_preserved += 1 + self.report.ergc_records_preserved += 1 continue # Otherwise delete current record @@ -200,7 +220,7 @@ def _scroll_garbage_records(self): f"record timestamp: {entry.requested_at}." ) ) - self.report.execution_request_cleanup_records_deleted += 1 + self.report.ergc_records_deleted += 1 yield entry def _delete_entry(self, entry: CleanupRecord) -> None: @@ -210,17 +230,31 @@ def _delete_entry(self, entry: CleanupRecord) -> None: ) self.graph.delete_entity(entry.urn, True) except Exception as e: - self.report.execution_request_cleanup_delete_errors += 1 + self.report.ergc_delete_errors += 1 logger.error( f"ergc({self.instance_id}): failed to delete ExecutionRequest {entry.request_id}: {e}" ) + def _reached_runtime_limit(self) -> bool: + if ( + self.config.runtime_limit_seconds + and self.report.ergc_start_time + and ( + datetime.datetime.now() - self.report.ergc_start_time + >= datetime.timedelta(seconds=self.config.runtime_limit_seconds) + ) + ): + logger.info(f"ergc({self.instance_id}): max runtime reached.") + return True + return False + def run(self) -> None: if not self.config.enabled: logger.info( f"ergc({self.instance_id}): ExecutionRequest cleaner is disabled." ) return + self.report.ergc_start_time = datetime.datetime.now() logger.info( ( @@ -232,8 +266,11 @@ def run(self) -> None: ) for entry in self._scroll_garbage_records(): + if self._reached_runtime_limit(): + break self._delete_entry(entry) + self.report.ergc_end_time = datetime.datetime.now() logger.info( f"ergc({self.instance_id}): Finished cleanup of ExecutionRequest records." ) diff --git a/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py b/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py index 42b3b648bd298..ce683e64b3f46 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py +++ b/metadata-ingestion/src/datahub/ingestion/source_report/ingestion_stage.py @@ -42,4 +42,5 @@ def report_ingestion_stage_start(self, stage: str) -> None: self._timer = PerfTimer() self.ingestion_stage = f"{stage} at {datetime.now(timezone.utc)}" + logger.info(f"Stage started: {self.ingestion_stage}") self._timer.start() From fe43f076dadc754313864f7a9ca286223ebb0275 Mon Sep 17 00:00:00 2001 From: Chakru <161002324+chakru-r@users.noreply.github.com> Date: Thu, 26 Dec 2024 21:22:16 +0530 Subject: [PATCH 10/27] Parallelize smoke test (#12225) --- .github/workflows/docker-unified.yml | 50 +++++++++++++------ smoke-test/.gitignore | 4 +- smoke-test/build.gradle | 38 ++++++-------- smoke-test/conftest.py | 52 ++++++++++++++++++++ smoke-test/smoke.sh | 25 ++++++---- smoke-test/tests/cypress/integration_test.py | 49 ++++++++++++------ 6 files changed, 155 insertions(+), 63 deletions(-) diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 03a9b3afc3bc5..47c26068347c0 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -1011,18 +1011,39 @@ jobs: needs: setup outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} + cypress_batch_count: ${{ steps.set-batch-count.outputs.cypress_batch_count }} + python_batch_count: ${{ steps.set-batch-count.outputs.python_batch_count }} steps: + - id: set-batch-count + # Tests are split simply to ensure the configured number of batches for parallelization. This may need some + # increase as a new tests added increase the duration where an additional parallel batch helps. + # python_batch_count is used to split pytests in the smoke-test (batches of actual test functions) + # cypress_batch_count is used to split the collection of cypress test specs into batches. + run: | + echo "cypress_batch_count=11" >> "$GITHUB_OUTPUT" + echo "python_batch_count=5" >> "$GITHUB_OUTPUT" + - id: set-matrix + # For m batches for python and n batches for cypress, we need a test matrix of python x m + cypress x n. + # while the github action matrix generation can handle these two parts individually, there isnt a way to use the + # two generated matrices for the same job. So, produce that matrix with scripting and use the include directive + # to add it to the test matrix. run: | - if [ '${{ needs.setup.outputs.frontend_only }}' == 'true' ]; then - echo 'matrix=["cypress_suite1","cypress_rest"]' >> "$GITHUB_OUTPUT" - elif [ '${{ needs.setup.outputs.ingestion_only }}' == 'true' ]; then - echo 'matrix=["no_cypress_suite0","no_cypress_suite1"]' >> "$GITHUB_OUTPUT" - elif [[ '${{ needs.setup.outputs.backend_change }}' == 'true' || '${{ needs.setup.outputs.smoke_test_change }}' == 'true' ]]; then - echo 'matrix=["no_cypress_suite0","no_cypress_suite1","cypress_suite1","cypress_rest"]' >> "$GITHUB_OUTPUT" - else - echo 'matrix=[]' >> "$GITHUB_OUTPUT" + python_batch_count=${{ steps.set-batch-count.outputs.python_batch_count }} + python_matrix=$(printf "{\"test_strategy\":\"pytests\",\"batch\":\"0\",\"batch_count\":\"$python_batch_count\"}"; for ((i=1;i> "$GITHUB_OUTPUT" smoke_test: name: Run Smoke Tests @@ -1043,8 +1064,7 @@ jobs: ] strategy: fail-fast: false - matrix: - test_strategy: ${{ fromJson(needs.smoke_test_matrix.outputs.matrix) }} + matrix: ${{ fromJson(needs.smoke_test_matrix.outputs.matrix) }} if: ${{ always() && !failure() && !cancelled() && needs.smoke_test_matrix.outputs.matrix != '[]' }} steps: - name: Free up disk space @@ -1220,6 +1240,8 @@ jobs: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} CLEANUP_DATA: "false" TEST_STRATEGY: ${{ matrix.test_strategy }} + BATCH_COUNT: ${{ matrix.batch_count }} + BATCH_NUMBER: ${{ matrix.batch }} run: | echo "$DATAHUB_VERSION" ./gradlew --stop @@ -1230,25 +1252,25 @@ jobs: if: failure() run: | docker ps -a - TEST_STRATEGY="-${{ matrix.test_strategy }}" + TEST_STRATEGY="-${{ matrix.test_strategy }}-${{ matrix.batch }}" source .github/scripts/docker_logs.sh - name: Upload logs uses: actions/upload-artifact@v3 if: failure() with: - name: docker-logs-${{ matrix.test_strategy }} + name: docker-logs-${{ matrix.test_strategy }}-${{ matrix.batch }} path: "docker_logs/*.log" retention-days: 5 - name: Upload screenshots uses: actions/upload-artifact@v3 if: failure() with: - name: cypress-snapshots-${{ matrix.test_strategy }} + name: cypress-snapshots-${{ matrix.test_strategy }}-${{ matrix.batch }} path: smoke-test/tests/cypress/cypress/screenshots/ - uses: actions/upload-artifact@v3 if: always() with: - name: Test Results (smoke tests) ${{ matrix.test_strategy }} + name: Test Results (smoke tests) ${{ matrix.test_strategy }} ${{ matrix.batch }} path: | **/build/reports/tests/test/** **/build/test-results/test/** diff --git a/smoke-test/.gitignore b/smoke-test/.gitignore index b8af2eef535a0..d8cfd65ff81b9 100644 --- a/smoke-test/.gitignore +++ b/smoke-test/.gitignore @@ -29,6 +29,8 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +**/cypress/node_modules + # PyInstaller # Usually these files are written by a python script from a template @@ -132,4 +134,4 @@ dmypy.json # Pyre type checker .pyre/ junit* -tests/cypress/onboarding.json \ No newline at end of file +tests/cypress/onboarding.json diff --git a/smoke-test/build.gradle b/smoke-test/build.gradle index 73ecdcb08ea14..60d08e0206cda 100644 --- a/smoke-test/build.gradle +++ b/smoke-test/build.gradle @@ -91,39 +91,31 @@ task pythonLintFix(type: Exec, dependsOn: installDev) { * The following tasks assume an already running quickstart. * ./gradlew quickstart (or another variation `quickstartDebug`) */ -task noCypressSuite0(type: Exec, dependsOn: [installDev, ':metadata-ingestion:installDev']) { - environment 'RUN_QUICKSTART', 'false' - environment 'TEST_STRATEGY', 'no_cypress_suite0' - - workingDir = project.projectDir - commandLine 'bash', '-c', - "source ${venv_name}/bin/activate && set -x && " + - "./smoke.sh" -} +// ./gradlew :smoke-test:pytest -PbatchNumber=2 (default 0) +task pytest(type: Exec, dependsOn: [installDev, ':metadata-ingestion:installDev']) { + // Get BATCH_NUMBER from command line argument with default value of 0 + def batchNumber = project.hasProperty('batchNumber') ? project.property('batchNumber') : '0' -task noCypressSuite1(type: Exec, dependsOn: [installDev, ':metadata-ingestion:installDev']) { environment 'RUN_QUICKSTART', 'false' - environment 'TEST_STRATEGY', 'no_cypress_suite1' + environment 'TEST_STRATEGY', 'pytests' + environment 'BATCH_COUNT', 5 + environment 'BATCH_NUMBER', batchNumber workingDir = project.projectDir commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "./smoke.sh" + "./smoke.sh" } -task cypressSuite1(type: Exec, dependsOn: [installDev, ':metadata-ingestion:installDev']) { - environment 'RUN_QUICKSTART', 'false' - environment 'TEST_STRATEGY', 'cypress_suite1' - - workingDir = project.projectDir - commandLine 'bash', '-c', - "source ${venv_name}/bin/activate && set -x && " + - "./smoke.sh" -} +// ./gradlew :smoke-test:cypressTest -PbatchNumber=2 (default 0) +task cypressTest(type: Exec, dependsOn: [installDev, ':metadata-ingestion:installDev']) { + // Get BATCH_NUMBER from command line argument with default value of 0 + def batchNumber = project.hasProperty('batchNumber') ? project.property('batchNumber') : '0' -task cypressRest(type: Exec, dependsOn: [installDev, ':metadata-ingestion:installDev']) { environment 'RUN_QUICKSTART', 'false' - environment 'TEST_STRATEGY', 'cypress_rest' + environment 'TEST_STRATEGY', 'cypress' + environment 'BATCH_COUNT', 11 + environment 'BATCH_NUMBER', batchNumber workingDir = project.projectDir commandLine 'bash', '-c', diff --git a/smoke-test/conftest.py b/smoke-test/conftest.py index 6d148db9886a4..d48a92b22ab48 100644 --- a/smoke-test/conftest.py +++ b/smoke-test/conftest.py @@ -1,6 +1,8 @@ import os import pytest +from typing import List, Tuple +from _pytest.nodes import Item import requests from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph @@ -45,3 +47,53 @@ def graph_client(auth_session) -> DataHubGraph: def pytest_sessionfinish(session, exitstatus): """whole test run finishes.""" send_message(exitstatus) + + +def get_batch_start_end(num_tests: int) -> Tuple[int, int]: + batch_count_env = os.getenv("BATCH_COUNT", 1) + batch_count = int(batch_count_env) + + batch_number_env = os.getenv("BATCH_NUMBER", 0) + batch_number = int(batch_number_env) + + if batch_count == 0 or batch_count > num_tests: + raise ValueError( + f"Invalid batch count {batch_count}: must be >0 and <= {num_tests} (num_tests)" + ) + if batch_number >= batch_count: + raise ValueError( + f"Invalid batch number: {batch_number}, must be less than {batch_count} (zer0 based index)" + ) + + batch_size = round(num_tests / batch_count) + + batch_start = batch_size * batch_number + batch_end = batch_start + batch_size + # We must have exactly as many batches as specified by BATCH_COUNT. + if ( + num_tests - batch_end < batch_size + ): # We must have exactly as many batches as specified by BATCH_COUNT, put the remaining in the last batch. + batch_end = num_tests + + if batch_count > 0: + print(f"Running tests for batch {batch_number} of {batch_count}") + + return batch_start, batch_end + + +def pytest_collection_modifyitems( + session: pytest.Session, config: pytest.Config, items: List[Item] +) -> None: + if os.getenv("TEST_STRATEGY") == "cypress": + return # We launch cypress via pytests, but needs a different batching mechanism at cypress level. + + # If BATCH_COUNT and BATCH_ENV vars are set, splits the pytests to batches and runs filters only the BATCH_NUMBER + # batch for execution. Enables multiple parallel launches. Current implementation assumes all test are of equal + # weight for batching. TODO. A weighted batching method can help make batches more equal sized by cost. + # this effectively is a no-op if BATCH_COUNT=1 + start_index, end_index = get_batch_start_end(num_tests=len(items)) + + items.sort(key=lambda x: x.nodeid) # we want the order to be stable across batches + # replace items with the filtered list + print(f"Running tests for batch {start_index}-{end_index}") + items[:] = items[start_index:end_index] diff --git a/smoke-test/smoke.sh b/smoke-test/smoke.sh index 888a60f488e1f..ec8188ebf5f4d 100755 --- a/smoke-test/smoke.sh +++ b/smoke-test/smoke.sh @@ -34,15 +34,20 @@ source ./set-cypress-creds.sh # set environment variables for the test source ./set-test-env-vars.sh -# no_cypress_suite0, no_cypress_suite1, cypress_suite1, cypress_rest -if [[ -z "${TEST_STRATEGY}" ]]; then - pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml +# TEST_STRATEGY: +# if set to pytests, runs all pytests, skips cypress tests(though cypress test launch is via a pytest). +# if set tp cypress, runs all cypress tests +# if blank, runs all. +# When invoked via the github action, BATCH_COUNT and BATCH_NUM env vars are set to run a slice of those tests per +# worker for parallelism. docker-unified.yml generates a test matrix of pytests/cypress in batches. As number of tests +# increase, the batch_count config (in docker-unified.yml) may need adjustment. +if [[ "${TEST_STRATEGY}" == "pytests" ]]; then + #pytests only - github test matrix runs pytests in one of the runners when applicable. + pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke-pytests.xml -k 'not test_run_cypress' +elif [[ "${TEST_STRATEGY}" == "cypress" ]]; then + # run only cypress tests. The test inspects BATCH_COUNT and BATCH_NUMBER and runs only a subset of tests in that batch. + # github workflow test matrix will invoke this in multiple runners for each batch. + pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke-cypress${BATCH_NUMBER}.xml tests/cypress/integration_test.py else - if [ "$TEST_STRATEGY" == "no_cypress_suite0" ]; then - pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -k 'not test_run_cypress' -m 'not no_cypress_suite1' - elif [ "$TEST_STRATEGY" == "no_cypress_suite1" ]; then - pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -m 'no_cypress_suite1' - else - pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_cypress_${TEST_STRATEGY}.xml tests/cypress/integration_test.py - fi + pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke-all.xml fi diff --git a/smoke-test/tests/cypress/integration_test.py b/smoke-test/tests/cypress/integration_test.py index 0d824a96810d0..33c67a923c278 100644 --- a/smoke-test/tests/cypress/integration_test.py +++ b/smoke-test/tests/cypress/integration_test.py @@ -1,10 +1,11 @@ import datetime import os import subprocess -from typing import List, Set +from typing import List import pytest +from conftest import get_batch_start_end from tests.setup.lineage.ingest_time_lineage import ( get_time_lineage_urns, ingest_time_lineage, @@ -169,10 +170,29 @@ def ingest_cleanup_data(auth_session, graph_client): print("deleted onboarding data") -def _get_spec_map(items: Set[str]) -> str: - if len(items) == 0: - return "" - return ",".join([f"**/{item}/*.js" for item in items]) +def _get_js_files(base_path: str): + file_paths = [] + for root, dirs, files in os.walk(base_path): + for file in files: + if file.endswith(".js"): + file_paths.append(os.path.relpath(os.path.join(root, file), base_path)) + return sorted(file_paths) # sort to make the order stable across batch runs + + +def _get_cypress_tests_batch(): + """ + Batching is configured via env vars BATCH_COUNT and BATCH_NUMBER. All cypress tests are split into exactly + BATCH_COUNT batches. When BATCH_NUMBER env var is set (zero based index), that batch alone is run. + Github workflow via test_matrix, runs all batches in parallel to speed up the test elapsed time. + If either of these vars are not set, all tests are run sequentially. + :return: + """ + all_tests = _get_js_files("tests/cypress/cypress/e2e") + + batch_start, batch_end = get_batch_start_end(num_tests=len(all_tests)) + + return all_tests[batch_start:batch_end] + # return test_batches[int(batch_number)] #if BATCH_NUMBER was set, we this test just runs that one batch. def test_run_cypress(auth_session): @@ -182,24 +202,23 @@ def test_run_cypress(auth_session): test_strategy = os.getenv("TEST_STRATEGY", None) if record_key: record_arg = " --record " - tag_arg = f" --tag {test_strategy} " + batch_number = os.getenv("BATCH_NUMBER") + batch_count = os.getenv("BATCH_COUNT") + if batch_number and batch_count: + batch_suffix = f"-{batch_number}{batch_count}" + else: + batch_suffix = "" + tag_arg = f" --tag {test_strategy}{batch_suffix}" else: record_arg = " " rest_specs = set(os.listdir("tests/cypress/cypress/e2e")) cypress_suite1_specs = {"mutations", "search", "views"} rest_specs.difference_update(set(cypress_suite1_specs)) - strategy_spec_map = { - "cypress_suite1": cypress_suite1_specs, - "cypress_rest": rest_specs, - } print(f"test strategy is {test_strategy}") test_spec_arg = "" - if test_strategy is not None: - specs = strategy_spec_map.get(test_strategy) - assert specs is not None - specs_str = _get_spec_map(specs) - test_spec_arg = f" --spec '{specs_str}' " + specs_str = ",".join([f"**/{f}" for f in _get_cypress_tests_batch()]) + test_spec_arg = f" --spec '{specs_str}' " print("Running Cypress tests with command") command = f"NO_COLOR=1 npx cypress run {record_arg} {test_spec_arg} {tag_arg}" From 5708bd9beba6ea08d66a37506867380a45718df3 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 26 Dec 2024 11:14:15 -0600 Subject: [PATCH 11/27] chore(bump): spring minor version bump 6.1.14 (#12228) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a3d807a733349..8929b4e644972 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,7 @@ buildscript { ext.pegasusVersion = '29.57.0' ext.mavenVersion = '3.6.3' ext.versionGradle = '8.11.1' - ext.springVersion = '6.1.13' + ext.springVersion = '6.1.14' ext.springBootVersion = '3.2.9' ext.springKafkaVersion = '3.1.6' ext.openTelemetryVersion = '1.18.0' From a920e9bec8ed8f0aa6ecfd482c8659afa4f3e034 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 26 Dec 2024 15:33:39 -0500 Subject: [PATCH 12/27] fix(ingest/lookml): emit warnings for resolution failures (#12215) --- .../source/looker/looker_dataclasses.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py index 327c9ebf99bd2..d771821a14d88 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py @@ -186,16 +186,16 @@ def resolve_includes( f"traversal_path={traversal_path}, included_files = {included_files}, seen_so_far: {seen_so_far}" ) if "*" not in inc and not included_files: - reporter.report_failure( + reporter.warning( title="Error Resolving Include", - message=f"Cannot resolve include {inc}", - context=f"Path: {path}", + message="Cannot resolve included file", + context=f"Include: {inc}, path: {path}, traversal_path: {traversal_path}", ) elif not included_files: - reporter.report_failure( + reporter.warning( title="Error Resolving Include", - message=f"Did not resolve anything for wildcard include {inc}", - context=f"Path: {path}", + message="Did not find anything matching the wildcard include", + context=f"Include: {inc}, path: {path}, traversal_path: {traversal_path}", ) # only load files that we haven't seen so far included_files = [x for x in included_files if x not in seen_so_far] @@ -231,9 +231,7 @@ def resolve_includes( source_config, reporter, seen_so_far, - traversal_path=traversal_path - + "." - + pathlib.Path(included_file).stem, + traversal_path=f"{traversal_path} -> {pathlib.Path(included_file).stem}", ) ) except Exception as e: From e1998dd371b1002ae8893d7a63d84e5bace079c7 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 26 Dec 2024 15:34:00 -0500 Subject: [PATCH 13/27] chore(ingest): remove `enable_logging` helper (#12222) --- .../powerbi/rest_api_wrapper/data_resolver.py | 31 ++-- .../powerbi/test_admin_only_api.py | 12 -- .../tests/integration/powerbi/test_powerbi.py | 141 +++++++----------- .../integration/powerbi/test_profiling.py | 10 -- .../tableau/test_tableau_ingest.py | 29 ---- 5 files changed, 70 insertions(+), 153 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py index e1301edef10b8..161975fa635fd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py @@ -84,13 +84,14 @@ def __init__( tenant_id: str, metadata_api_timeout: int, ): - self.__access_token: Optional[str] = None - self.__access_token_expiry_time: Optional[datetime] = None - self.__tenant_id = tenant_id + self._access_token: Optional[str] = None + self._access_token_expiry_time: Optional[datetime] = None + + self._tenant_id = tenant_id # Test connection by generating access token logger.info(f"Trying to connect to {self._get_authority_url()}") # Power-Bi Auth (Service Principal Auth) - self.__msal_client = msal.ConfidentialClientApplication( + self._msal_client = msal.ConfidentialClientApplication( client_id, client_credential=client_secret, authority=DataResolverBase.AUTHORITY + tenant_id, @@ -168,18 +169,18 @@ def _get_app( pass def _get_authority_url(self): - return f"{DataResolverBase.AUTHORITY}{self.__tenant_id}" + return f"{DataResolverBase.AUTHORITY}{self._tenant_id}" def get_authorization_header(self): return {Constant.Authorization: self.get_access_token()} - def get_access_token(self): - if self.__access_token is not None and not self._is_access_token_expired(): - return self.__access_token + def get_access_token(self) -> str: + if self._access_token is not None and not self._is_access_token_expired(): + return self._access_token logger.info("Generating PowerBi access token") - auth_response = self.__msal_client.acquire_token_for_client( + auth_response = self._msal_client.acquire_token_for_client( scopes=[DataResolverBase.SCOPE] ) @@ -193,24 +194,24 @@ def get_access_token(self): logger.info("Generated PowerBi access token") - self.__access_token = "Bearer {}".format( + self._access_token = "Bearer {}".format( auth_response.get(Constant.ACCESS_TOKEN) ) safety_gap = 300 - self.__access_token_expiry_time = datetime.now() + timedelta( + self._access_token_expiry_time = datetime.now() + timedelta( seconds=( max(auth_response.get(Constant.ACCESS_TOKEN_EXPIRY, 0) - safety_gap, 0) ) ) - logger.debug(f"{Constant.PBIAccessToken}={self.__access_token}") + logger.debug(f"{Constant.PBIAccessToken}={self._access_token}") - return self.__access_token + return self._access_token def _is_access_token_expired(self) -> bool: - if not self.__access_token_expiry_time: + if not self._access_token_expiry_time: return True - return self.__access_token_expiry_time < datetime.now() + return self._access_token_expiry_time < datetime.now() def get_dashboards(self, workspace: Workspace) -> List[Dashboard]: """ diff --git a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py index b636c12cfda06..00dc79ed38cfb 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py +++ b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py @@ -1,5 +1,3 @@ -import logging -import sys from typing import Any, Dict from unittest import mock @@ -483,12 +481,6 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None ) -def enable_logging(): - # set logging to console - logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) - logging.getLogger().setLevel(logging.DEBUG) - - def mock_msal_cca(*args, **kwargs): class MsalClient: def acquire_token_for_client(self, *args, **kwargs): @@ -527,8 +519,6 @@ def default_source_config(): @freeze_time(FROZEN_TIME) @mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) def test_admin_only_apis(mock_msal, pytestconfig, tmp_path, mock_time, requests_mock): - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_admin_api(request_mock=requests_mock) @@ -567,8 +557,6 @@ def test_admin_only_apis(mock_msal, pytestconfig, tmp_path, mock_time, requests_ def test_most_config_and_modified_since( mock_msal, pytestconfig, tmp_path, mock_time, requests_mock ): - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_admin_api(request_mock=requests_mock) diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index edde11ff87d29..739be7cc8408d 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -1,8 +1,6 @@ import datetime import json -import logging import re -import sys from pathlib import Path from typing import Any, Dict, List, Optional, Union, cast from unittest import mock @@ -31,29 +29,21 @@ FROZEN_TIME = "2022-02-03 07:00:00" -def enable_logging(): - # set logging to console - logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) - logging.getLogger().setLevel(logging.DEBUG) - - -class MsalClient: - call_num = 0 - token: Dict[str, Any] = { - "access_token": "dummy", - } - - @staticmethod - def acquire_token_for_client(*args, **kwargs): - MsalClient.call_num += 1 - return MsalClient.token +def mock_msal_cca(*args, **kwargs): + class MsalClient: + def __init__(self): + self.call_num = 0 + self.token: Dict[str, Any] = { + "access_token": "dummy", + } - @staticmethod - def reset(): - MsalClient.call_num = 0 + def acquire_token_for_client(self, *args, **kwargs): + self.call_num += 1 + return self.token + def reset(self): + self.call_num = 0 -def mock_msal_cca(*args, **kwargs): return MsalClient() @@ -154,8 +144,6 @@ def test_powerbi_ingest( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) @@ -199,8 +187,6 @@ def test_powerbi_workspace_type_filter( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api( @@ -260,8 +246,6 @@ def test_powerbi_ingest_patch_disabled( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) @@ -327,8 +311,6 @@ def test_powerbi_platform_instance_ingest( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) @@ -515,8 +497,6 @@ def test_extract_reports( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) @@ -561,8 +541,6 @@ def test_extract_lineage( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) @@ -660,8 +638,6 @@ def test_admin_access_is_not_allowed( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api( @@ -723,8 +699,6 @@ def test_workspace_container( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) @@ -764,85 +738,84 @@ def test_workspace_container( ) -@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) def test_access_token_expiry_with_long_expiry( - mock_msal: MagicMock, pytestconfig: pytest.Config, tmp_path: str, mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) - pipeline = Pipeline.create( - { - "run_id": "powerbi-test", - "source": { - "type": "powerbi", - "config": { - **default_source_config(), + mock_msal = mock_msal_cca() + + with mock.patch("msal.ConfidentialClientApplication", return_value=mock_msal): + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **default_source_config(), + }, }, - }, - "sink": { - "type": "file", - "config": { - "filename": f"{tmp_path}/powerbi_access_token_mces.json", + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_access_token_mces.json", + }, }, - }, - } - ) + } + ) # for long expiry, the token should only be requested once. - MsalClient.token = { + mock_msal.token = { "access_token": "dummy2", "expires_in": 3600, } + mock_msal.reset() - MsalClient.reset() pipeline.run() # We expect the token to be requested twice (once for AdminApiResolver and one for RegularApiResolver) - assert MsalClient.call_num == 2 + assert mock_msal.call_num == 2 -@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) def test_access_token_expiry_with_short_expiry( - mock_msal: MagicMock, pytestconfig: pytest.Config, tmp_path: str, mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) - pipeline = Pipeline.create( - { - "run_id": "powerbi-test", - "source": { - "type": "powerbi", - "config": { - **default_source_config(), + mock_msal = mock_msal_cca() + with mock.patch("msal.ConfidentialClientApplication", return_value=mock_msal): + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **default_source_config(), + }, }, - }, - "sink": { - "type": "file", - "config": { - "filename": f"{tmp_path}/powerbi_access_token_mces.json", + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_access_token_mces.json", + }, }, - }, - } - ) + } + ) # for short expiry, the token should be requested when expires. - MsalClient.token = { + mock_msal.token = { "access_token": "dummy", "expires_in": 0, } + mock_msal.reset() + pipeline.run() - assert MsalClient.call_num > 2 + assert mock_msal.call_num > 2 def dataset_type_mapping_set_to_all_platform(pipeline: Pipeline) -> None: @@ -940,8 +913,6 @@ def test_dataset_type_mapping_error( def test_server_to_platform_map( mock_msal, pytestconfig, tmp_path, mock_time, requests_mock ): - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" new_config: dict = { **default_source_config(), @@ -1416,8 +1387,6 @@ def test_powerbi_cross_workspace_reference_info_message( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() - register_mock_api( pytestconfig=pytestconfig, request_mock=requests_mock, @@ -1495,8 +1464,6 @@ def common_app_ingest( output_mcp_path: str, override_config: dict = {}, ) -> Pipeline: - enable_logging() - register_mock_api( pytestconfig=pytestconfig, request_mock=requests_mock, diff --git a/metadata-ingestion/tests/integration/powerbi/test_profiling.py b/metadata-ingestion/tests/integration/powerbi/test_profiling.py index 4b48bed003b1e..78d35cf31a26d 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_profiling.py +++ b/metadata-ingestion/tests/integration/powerbi/test_profiling.py @@ -1,5 +1,3 @@ -import logging -import sys from typing import Any, Dict from unittest import mock @@ -271,12 +269,6 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None ) -def enable_logging(): - # set logging to console - logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) - logging.getLogger().setLevel(logging.DEBUG) - - def mock_msal_cca(*args, **kwargs): class MsalClient: def acquire_token_for_client(self, *args, **kwargs): @@ -311,8 +303,6 @@ def default_source_config(): @freeze_time(FROZEN_TIME) @mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) def test_profiling(mock_msal, pytestconfig, tmp_path, mock_time, requests_mock): - enable_logging() - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_admin_api(request_mock=requests_mock) diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index 902ff243c802a..71e5ad10c2fc5 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -1,7 +1,5 @@ import json -import logging import pathlib -import sys from typing import Any, Dict, List, cast from unittest import mock @@ -88,12 +86,6 @@ } -def enable_logging(): - # set logging to console - logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) - logging.getLogger().setLevel(logging.DEBUG) - - def read_response(file_name): response_json_path = f"{test_resources_dir}/setup/{file_name}" with open(response_json_path) as file: @@ -376,7 +368,6 @@ def tableau_ingest_common( @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_tableau_ingest(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_mces.json" golden_file_name: str = "tableau_mces_golden.json" tableau_ingest_common( @@ -454,7 +445,6 @@ def mock_data() -> List[dict]: @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_tableau_cll_ingest(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_mces_cll.json" golden_file_name: str = "tableau_cll_mces_golden.json" @@ -481,7 +471,6 @@ def test_tableau_cll_ingest(pytestconfig, tmp_path, mock_datahub_graph): @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_project_pattern(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_project_pattern_mces.json" golden_file_name: str = "tableau_mces_golden.json" @@ -505,7 +494,6 @@ def test_project_pattern(pytestconfig, tmp_path, mock_datahub_graph): @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_project_path_pattern(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_project_path_mces.json" golden_file_name: str = "tableau_project_path_mces_golden.json" @@ -529,8 +517,6 @@ def test_project_path_pattern(pytestconfig, tmp_path, mock_datahub_graph): @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_project_hierarchy(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() - output_file_name: str = "tableau_nested_project_mces.json" golden_file_name: str = "tableau_nested_project_mces_golden.json" @@ -554,7 +540,6 @@ def test_project_hierarchy(pytestconfig, tmp_path, mock_datahub_graph): @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_extract_all_project(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_extract_all_project_mces.json" golden_file_name: str = "tableau_extract_all_project_mces_golden.json" @@ -644,7 +629,6 @@ def test_project_path_pattern_deny(pytestconfig, tmp_path, mock_datahub_graph): def test_tableau_ingest_with_platform_instance( pytestconfig, tmp_path, mock_datahub_graph ): - enable_logging() output_file_name: str = "tableau_with_platform_instance_mces.json" golden_file_name: str = "tableau_with_platform_instance_mces_golden.json" @@ -691,7 +675,6 @@ def test_tableau_ingest_with_platform_instance( def test_lineage_overrides(): - enable_logging() # Simple - specify platform instance to presto table assert ( TableauUpstreamReference( @@ -745,7 +728,6 @@ def test_lineage_overrides(): def test_database_hostname_to_platform_instance_map(): - enable_logging() # Simple - snowflake table assert ( TableauUpstreamReference( @@ -916,7 +898,6 @@ def test_tableau_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph) def test_tableau_no_verify(): - enable_logging() # This test ensures that we can connect to a self-signed certificate # when ssl_verify is set to False. @@ -941,7 +922,6 @@ def test_tableau_no_verify(): @freeze_time(FROZEN_TIME) @pytest.mark.integration_batch_2 def test_tableau_signout_timeout(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_signout_timeout_mces.json" golden_file_name: str = "tableau_signout_timeout_mces_golden.json" tableau_ingest_common( @@ -1073,7 +1053,6 @@ def test_get_all_datasources_failure(pytestconfig, tmp_path, mock_datahub_graph) @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_tableau_ingest_multiple_sites(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_mces_multiple_sites.json" golden_file_name: str = "tableau_multiple_sites_mces_golden.json" @@ -1135,7 +1114,6 @@ def test_tableau_ingest_multiple_sites(pytestconfig, tmp_path, mock_datahub_grap @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_tableau_ingest_sites_as_container(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_mces_ingest_sites_as_container.json" golden_file_name: str = "tableau_sites_as_container_mces_golden.json" @@ -1159,7 +1137,6 @@ def test_tableau_ingest_sites_as_container(pytestconfig, tmp_path, mock_datahub_ @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_site_name_pattern(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_site_name_pattern_mces.json" golden_file_name: str = "tableau_site_name_pattern_mces_golden.json" @@ -1183,7 +1160,6 @@ def test_site_name_pattern(pytestconfig, tmp_path, mock_datahub_graph): @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_permission_ingestion(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_permission_ingestion_mces.json" golden_file_name: str = "tableau_permission_ingestion_mces_golden.json" @@ -1209,7 +1185,6 @@ def test_permission_ingestion(pytestconfig, tmp_path, mock_datahub_graph): @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_no_hidden_assets(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_no_hidden_assets_mces.json" golden_file_name: str = "tableau_no_hidden_assets_mces_golden.json" @@ -1232,7 +1207,6 @@ def test_no_hidden_assets(pytestconfig, tmp_path, mock_datahub_graph): @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_ingest_tags_disabled(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_ingest_tags_disabled_mces.json" golden_file_name: str = "tableau_ingest_tags_disabled_mces_golden.json" @@ -1254,7 +1228,6 @@ def test_ingest_tags_disabled(pytestconfig, tmp_path, mock_datahub_graph): @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_hidden_asset_tags(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() output_file_name: str = "tableau_hidden_asset_tags_mces.json" golden_file_name: str = "tableau_hidden_asset_tags_mces_golden.json" @@ -1277,8 +1250,6 @@ def test_hidden_asset_tags(pytestconfig, tmp_path, mock_datahub_graph): @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_hidden_assets_without_ingest_tags(pytestconfig, tmp_path, mock_datahub_graph): - enable_logging() - new_config = config_source_default.copy() new_config["tags_for_hidden_assets"] = ["hidden", "private"] new_config["ingest_tags"] = False From 172736a9b3d291d6cb8fcaf775114abdf8853e1f Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 26 Dec 2024 21:34:31 -0500 Subject: [PATCH 14/27] feat(ingest/dbt): support "Explore" page in dbt cloud (#12223) --- docs/how/updating-datahub.md | 1 + .../src/datahub/ingestion/source/dbt/dbt_cloud.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index a742ebe0cd896..d6620fde0bf79 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -42,6 +42,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #12077: `Kafka` source no longer ingests schemas from schema registry as separate entities by default, set `ingest_schemas_as_entities` to `true` to ingest them - OpenAPI Update: PIT Keep Alive parameter added to scroll. NOTE: This parameter requires the `pointInTimeCreationEnabled` feature flag to be enabled and the `elasticSearch.implementation` configuration to be `elasticsearch`. This feature is not supported for OpenSearch at this time and the parameter will not be respected without both of these set. - OpenAPI Update 2: Previously there was an incorrectly marked parameter named `sort` on the generic list entities endpoint for v3. This parameter is deprecated and only supports a single string value while the documentation indicates it supports a list of strings. This documentation error has been fixed and the correct field, `sortCriteria`, is now documented which supports a list of strings. +- #12223: For dbt Cloud ingestion, the "View in dbt" link will point at the "Explore" page in the dbt Cloud UI. You can revert to the old behavior of linking to the dbt Cloud IDE by setting `external_url_mode: ide". ### Breaking Changes diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py index 66c5ef7179af4..5042f6d69b261 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py @@ -1,7 +1,7 @@ import logging from datetime import datetime from json import JSONDecodeError -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Literal, Optional, Tuple from urllib.parse import urlparse import dateutil.parser @@ -62,6 +62,11 @@ class DBTCloudConfig(DBTCommonConfig): description="The ID of the run to ingest metadata from. If not specified, we'll default to the latest run.", ) + external_url_mode: Literal["explore", "ide"] = Field( + default="explore", + description='Where should the "View in dbt" link point to - either the "Explore" UI or the dbt Cloud IDE', + ) + @root_validator(pre=True) def set_metadata_endpoint(cls, values: dict) -> dict: if values.get("access_url") and not values.get("metadata_endpoint"): @@ -527,5 +532,7 @@ def _parse_into_dbt_column( ) def get_external_url(self, node: DBTNode) -> Optional[str]: - # TODO: Once dbt Cloud supports deep linking to specific files, we can use that. - return f"{self.config.access_url}/develop/{self.config.account_id}/projects/{self.config.project_id}" + if self.config.external_url_mode == "explore": + return f"{self.config.access_url}/explore/{self.config.account_id}/projects/{self.config.project_id}/environments/production/details/{node.dbt_name}" + else: + return f"{self.config.access_url}/develop/{self.config.account_id}/projects/{self.config.project_id}" From 3ca8d09100eb649a5f191a32f2af8d300424818b Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:40:00 +0530 Subject: [PATCH 15/27] feat(ingest/snowflake): support email_as_user_identifier for queries v2 (#12219) --- .../source/snowflake/snowflake_config.py | 19 ++++--- .../source/snowflake/snowflake_queries.py | 45 ++++++++++++--- .../source/snowflake/snowflake_query.py | 6 +- .../source/snowflake/snowflake_usage_v2.py | 7 +-- .../source/snowflake/snowflake_utils.py | 40 ++++++++------ .../snowflake/test_snowflake_queries.py | 55 +++++++++++++++++++ 6 files changed, 132 insertions(+), 40 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index 2b2dcf860cdb0..12e5fb72b00de 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -138,12 +138,20 @@ class SnowflakeIdentifierConfig( description="Whether to convert dataset urns to lowercase.", ) - -class SnowflakeUsageConfig(BaseUsageConfig): email_domain: Optional[str] = pydantic.Field( default=None, description="Email domain of your organization so users can be displayed on UI appropriately.", ) + + email_as_user_identifier: bool = Field( + default=True, + description="Format user urns as an email, if the snowflake user's email is set. If `email_domain` is " + "provided, generates email addresses for snowflake users with unset emails, based on their " + "username.", + ) + + +class SnowflakeUsageConfig(BaseUsageConfig): apply_view_usage_to_tables: bool = pydantic.Field( default=False, description="Whether to apply view's usage to its base tables. If set to True, usage is applied to base tables only.", @@ -267,13 +275,6 @@ class SnowflakeV2Config( " Map of share name -> details of share.", ) - email_as_user_identifier: bool = Field( - default=True, - description="Format user urns as an email, if the snowflake user's email is set. If `email_domain` is " - "provided, generates email addresses for snowflake users with unset emails, based on their " - "username.", - ) - include_assertion_results: bool = Field( default=False, description="Whether to ingest assertion run results for assertions created using Datahub" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py index 174aad0bddd4a..36825dc33fe7d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py @@ -66,6 +66,11 @@ logger = logging.getLogger(__name__) +# Define a type alias +UserName = str +UserEmail = str +UsersMapping = Dict[UserName, UserEmail] + class SnowflakeQueriesExtractorConfig(ConfigModel): # TODO: Support stateful ingestion for the time windows. @@ -114,11 +119,13 @@ class SnowflakeQueriesSourceConfig( class SnowflakeQueriesExtractorReport(Report): copy_history_fetch_timer: PerfTimer = dataclasses.field(default_factory=PerfTimer) query_log_fetch_timer: PerfTimer = dataclasses.field(default_factory=PerfTimer) + users_fetch_timer: PerfTimer = dataclasses.field(default_factory=PerfTimer) audit_log_load_timer: PerfTimer = dataclasses.field(default_factory=PerfTimer) sql_aggregator: Optional[SqlAggregatorReport] = None num_ddl_queries_dropped: int = 0 + num_users: int = 0 @dataclass @@ -225,6 +232,9 @@ def is_allowed_table(self, name: str) -> bool: def get_workunits_internal( self, ) -> Iterable[MetadataWorkUnit]: + with self.report.users_fetch_timer: + users = self.fetch_users() + # TODO: Add some logic to check if the cached audit log is stale or not. audit_log_file = self.local_temp_path / "audit_log.sqlite" use_cached_audit_log = audit_log_file.exists() @@ -248,7 +258,7 @@ def get_workunits_internal( queries.append(entry) with self.report.query_log_fetch_timer: - for entry in self.fetch_query_log(): + for entry in self.fetch_query_log(users): queries.append(entry) with self.report.audit_log_load_timer: @@ -263,6 +273,25 @@ def get_workunits_internal( shared_connection.close() audit_log_file.unlink(missing_ok=True) + def fetch_users(self) -> UsersMapping: + users: UsersMapping = dict() + with self.structured_reporter.report_exc("Error fetching users from Snowflake"): + logger.info("Fetching users from Snowflake") + query = SnowflakeQuery.get_all_users() + resp = self.connection.query(query) + + for row in resp: + try: + users[row["NAME"]] = row["EMAIL"] + self.report.num_users += 1 + except Exception as e: + self.structured_reporter.warning( + "Error parsing user row", + context=f"{row}", + exc=e, + ) + return users + def fetch_copy_history(self) -> Iterable[KnownLineageMapping]: # Derived from _populate_external_lineage_from_copy_history. @@ -298,7 +327,7 @@ def fetch_copy_history(self) -> Iterable[KnownLineageMapping]: yield result def fetch_query_log( - self, + self, users: UsersMapping ) -> Iterable[Union[PreparsedQuery, TableRename, TableSwap]]: query_log_query = _build_enriched_query_log_query( start_time=self.config.window.start_time, @@ -319,7 +348,7 @@ def fetch_query_log( assert isinstance(row, dict) try: - entry = self._parse_audit_log_row(row) + entry = self._parse_audit_log_row(row, users) except Exception as e: self.structured_reporter.warning( "Error parsing query log row", @@ -331,7 +360,7 @@ def fetch_query_log( yield entry def _parse_audit_log_row( - self, row: Dict[str, Any] + self, row: Dict[str, Any], users: UsersMapping ) -> Optional[Union[TableRename, TableSwap, PreparsedQuery]]: json_fields = { "DIRECT_OBJECTS_ACCESSED", @@ -430,9 +459,11 @@ def _parse_audit_log_row( ) ) - # TODO: Fetch email addresses from Snowflake to map user -> email - # TODO: Support email_domain fallback for generating user urns. - user = CorpUserUrn(self.identifiers.snowflake_identifier(res["user_name"])) + user = CorpUserUrn( + self.identifiers.get_user_identifier( + res["user_name"], users.get(res["user_name"]) + ) + ) timestamp: datetime = res["query_start_time"] timestamp = timestamp.astimezone(timezone.utc) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index a94b39476b2c2..40bcfb514efd2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -947,4 +947,8 @@ def dmf_assertion_results(start_time_millis: int, end_time_millis: int) -> str: AND METRIC_NAME ilike '{pattern}' escape '{escape_pattern}' ORDER BY MEASUREMENT_TIME ASC; -""" + """ + + @staticmethod + def get_all_users() -> str: + return """SELECT name as "NAME", email as "EMAIL" FROM SNOWFLAKE.ACCOUNT_USAGE.USERS""" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py index aff15386c5083..4bdf559f293b5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py @@ -342,10 +342,9 @@ def _map_user_counts( filtered_user_counts.append( DatasetUserUsageCounts( user=make_user_urn( - self.get_user_identifier( + self.identifiers.get_user_identifier( user_count["user_name"], user_email, - self.config.email_as_user_identifier, ) ), count=user_count["total"], @@ -453,9 +452,7 @@ def _get_operation_aspect_work_unit( reported_time: int = int(time.time() * 1000) last_updated_timestamp: int = int(start_time.timestamp() * 1000) user_urn = make_user_urn( - self.get_user_identifier( - user_name, user_email, self.config.email_as_user_identifier - ) + self.identifiers.get_user_identifier(user_name, user_email) ) # NOTE: In earlier `snowflake-usage` connector this was base_objects_accessed, which is incorrect diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py index 8e0c97aa135e8..885bee1ccdb90 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py @@ -300,6 +300,28 @@ def get_quoted_identifier_for_schema(db_name, schema_name): def get_quoted_identifier_for_table(db_name, schema_name, table_name): return f'"{db_name}"."{schema_name}"."{table_name}"' + # Note - decide how to construct user urns. + # Historically urns were created using part before @ from user's email. + # Users without email were skipped from both user entries as well as aggregates. + # However email is not mandatory field in snowflake user, user_name is always present. + def get_user_identifier( + self, + user_name: str, + user_email: Optional[str], + ) -> str: + if user_email: + return self.snowflake_identifier( + user_email + if self.identifier_config.email_as_user_identifier is True + else user_email.split("@")[0] + ) + return self.snowflake_identifier( + f"{user_name}@{self.identifier_config.email_domain}" + if self.identifier_config.email_as_user_identifier is True + and self.identifier_config.email_domain is not None + else user_name + ) + class SnowflakeCommonMixin(SnowflakeStructuredReportMixin): platform = "snowflake" @@ -315,24 +337,6 @@ def structured_reporter(self) -> SourceReport: def identifiers(self) -> SnowflakeIdentifierBuilder: return SnowflakeIdentifierBuilder(self.config, self.report) - # Note - decide how to construct user urns. - # Historically urns were created using part before @ from user's email. - # Users without email were skipped from both user entries as well as aggregates. - # However email is not mandatory field in snowflake user, user_name is always present. - def get_user_identifier( - self, - user_name: str, - user_email: Optional[str], - email_as_user_identifier: bool, - ) -> str: - if user_email: - return self.identifiers.snowflake_identifier( - user_email - if email_as_user_identifier is True - else user_email.split("@")[0] - ) - return self.identifiers.snowflake_identifier(user_name) - # TODO: Revisit this after stateful ingestion can commit checkpoint # for failures that do not affect the checkpoint # TODO: Add additional parameters to match the signature of the .warning and .failure methods diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py index 82f5691bcee3d..ae0f23d93215d 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_queries.py @@ -22,3 +22,58 @@ def test_source_close_cleans_tmp(snowflake_connect, tmp_path): # This closes QueriesExtractor which in turn closes SqlParsingAggregator source.close() assert len(os.listdir(tmp_path)) == 0 + + +@patch("snowflake.connector.connect") +def test_user_identifiers_email_as_identifier(snowflake_connect, tmp_path): + source = SnowflakeQueriesSource.create( + { + "connection": { + "account_id": "ABC12345.ap-south-1.aws", + "username": "TST_USR", + "password": "TST_PWD", + }, + "email_as_user_identifier": True, + "email_domain": "example.com", + }, + PipelineContext("run-id"), + ) + assert ( + source.identifiers.get_user_identifier("username", "username@example.com") + == "username@example.com" + ) + assert ( + source.identifiers.get_user_identifier("username", None) + == "username@example.com" + ) + + # We'd do best effort to use email as identifier, but would keep username as is, + # if email can't be formed. + source.identifiers.identifier_config.email_domain = None + + assert ( + source.identifiers.get_user_identifier("username", "username@example.com") + == "username@example.com" + ) + + assert source.identifiers.get_user_identifier("username", None) == "username" + + +@patch("snowflake.connector.connect") +def test_user_identifiers_username_as_identifier(snowflake_connect, tmp_path): + source = SnowflakeQueriesSource.create( + { + "connection": { + "account_id": "ABC12345.ap-south-1.aws", + "username": "TST_USR", + "password": "TST_PWD", + }, + "email_as_user_identifier": False, + }, + PipelineContext("run-id"), + ) + assert ( + source.identifiers.get_user_identifier("username", "username@example.com") + == "username" + ) + assert source.identifiers.get_user_identifier("username", None) == "username" From 29e4528ae5117ccdb6f0685b8571c2afdcc19f57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Fri, 27 Dec 2024 11:12:40 +0100 Subject: [PATCH 16/27] fix(tableau): retry if 502 error code (#12233) --- .../datahub/ingestion/source/tableau/tableau.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 508500ffe489b..df59cae3fad23 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -186,6 +186,15 @@ except ImportError: REAUTHENTICATE_ERRORS = (NonXMLResponseError,) +RETRIABLE_ERROR_CODES = [ + 408, # Request Timeout + 429, # Too Many Requests + 500, # Internal Server Error + 502, # Bad Gateway + 503, # Service Unavailable + 504, # Gateway Timeout +] + logger: logging.Logger = logging.getLogger(__name__) # Replace / with | @@ -287,7 +296,7 @@ def make_tableau_client(self, site: str) -> Server: max_retries=Retry( total=self.max_retries, backoff_factor=1, - status_forcelist=[429, 500, 502, 503, 504], + status_forcelist=RETRIABLE_ERROR_CODES, ) ) server._session.mount("http://", adapter) @@ -1212,9 +1221,11 @@ def get_connection_object_page( except InternalServerError as ise: # In some cases Tableau Server returns 504 error, which is a timeout error, so it worths to retry. - if ise.code == 504: + # Extended with other retryable errors. + if ise.code in RETRIABLE_ERROR_CODES: if retries_remaining <= 0: raise ise + logger.info(f"Retrying query due to error {ise.code}") return self.get_connection_object_page( query=query, connection_type=connection_type, From d7de7eb2a65385b0a4458f9c26cc8b1a42158cc1 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Fri, 27 Dec 2024 17:51:43 +0530 Subject: [PATCH 17/27] ci: remove qodana (#12227) --- .github/workflows/qodana-scan.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .github/workflows/qodana-scan.yml diff --git a/.github/workflows/qodana-scan.yml b/.github/workflows/qodana-scan.yml deleted file mode 100644 index 750cf24ad38e5..0000000000000 --- a/.github/workflows/qodana-scan.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Qodana -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - qodana: - runs-on: ubuntu-latest - steps: - - uses: acryldata/sane-checkout-action@v3 - - name: "Qodana Scan" - uses: JetBrains/qodana-action@v2022.3.4 - - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: ${{ runner.temp }}/qodana/results/qodana.sarif.json - cache-default-branch-only: true From ac8e539457ef984cb61329a449585fa86fc5d3c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Fri, 27 Dec 2024 16:14:32 +0100 Subject: [PATCH 18/27] chore(tableau): adjust visibility of info message (#12235) --- .../src/datahub/ingestion/source/tableau/tableau.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index df59cae3fad23..d47e10c9eb5c6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -920,10 +920,7 @@ def dataset_browse_prefix(self) -> str: return f"/{self.config.env.lower()}{self.no_env_browse_prefix}" def _re_authenticate(self) -> None: - self.report.info( - message="Re-authenticating to Tableau", - context=f"site='{self.site_content_url}'", - ) + logger.info(f"Re-authenticating to Tableau site '{self.site_content_url}'") # Sign-in again may not be enough because Tableau sometimes caches invalid sessions # so we need to recreate the Tableau Server object self.server = self.config.make_tableau_client(self.site_content_url) From ed8639e401d30b842fac66b52636f5c1ab0c71b7 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 27 Dec 2024 13:46:49 -0500 Subject: [PATCH 19/27] chore(python): test with python 3.11 (#11280) Co-authored-by: Tamas Nemeth Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> --- .github/workflows/dagster-plugin.yml | 6 +++--- .github/workflows/metadata-ingestion.yml | 4 ++-- .github/workflows/prefect-plugin.yml | 4 ++-- metadata-ingestion-modules/airflow-plugin/setup.py | 4 ---- metadata-ingestion-modules/dagster-plugin/README.md | 3 +-- metadata-ingestion-modules/dagster-plugin/setup.py | 3 --- metadata-ingestion-modules/gx-plugin/README.md | 3 +-- metadata-ingestion-modules/gx-plugin/setup.py | 3 --- metadata-ingestion-modules/prefect-plugin/README.md | 2 +- metadata-ingestion-modules/prefect-plugin/setup.py | 6 +----- metadata-ingestion/setup.py | 10 ++++------ .../src/datahub/ingestion/source/s3/source.py | 2 +- .../tests/integration/feast/test_feast_repository.py | 8 ++++++++ 13 files changed, 24 insertions(+), 34 deletions(-) diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml index d8a9cd7bfd6a3..ae9a0b1605cdf 100644 --- a/.github/workflows/dagster-plugin.yml +++ b/.github/workflows/dagster-plugin.yml @@ -30,11 +30,11 @@ jobs: DATAHUB_TELEMETRY_ENABLED: false strategy: matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.11"] include: - python-version: "3.9" extraPythonRequirement: "dagster>=1.3.3" - - python-version: "3.10" + - python-version: "3.11" extraPythonRequirement: "dagster>=1.3.3" fail-fast: false steps: @@ -57,7 +57,7 @@ jobs: if: always() run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze - uses: actions/upload-artifact@v4 - if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }} + if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }} with: name: Test Results (dagster Plugin ${{ matrix.python-version}}) path: | diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index ad00c6d1551d1..106cba1473982 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -33,7 +33,7 @@ jobs: # DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }} strategy: matrix: - python-version: ["3.8", "3.10"] + python-version: ["3.8", "3.11"] command: [ "testQuick", @@ -43,7 +43,7 @@ jobs: ] include: - python-version: "3.8" - - python-version: "3.10" + - python-version: "3.11" fail-fast: false steps: - name: Free up disk space diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml index e4a70426f3a61..d77142a1f00de 100644 --- a/.github/workflows/prefect-plugin.yml +++ b/.github/workflows/prefect-plugin.yml @@ -30,7 +30,7 @@ jobs: DATAHUB_TELEMETRY_ENABLED: false strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] fail-fast: false steps: - name: Set up JDK 17 @@ -52,7 +52,7 @@ jobs: if: always() run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze - uses: actions/upload-artifact@v4 - if: ${{ always() && matrix.python-version == '3.10'}} + if: ${{ always() && matrix.python-version == '3.11'}} with: name: Test Results (Prefect Plugin ${{ matrix.python-version}}) path: | diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py index 3209233184d55..2693aab0700da 100644 --- a/metadata-ingestion-modules/airflow-plugin/setup.py +++ b/metadata-ingestion-modules/airflow-plugin/setup.py @@ -148,10 +148,6 @@ def get_long_description(): "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", diff --git a/metadata-ingestion-modules/dagster-plugin/README.md b/metadata-ingestion-modules/dagster-plugin/README.md index 8e1460957ed9f..5113fc37dcc22 100644 --- a/metadata-ingestion-modules/dagster-plugin/README.md +++ b/metadata-ingestion-modules/dagster-plugin/README.md @@ -1,4 +1,3 @@ # Datahub Dagster Plugin -See the DataHub Dagster docs for details. - +See the [DataHub Dagster docs](https://datahubproject.io/docs/lineage/dagster/) for details. diff --git a/metadata-ingestion-modules/dagster-plugin/setup.py b/metadata-ingestion-modules/dagster-plugin/setup.py index 0e0685cb378c1..22c15497bd807 100644 --- a/metadata-ingestion-modules/dagster-plugin/setup.py +++ b/metadata-ingestion-modules/dagster-plugin/setup.py @@ -107,9 +107,6 @@ def get_long_description(): "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", diff --git a/metadata-ingestion-modules/gx-plugin/README.md b/metadata-ingestion-modules/gx-plugin/README.md index 1ffd87a955432..9d50235a093d6 100644 --- a/metadata-ingestion-modules/gx-plugin/README.md +++ b/metadata-ingestion-modules/gx-plugin/README.md @@ -1,4 +1,3 @@ # Datahub GX Plugin -See the DataHub GX docs for details. - +See the [DataHub GX docs](https://datahubproject.io/docs/metadata-ingestion/integration_docs/great-expectations) for details. diff --git a/metadata-ingestion-modules/gx-plugin/setup.py b/metadata-ingestion-modules/gx-plugin/setup.py index 73d5d1a9a02f1..40afc81a98f9c 100644 --- a/metadata-ingestion-modules/gx-plugin/setup.py +++ b/metadata-ingestion-modules/gx-plugin/setup.py @@ -118,9 +118,6 @@ def get_long_description(): "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", diff --git a/metadata-ingestion-modules/prefect-plugin/README.md b/metadata-ingestion-modules/prefect-plugin/README.md index 0896942e78ef6..f21e00b494513 100644 --- a/metadata-ingestion-modules/prefect-plugin/README.md +++ b/metadata-ingestion-modules/prefect-plugin/README.md @@ -28,7 +28,7 @@ The `prefect-datahub` collection allows you to easily integrate DataHub's metada ## Prerequisites -- Python 3.7+ +- Python 3.8+ - Prefect 2.0.0+ and < 3.0.0+ - A running instance of DataHub diff --git a/metadata-ingestion-modules/prefect-plugin/setup.py b/metadata-ingestion-modules/prefect-plugin/setup.py index 7e56fe8b6ad11..70b0e95819564 100644 --- a/metadata-ingestion-modules/prefect-plugin/setup.py +++ b/metadata-ingestion-modules/prefect-plugin/setup.py @@ -103,10 +103,6 @@ def get_long_description(): "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", @@ -120,7 +116,7 @@ def get_long_description(): ], # Package info. zip_safe=False, - python_requires=">=3.7", + python_requires=">=3.8", package_dir={"": "src"}, packages=setuptools.find_namespace_packages(where="./src"), entry_points=entry_points, diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index c6994dd6d5aa6..986dc189cb29b 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -298,8 +298,8 @@ } data_lake_profiling = { - "pydeequ~=1.1.0", - "pyspark~=3.3.0", + "pydeequ>=1.1.0", + "pyspark~=3.5.0", } delta_lake = { @@ -318,7 +318,7 @@ # 0.1.11 appears to have authentication issues with azure databricks # 0.22.0 has support for `include_browse` in metadata list apis "databricks-sdk>=0.30.0", - "pyspark~=3.3.0", + "pyspark~=3.5.0", "requests", # Version 2.4.0 includes sqlalchemy dialect, 2.8.0 includes some bug fixes # Version 3.0.0 required SQLAlchemy > 2.0.21 @@ -874,9 +874,6 @@ "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", @@ -917,6 +914,7 @@ "sync-file-emitter", "sql-parser", "iceberg", + "feast", } else set() ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py index 3ddf47b70cdf8..ceac9e96d1ddd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py @@ -225,7 +225,7 @@ def __init__(self, config: DataLakeSourceConfig, ctx: PipelineContext): self.init_spark() def init_spark(self): - os.environ.setdefault("SPARK_VERSION", "3.3") + os.environ.setdefault("SPARK_VERSION", "3.5") spark_version = os.environ["SPARK_VERSION"] # Importing here to avoid Deequ dependency for non profiling use cases diff --git a/metadata-ingestion/tests/integration/feast/test_feast_repository.py b/metadata-ingestion/tests/integration/feast/test_feast_repository.py index 7f04337145dc3..80d7c6311a958 100644 --- a/metadata-ingestion/tests/integration/feast/test_feast_repository.py +++ b/metadata-ingestion/tests/integration/feast/test_feast_repository.py @@ -1,3 +1,6 @@ +import sys + +import pytest from freezegun import freeze_time from datahub.ingestion.run.pipeline import Pipeline @@ -6,6 +9,11 @@ FROZEN_TIME = "2020-04-14 07:00:00" +# The test is skipped for python 3.11 due to conflicting dependencies in installDev +# setup that requires pydantic < 2 for majority plugins. Note that the test works with +# python 3.11 if run with standalone virtual env setup with feast plugin alone using +# `pip install acryl-datahub[feast]` since it allows pydantic > 2 +@pytest.mark.skipif(sys.version_info > (3, 11), reason="Skipped on Python 3.11+") @freeze_time(FROZEN_TIME) def test_feast_repository_ingest(pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/feast" From d0423547ba559c6059ffc35f9ed153036bf0e45d Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 27 Dec 2024 13:50:28 -0500 Subject: [PATCH 20/27] feat(ingest): add parse_ts_millis helper (#12231) --- .../assertion_circuit_breaker.py | 9 ++--- .../src/datahub/emitter/mce_builder.py | 18 +++++++++- .../src/datahub/emitter/mcp_builder.py | 9 ++--- .../src/datahub/emitter/mcp_patch_builder.py | 4 +-- .../src/datahub/emitter/rest_emitter.py | 4 +-- .../datahub/ingestion/api/source_helpers.py | 8 ++--- .../source/bigquery_v2/bigquery_schema.py | 25 +++---------- .../source/datahub/datahub_kafka_reader.py | 3 +- .../source/sql/sql_generic_profiler.py | 11 +++--- .../ingestion/source/state/checkpoint.py | 3 +- .../datahub/ingestion/source/unity/proxy.py | 35 +++++-------------- .../src/datahub/utilities/time.py | 11 ++++-- .../dbt_enabled_with_schemas_mces_golden.json | 10 +++--- .../dbt_test_column_meta_mapping_golden.json | 10 +++--- ...test_prefer_sql_parser_lineage_golden.json | 34 +++++++++--------- ...bt_test_test_model_performance_golden.json | 34 +++++++++--------- ...th_complex_owner_patterns_mces_golden.json | 10 +++--- ...th_data_platform_instance_mces_golden.json | 10 +++--- ...h_non_incremental_lineage_mces_golden.json | 10 +++--- ..._target_platform_instance_mces_golden.json | 10 +++--- .../tests/unit/sdk/test_mce_builder.py | 17 +++++++++ .../tests/unit/serde/test_codegen.py | 6 ++-- smoke-test/smoke.sh | 2 ++ 23 files changed, 145 insertions(+), 148 deletions(-) diff --git a/metadata-ingestion/src/datahub/api/circuit_breaker/assertion_circuit_breaker.py b/metadata-ingestion/src/datahub/api/circuit_breaker/assertion_circuit_breaker.py index 9d2a65663ba37..283cdaa833333 100644 --- a/metadata-ingestion/src/datahub/api/circuit_breaker/assertion_circuit_breaker.py +++ b/metadata-ingestion/src/datahub/api/circuit_breaker/assertion_circuit_breaker.py @@ -1,6 +1,6 @@ import logging from dataclasses import dataclass -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional from pydantic import Field @@ -10,6 +10,7 @@ CircuitBreakerConfig, ) from datahub.api.graphql import Assertion, Operation +from datahub.emitter.mce_builder import parse_ts_millis logger: logging.Logger = logging.getLogger(__name__) @@ -49,7 +50,7 @@ def get_last_updated(self, urn: str) -> Optional[datetime]: if not operations: return None else: - return datetime.fromtimestamp(operations[0]["lastUpdatedTimestamp"] / 1000) + return parse_ts_millis(operations[0]["lastUpdatedTimestamp"]) def _check_if_assertion_failed( self, assertions: List[Dict[str, Any]], last_updated: Optional[datetime] = None @@ -93,7 +94,7 @@ class AssertionResult: logger.info(f"Found successful assertion: {assertion_urn}") result = False if last_updated is not None: - last_run = datetime.fromtimestamp(last_assertion.time / 1000) + last_run = parse_ts_millis(last_assertion.time) if last_updated > last_run: logger.error( f"Missing assertion run for {assertion_urn}. The dataset was updated on {last_updated} but the last assertion run was at {last_run}" @@ -117,7 +118,7 @@ def is_circuit_breaker_active(self, urn: str) -> bool: ) if not last_updated: - last_updated = datetime.now() - self.config.time_delta + last_updated = datetime.now(tz=timezone.utc) - self.config.time_delta logger.info( f"Dataset {urn} doesn't have last updated or check_last_assertion_time is false, using calculated min assertion date {last_updated}" ) diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py index 69946c575908b..110624aa61cb8 100644 --- a/metadata-ingestion/src/datahub/emitter/mce_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py @@ -6,7 +6,7 @@ import os import re import time -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from typing import ( TYPE_CHECKING, @@ -103,6 +103,22 @@ def make_ts_millis(ts: Optional[datetime]) -> Optional[int]: return int(ts.timestamp() * 1000) +@overload +def parse_ts_millis(ts: float) -> datetime: + ... + + +@overload +def parse_ts_millis(ts: None) -> None: + ... + + +def parse_ts_millis(ts: Optional[float]) -> Optional[datetime]: + if ts is None: + return None + return datetime.fromtimestamp(ts / 1000, tz=timezone.utc) + + def make_data_platform_urn(platform: str) -> str: if platform.startswith("urn:li:dataPlatform:"): return platform diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py index 293157f8a1ed0..c8eb62a2e1de2 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py @@ -4,8 +4,8 @@ from pydantic.main import BaseModel from datahub.cli.env_utils import get_boolean_env_variable -from datahub.emitter.enum_helpers import get_enum_options from datahub.emitter.mce_builder import ( + ALL_ENV_TYPES, Aspect, datahub_guid, make_container_urn, @@ -25,7 +25,6 @@ ContainerClass, DomainsClass, EmbedClass, - FabricTypeClass, GlobalTagsClass, MetadataChangeEventClass, OwnerClass, @@ -206,11 +205,7 @@ def gen_containers( # Extra validation on the env field. # In certain cases (mainly for backwards compatibility), the env field will actually # have a platform instance name. - env = ( - container_key.env - if container_key.env in get_enum_options(FabricTypeClass) - else None - ) + env = container_key.env if container_key.env in ALL_ENV_TYPES else None container_urn = container_key.as_urn() diff --git a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py index 779b42e1e1ee9..1ed8ce1d5a615 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py @@ -2,7 +2,7 @@ import time from collections import defaultdict from dataclasses import dataclass -from typing import Any, Dict, Iterable, List, Optional, Sequence, Union +from typing import Any, Dict, List, Optional, Sequence, Union from datahub.emitter.aspect import JSON_PATCH_CONTENT_TYPE from datahub.emitter.serialization_helper import pre_json_transform @@ -75,7 +75,7 @@ def _add_patch( # TODO: Validate that aspectName is a valid aspect for this entityType self.patches[aspect_name].append(_Patch(op, path, value)) - def build(self) -> Iterable[MetadataChangeProposalClass]: + def build(self) -> List[MetadataChangeProposalClass]: return [ MetadataChangeProposalClass( entityUrn=self.urn, diff --git a/metadata-ingestion/src/datahub/emitter/rest_emitter.py b/metadata-ingestion/src/datahub/emitter/rest_emitter.py index 675717b5ec482..04242c8bf45d2 100644 --- a/metadata-ingestion/src/datahub/emitter/rest_emitter.py +++ b/metadata-ingestion/src/datahub/emitter/rest_emitter.py @@ -3,7 +3,7 @@ import logging import os from json.decoder import JSONDecodeError -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Union import requests from deprecated import deprecated @@ -288,7 +288,7 @@ def emit_mcp( def emit_mcps( self, - mcps: List[Union[MetadataChangeProposal, MetadataChangeProposalWrapper]], + mcps: Sequence[Union[MetadataChangeProposal, MetadataChangeProposalWrapper]], async_flag: Optional[bool] = None, ) -> int: logger.debug("Attempting to emit batch mcps") diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py index 7791ea2797be3..f3e5b1db6a1c8 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py @@ -1,5 +1,4 @@ import logging -from datetime import datetime, timezone from typing import ( TYPE_CHECKING, Dict, @@ -14,7 +13,7 @@ ) from datahub.configuration.time_window_config import BaseTimeWindowConfig -from datahub.emitter.mce_builder import make_dataplatform_instance_urn +from datahub.emitter.mce_builder import make_dataplatform_instance_urn, parse_ts_millis from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import entity_supports_aspect from datahub.ingestion.api.workunit import MetadataWorkUnit @@ -479,10 +478,7 @@ def auto_empty_dataset_usage_statistics( if invalid_timestamps: logger.warning( f"Usage statistics with unexpected timestamps, bucket_duration={config.bucket_duration}:\n" - ", ".join( - str(datetime.fromtimestamp(ts / 1000, tz=timezone.utc)) - for ts in invalid_timestamps - ) + ", ".join(str(parse_ts_millis(ts)) for ts in invalid_timestamps) ) for bucket in bucket_timestamps: diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py index 3ce34be8dc89d..cbe1f6eb97824 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py @@ -1,7 +1,7 @@ import logging from collections import defaultdict from dataclasses import dataclass, field -from datetime import datetime, timezone +from datetime import datetime from functools import lru_cache from typing import Any, Dict, FrozenSet, Iterable, Iterator, List, Optional @@ -15,6 +15,7 @@ TimePartitioningType, ) +from datahub.emitter.mce_builder import parse_ts_millis from datahub.ingestion.api.source import SourceReport from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier from datahub.ingestion.source.bigquery_v2.bigquery_helper import parse_labels @@ -393,13 +394,7 @@ def _make_bigquery_table( name=table.table_name, created=table.created, table_type=table.table_type, - last_altered=( - datetime.fromtimestamp( - table.get("last_altered") / 1000, tz=timezone.utc - ) - if table.get("last_altered") is not None - else None - ), + last_altered=parse_ts_millis(table.get("last_altered")), size_in_bytes=table.get("bytes"), rows_count=table.get("row_count"), comment=table.comment, @@ -460,11 +455,7 @@ def _make_bigquery_view(view: bigquery.Row) -> BigqueryView: return BigqueryView( name=view.table_name, created=view.created, - last_altered=( - datetime.fromtimestamp(view.get("last_altered") / 1000, tz=timezone.utc) - if view.get("last_altered") is not None - else None - ), + last_altered=(parse_ts_millis(view.get("last_altered"))), comment=view.comment, view_definition=view.view_definition, materialized=view.table_type == BigqueryTableType.MATERIALIZED_VIEW, @@ -705,13 +696,7 @@ def _make_bigquery_table_snapshot(snapshot: bigquery.Row) -> BigqueryTableSnapsh return BigqueryTableSnapshot( name=snapshot.table_name, created=snapshot.created, - last_altered=( - datetime.fromtimestamp( - snapshot.get("last_altered") / 1000, tz=timezone.utc - ) - if snapshot.get("last_altered") is not None - else None - ), + last_altered=parse_ts_millis(snapshot.get("last_altered")), comment=snapshot.comment, ddl=snapshot.ddl, snapshot_time=snapshot.snapshot_time, diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py index 56a3d55abb184..ba073533eccfb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py +++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_kafka_reader.py @@ -12,6 +12,7 @@ from confluent_kafka.schema_registry.avro import AvroDeserializer from datahub.configuration.kafka import KafkaConsumerConnectionConfig +from datahub.emitter.mce_builder import parse_ts_millis from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.source.datahub.config import DataHubSourceConfig @@ -92,7 +93,7 @@ def _poll_partition( if mcl.created and mcl.created.time > stop_time.timestamp() * 1000: logger.info( f"Stopped reading from kafka, reached MCL " - f"with audit stamp {datetime.fromtimestamp(mcl.created.time / 1000)}" + f"with audit stamp {parse_ts_millis(mcl.created.time)}" ) break diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py index bd6c23cc2d464..c91be9b494c00 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py @@ -7,7 +7,10 @@ from sqlalchemy import create_engine, inspect from sqlalchemy.engine.reflection import Inspector -from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance +from datahub.emitter.mce_builder import ( + make_dataset_urn_with_platform_instance, + parse_ts_millis, +) from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.ge_data_profiler import ( @@ -245,11 +248,7 @@ def is_dataset_eligible_for_profiling( # If profiling state exists we have to carry over to the new state self.state_handler.add_to_state(dataset_urn, last_profiled) - threshold_time: Optional[datetime] = ( - datetime.fromtimestamp(last_profiled / 1000, timezone.utc) - if last_profiled - else None - ) + threshold_time: Optional[datetime] = parse_ts_millis(last_profiled) if ( not threshold_time and self.config.profiling.profile_if_updated_since_days is not None diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/checkpoint.py b/metadata-ingestion/src/datahub/ingestion/source/state/checkpoint.py index 5bfd48eb754d5..2c7a4a8b6c137 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/checkpoint.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/checkpoint.py @@ -12,6 +12,7 @@ import pydantic from datahub.configuration.common import ConfigModel +from datahub.emitter.mce_builder import parse_ts_millis from datahub.metadata.schema_classes import ( DatahubIngestionCheckpointClass, IngestionCheckpointStateClass, @@ -144,7 +145,7 @@ def create_from_checkpoint_aspect( ) logger.info( f"Successfully constructed last checkpoint state for job {job_name} " - f"with timestamp {datetime.fromtimestamp(checkpoint_aspect.timestampMillis/1000, tz=timezone.utc)}" + f"with timestamp {parse_ts_millis(checkpoint_aspect.timestampMillis)}" ) return checkpoint return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py index 11827bace4b5a..9b96953794dcd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py @@ -4,7 +4,7 @@ import dataclasses import logging -from datetime import datetime, timezone +from datetime import datetime from typing import Any, Dict, Iterable, List, Optional, Union, cast from unittest.mock import patch @@ -27,6 +27,7 @@ from databricks.sdk.service.workspace import ObjectType import datahub +from datahub.emitter.mce_builder import parse_ts_millis from datahub.ingestion.source.unity.hive_metastore_proxy import HiveMetastoreProxy from datahub.ingestion.source.unity.proxy_profiling import ( UnityCatalogProxyProfilingMixin, @@ -211,16 +212,8 @@ def workspace_notebooks(self) -> Iterable[Notebook]: id=obj.object_id, path=obj.path, language=obj.language, - created_at=( - datetime.fromtimestamp(obj.created_at / 1000, tz=timezone.utc) - if obj.created_at - else None - ), - modified_at=( - datetime.fromtimestamp(obj.modified_at / 1000, tz=timezone.utc) - if obj.modified_at - else None - ), + created_at=parse_ts_millis(obj.created_at), + modified_at=parse_ts_millis(obj.modified_at), ) def query_history( @@ -452,17 +445,9 @@ def _create_table( properties=obj.properties or {}, owner=obj.owner, generation=obj.generation, - created_at=( - datetime.fromtimestamp(obj.created_at / 1000, tz=timezone.utc) - if obj.created_at - else None - ), + created_at=(parse_ts_millis(obj.created_at) if obj.created_at else None), created_by=obj.created_by, - updated_at=( - datetime.fromtimestamp(obj.updated_at / 1000, tz=timezone.utc) - if obj.updated_at - else None - ), + updated_at=(parse_ts_millis(obj.updated_at) if obj.updated_at else None), updated_by=obj.updated_by, table_id=obj.table_id, comment=obj.comment, @@ -500,12 +485,8 @@ def _create_query(self, info: QueryInfo) -> Optional[Query]: query_id=info.query_id, query_text=info.query_text, statement_type=info.statement_type, - start_time=datetime.fromtimestamp( - info.query_start_time_ms / 1000, tz=timezone.utc - ), - end_time=datetime.fromtimestamp( - info.query_end_time_ms / 1000, tz=timezone.utc - ), + start_time=parse_ts_millis(info.query_start_time_ms), + end_time=parse_ts_millis(info.query_end_time_ms), user_id=info.user_id, user_name=info.user_name, executed_as_user_id=info.executed_as_user_id, diff --git a/metadata-ingestion/src/datahub/utilities/time.py b/metadata-ingestion/src/datahub/utilities/time.py index 0df7afb19935f..e8338ce068c84 100644 --- a/metadata-ingestion/src/datahub/utilities/time.py +++ b/metadata-ingestion/src/datahub/utilities/time.py @@ -1,6 +1,8 @@ import time from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime + +from datahub.emitter.mce_builder import make_ts_millis, parse_ts_millis def get_current_time_in_seconds() -> int: @@ -9,12 +11,15 @@ def get_current_time_in_seconds() -> int: def ts_millis_to_datetime(ts_millis: int) -> datetime: """Converts input timestamp in milliseconds to a datetime object with UTC timezone""" - return datetime.fromtimestamp(ts_millis / 1000, tz=timezone.utc) + return parse_ts_millis(ts_millis) def datetime_to_ts_millis(dt: datetime) -> int: """Converts a datetime object to timestamp in milliseconds""" - return int(round(dt.timestamp() * 1000)) + # TODO: Deprecate these helpers in favor of make_ts_millis and parse_ts_millis. + # The other ones support None with a typing overload. + # Also possibly move those helpers to this file. + return make_ts_millis(dt) @dataclass diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json index dc8c400b29157..fb25531e68526 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json @@ -2658,7 +2658,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1580505371997, + "time": 1580505371996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -2930,7 +2930,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1582319845997, + "time": 1582319845996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3180,7 +3180,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1584998318997, + "time": 1584998318996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3430,7 +3430,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1588287228997, + "time": 1588287228996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3680,7 +3680,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1589460269997, + "time": 1589460269996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json index 60f5bf4fbca9a..69c4b9cce0b17 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json @@ -3024,7 +3024,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1580505371997, + "time": 1580505371996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3296,7 +3296,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1582319845997, + "time": 1582319845996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3546,7 +3546,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1584998318997, + "time": 1584998318996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3796,7 +3796,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1588287228997, + "time": 1588287228996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -4046,7 +4046,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1589460269997, + "time": 1589460269996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_prefer_sql_parser_lineage_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_prefer_sql_parser_lineage_golden.json index 42a416473ae24..0361e899b5b39 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_prefer_sql_parser_lineage_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_prefer_sql_parser_lineage_golden.json @@ -564,7 +564,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1663355198240, + "time": 1663355198239, "actor": "urn:li:corpuser:datahub" } } @@ -636,7 +636,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198240, + "timestampMillis": 1663355198239, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -657,7 +657,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198242, + "timestampMillis": 1663355198241, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1019,7 +1019,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1663355198240, + "time": 1663355198239, "actor": "urn:li:corpuser:datahub" } } @@ -1095,7 +1095,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198240, + "timestampMillis": 1663355198239, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1116,7 +1116,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198242, + "timestampMillis": 1663355198241, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1347,7 +1347,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1663355198240, + "time": 1663355198239, "actor": "urn:li:corpuser:datahub" } } @@ -1418,7 +1418,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198240, + "timestampMillis": 1663355198239, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1439,7 +1439,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198242, + "timestampMillis": 1663355198241, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1871,7 +1871,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1663355198240, + "time": 1663355198239, "actor": "urn:li:corpuser:datahub" } } @@ -1942,7 +1942,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198240, + "timestampMillis": 1663355198239, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1963,7 +1963,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198242, + "timestampMillis": 1663355198241, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -3140,7 +3140,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1580505371997, + "time": 1580505371996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3341,7 +3341,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1582319845997, + "time": 1582319845996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3523,7 +3523,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1584998318997, + "time": 1584998318996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3705,7 +3705,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1588287228997, + "time": 1588287228996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3887,7 +3887,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1589460269997, + "time": 1589460269996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json index c281ea3eed0fa..c59620f010343 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json @@ -564,7 +564,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1663355198240, + "time": 1663355198239, "actor": "urn:li:corpuser:datahub" } } @@ -636,7 +636,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198240, + "timestampMillis": 1663355198239, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -657,7 +657,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198242, + "timestampMillis": 1663355198241, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1019,7 +1019,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1663355198240, + "time": 1663355198239, "actor": "urn:li:corpuser:datahub" } } @@ -1095,7 +1095,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198240, + "timestampMillis": 1663355198239, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1116,7 +1116,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198242, + "timestampMillis": 1663355198241, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1347,7 +1347,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1663355198240, + "time": 1663355198239, "actor": "urn:li:corpuser:datahub" } } @@ -1418,7 +1418,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198240, + "timestampMillis": 1663355198239, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1439,7 +1439,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198242, + "timestampMillis": 1663355198241, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1871,7 +1871,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1663355198240, + "time": 1663355198239, "actor": "urn:li:corpuser:datahub" } } @@ -1942,7 +1942,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198240, + "timestampMillis": 1663355198239, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -1963,7 +1963,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1663355198242, + "timestampMillis": 1663355198241, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -3504,7 +3504,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1580505371997, + "time": 1580505371996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3773,7 +3773,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1582319845997, + "time": 1582319845996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -4023,7 +4023,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1584998318997, + "time": 1584998318996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -4273,7 +4273,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1588287228997, + "time": 1588287228996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -4523,7 +4523,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1589460269997, + "time": 1589460269996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json index 495fa32569f56..23b5525b712d0 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json @@ -2598,7 +2598,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1580505371997, + "time": 1580505371996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -2867,7 +2867,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1582319845997, + "time": 1582319845996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3117,7 +3117,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1584998318997, + "time": 1584998318996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3367,7 +3367,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1588287228997, + "time": 1588287228996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3617,7 +3617,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1589460269997, + "time": 1589460269996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json index 20b7cf4a1c26c..da22458f5624c 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json @@ -2610,7 +2610,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1580505371997, + "time": 1580505371996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -2880,7 +2880,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1582319845997, + "time": 1582319845996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3131,7 +3131,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1584998318997, + "time": 1584998318996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3382,7 +3382,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1588287228997, + "time": 1588287228996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3633,7 +3633,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1589460269997, + "time": 1589460269996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json index 80ca85a5e6c61..0b44fe77cd62a 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json @@ -2599,7 +2599,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1580505371997, + "time": 1580505371996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -2868,7 +2868,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1582319845997, + "time": 1582319845996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3118,7 +3118,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1584998318997, + "time": 1584998318996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3368,7 +3368,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1588287228997, + "time": 1588287228996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3618,7 +3618,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1589460269997, + "time": 1589460269996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json index 1e6e4d8ba94a2..3174847dd7e7a 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json @@ -2599,7 +2599,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1580505371997, + "time": 1580505371996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -2868,7 +2868,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1582319845997, + "time": 1582319845996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3118,7 +3118,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1584998318997, + "time": 1584998318996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3368,7 +3368,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1588287228997, + "time": 1588287228996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", @@ -3618,7 +3618,7 @@ "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1589460269997, + "time": 1589460269996, "actor": "urn:li:corpuser:dbt_executor" }, "hash": "", diff --git a/metadata-ingestion/tests/unit/sdk/test_mce_builder.py b/metadata-ingestion/tests/unit/sdk/test_mce_builder.py index d7c84f7863b40..3bdbf07bf28b7 100644 --- a/metadata-ingestion/tests/unit/sdk/test_mce_builder.py +++ b/metadata-ingestion/tests/unit/sdk/test_mce_builder.py @@ -1,3 +1,5 @@ +from datetime import datetime, timezone + import datahub.emitter.mce_builder as builder from datahub.metadata.schema_classes import ( DataFlowInfoClass, @@ -55,3 +57,18 @@ def test_make_group_urn() -> None: assert ( builder.make_group_urn("urn:li:corpuser:someUser") == "urn:li:corpuser:someUser" ) + + +def test_ts_millis() -> None: + assert builder.make_ts_millis(None) is None + assert builder.parse_ts_millis(None) is None + + assert ( + builder.make_ts_millis(datetime(2024, 1, 1, 2, 3, 4, 5, timezone.utc)) + == 1704074584000 + ) + + # We only have millisecond precision, don't support microseconds. + ts = datetime.now(timezone.utc).replace(microsecond=0) + ts_millis = builder.make_ts_millis(ts) + assert builder.parse_ts_millis(ts_millis) == ts diff --git a/metadata-ingestion/tests/unit/serde/test_codegen.py b/metadata-ingestion/tests/unit/serde/test_codegen.py index 98d62d5643ff2..b49f715312913 100644 --- a/metadata-ingestion/tests/unit/serde/test_codegen.py +++ b/metadata-ingestion/tests/unit/serde/test_codegen.py @@ -6,11 +6,10 @@ import pytest import typing_inspect -from datahub.emitter.enum_helpers import get_enum_options +from datahub.emitter.mce_builder import ALL_ENV_TYPES from datahub.metadata.schema_classes import ( ASPECT_CLASSES, KEY_ASPECTS, - FabricTypeClass, FineGrainedLineageClass, MetadataChangeEventClass, OwnershipClass, @@ -164,8 +163,7 @@ def _err(msg: str) -> None: def test_enum_options(): # This is mainly a sanity check to ensure that it doesn't do anything too crazy. - env_options = get_enum_options(FabricTypeClass) - assert "PROD" in env_options + assert "PROD" in ALL_ENV_TYPES def test_urn_types() -> None: diff --git a/smoke-test/smoke.sh b/smoke-test/smoke.sh index ec8188ebf5f4d..1d209b4ba8219 100755 --- a/smoke-test/smoke.sh +++ b/smoke-test/smoke.sh @@ -22,7 +22,9 @@ else echo "datahub:datahub" > ~/.datahub/plugins/frontend/auth/user.props python3 -m venv venv + set +x source venv/bin/activate + set -x python -m pip install --upgrade 'uv>=0.1.10' uv pip install -r requirements.txt fi From 4e3103e2661f3149f823d1cdda0980fffb7010d3 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 27 Dec 2024 13:50:43 -0500 Subject: [PATCH 21/27] fix(ingest): use `typing_extensions.Self` (#12230) --- metadata-ingestion/scripts/avro_codegen.py | 7 +++---- metadata-ingestion/setup.py | 2 +- .../src/datahub/configuration/common.py | 7 ++----- .../src/datahub/ingestion/api/closeable.py | 6 +++--- .../api/ingestion_job_checkpointing_provider_base.py | 11 ++++------- .../src/datahub/ingestion/api/report.py | 5 ++++- metadata-ingestion/src/datahub/ingestion/api/sink.py | 7 ++++--- .../src/datahub/utilities/urns/_urn_base.py | 12 +++++------- 8 files changed, 26 insertions(+), 31 deletions(-) diff --git a/metadata-ingestion/scripts/avro_codegen.py b/metadata-ingestion/scripts/avro_codegen.py index e5792da32fb5d..2841985ad0780 100644 --- a/metadata-ingestion/scripts/avro_codegen.py +++ b/metadata-ingestion/scripts/avro_codegen.py @@ -154,7 +154,6 @@ def merge_schemas(schemas_obj: List[dict]) -> str: # Patch add_name method to NOT complain about duplicate names. class NamesWithDups(avro.schema.Names): def add_name(self, name_attr, space_attr, new_schema): - to_add = avro.schema.Name(name_attr, space_attr, self.default_namespace) assert to_add.name assert to_add.space @@ -626,7 +625,7 @@ def generate_urn_class(entity_type: str, key_aspect: dict) -> str: class {class_name}(_SpecificUrn): ENTITY_TYPE: ClassVar[str] = "{entity_type}" - URN_PARTS: ClassVar[int] = {arg_count} + _URN_PARTS: ClassVar[int] = {arg_count} def __init__(self, {init_args}, *, _allow_coercion: bool = True) -> None: if _allow_coercion: @@ -640,8 +639,8 @@ def __init__(self, {init_args}, *, _allow_coercion: bool = True) -> None: @classmethod def _parse_ids(cls, entity_ids: List[str]) -> "{class_name}": - if len(entity_ids) != cls.URN_PARTS: - raise InvalidUrnError(f"{class_name} should have {{cls.URN_PARTS}} parts, got {{len(entity_ids)}}: {{entity_ids}}") + if len(entity_ids) != cls._URN_PARTS: + raise InvalidUrnError(f"{class_name} should have {{cls._URN_PARTS}} parts, got {{len(entity_ids)}}: {{entity_ids}}") return cls({parse_ids_mapping}, _allow_coercion=False) @classmethod diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 986dc189cb29b..8357262537bcf 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -15,7 +15,7 @@ base_requirements = { # Our min version of typing_extensions is somewhat constrained by Airflow. - "typing_extensions>=3.10.0.2", + "typing_extensions>=4.2.0", # Actual dependencies. "typing-inspect", # pydantic 1.8.2 is incompatible with mypy 0.910. diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py index 7df007e087979..08817d9d5fdb9 100644 --- a/metadata-ingestion/src/datahub/configuration/common.py +++ b/metadata-ingestion/src/datahub/configuration/common.py @@ -10,7 +10,6 @@ List, Optional, Type, - TypeVar, Union, runtime_checkable, ) @@ -19,14 +18,12 @@ from cached_property import cached_property from pydantic import BaseModel, Extra, ValidationError from pydantic.fields import Field -from typing_extensions import Protocol +from typing_extensions import Protocol, Self from datahub.configuration._config_enum import ConfigEnum as ConfigEnum # noqa: I250 from datahub.configuration.pydantic_migration_helpers import PYDANTIC_VERSION_2 from datahub.utilities.dedup_list import deduplicate_list -_ConfigSelf = TypeVar("_ConfigSelf", bound="ConfigModel") - REDACT_KEYS = { "password", "token", @@ -109,7 +106,7 @@ def _schema_extra(schema: Dict[str, Any], model: Type["ConfigModel"]) -> None: schema_extra = _schema_extra @classmethod - def parse_obj_allow_extras(cls: Type[_ConfigSelf], obj: Any) -> _ConfigSelf: + def parse_obj_allow_extras(cls, obj: Any) -> Self: if PYDANTIC_VERSION_2: try: with unittest.mock.patch.dict( diff --git a/metadata-ingestion/src/datahub/ingestion/api/closeable.py b/metadata-ingestion/src/datahub/ingestion/api/closeable.py index 80a5008ed6368..7b8e1a36162c9 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/closeable.py +++ b/metadata-ingestion/src/datahub/ingestion/api/closeable.py @@ -1,9 +1,9 @@ from abc import abstractmethod from contextlib import AbstractContextManager from types import TracebackType -from typing import Optional, Type, TypeVar +from typing import Optional, Type -_Self = TypeVar("_Self", bound="Closeable") +from typing_extensions import Self class Closeable(AbstractContextManager): @@ -11,7 +11,7 @@ class Closeable(AbstractContextManager): def close(self) -> None: pass - def __enter__(self: _Self) -> _Self: + def __enter__(self) -> Self: # This method is mainly required for type checking. return self diff --git a/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py b/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py index 3680546d307d9..c1a49ce82e6e0 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py +++ b/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py @@ -1,6 +1,8 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Any, Dict, NewType, Optional, Type, TypeVar +from typing import Any, Dict, NewType, Optional + +from typing_extensions import Self import datahub.emitter.mce_builder as builder from datahub.configuration.common import ConfigModel @@ -17,9 +19,6 @@ class IngestionCheckpointingProviderConfig(ConfigModel): pass -_Self = TypeVar("_Self", bound="IngestionCheckpointingProviderBase") - - @dataclass() class IngestionCheckpointingProviderBase(StatefulCommittable[CheckpointJobStatesMap]): """ @@ -32,9 +31,7 @@ def __init__(self, name: str, commit_policy: CommitPolicy = CommitPolicy.ALWAYS) @classmethod @abstractmethod - def create( - cls: Type[_Self], config_dict: Dict[str, Any], ctx: PipelineContext - ) -> "_Self": + def create(cls, config_dict: Dict[str, Any], ctx: PipelineContext) -> Self: pass @abstractmethod diff --git a/metadata-ingestion/src/datahub/ingestion/api/report.py b/metadata-ingestion/src/datahub/ingestion/api/report.py index ade2832f1b669..32810189acd00 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/report.py +++ b/metadata-ingestion/src/datahub/ingestion/api/report.py @@ -42,7 +42,10 @@ def to_pure_python_obj(some_val: Any) -> Any: return some_val.as_obj() elif isinstance(some_val, pydantic.BaseModel): return Report.to_pure_python_obj(some_val.dict()) - elif dataclasses.is_dataclass(some_val): + elif dataclasses.is_dataclass(some_val) and not isinstance(some_val, type): + # The `is_dataclass` function returns `True` for both instances and classes. + # We need an extra check to ensure an instance was passed in. + # https://docs.python.org/3/library/dataclasses.html#dataclasses.is_dataclass return dataclasses.asdict(some_val) elif isinstance(some_val, list): return [Report.to_pure_python_obj(v) for v in some_val if v is not None] diff --git a/metadata-ingestion/src/datahub/ingestion/api/sink.py b/metadata-ingestion/src/datahub/ingestion/api/sink.py index 62feb7b5a02e6..655e6bb22fa8d 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/sink.py +++ b/metadata-ingestion/src/datahub/ingestion/api/sink.py @@ -3,6 +3,8 @@ from dataclasses import dataclass, field from typing import Any, Generic, Optional, Type, TypeVar, cast +from typing_extensions import Self + from datahub.configuration.common import ConfigModel from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.api.common import PipelineContext, RecordEnvelope, WorkUnit @@ -79,7 +81,6 @@ def on_failure( SinkReportType = TypeVar("SinkReportType", bound=SinkReport, covariant=True) SinkConfig = TypeVar("SinkConfig", bound=ConfigModel, covariant=True) -Self = TypeVar("Self", bound="Sink") class Sink(Generic[SinkConfig, SinkReportType], Closeable, metaclass=ABCMeta): @@ -90,7 +91,7 @@ class Sink(Generic[SinkConfig, SinkReportType], Closeable, metaclass=ABCMeta): report: SinkReportType @classmethod - def get_config_class(cls: Type[Self]) -> Type[SinkConfig]: + def get_config_class(cls) -> Type[SinkConfig]: config_class = get_class_from_annotation(cls, Sink, ConfigModel) assert config_class, "Sink subclasses must define a config class" return cast(Type[SinkConfig], config_class) @@ -112,7 +113,7 @@ def __post_init__(self) -> None: pass @classmethod - def create(cls: Type[Self], config_dict: dict, ctx: PipelineContext) -> "Self": + def create(cls, config_dict: dict, ctx: PipelineContext) -> "Self": return cls(ctx, cls.get_config_class().parse_obj(config_dict)) def handle_work_unit_start(self, workunit: WorkUnit) -> None: diff --git a/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py b/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py index 7dadd16fb7f1c..7996fe0d7b89b 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py +++ b/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py @@ -1,9 +1,10 @@ import functools import urllib.parse from abc import abstractmethod -from typing import ClassVar, Dict, List, Optional, Type, TypeVar +from typing import ClassVar, Dict, List, Optional, Type from deprecated import deprecated +from typing_extensions import Self from datahub.utilities.urns.error import InvalidUrnError @@ -42,9 +43,6 @@ def _split_entity_id(entity_id: str) -> List[str]: return parts -_UrnSelf = TypeVar("_UrnSelf", bound="Urn") - - @functools.total_ordering class Urn: """ @@ -88,7 +86,7 @@ def entity_ids(self) -> List[str]: return self._entity_ids @classmethod - def from_string(cls: Type[_UrnSelf], urn_str: str) -> "_UrnSelf": + def from_string(cls, urn_str: str) -> Self: """ Creates an Urn from its string representation. @@ -174,7 +172,7 @@ def __hash__(self) -> int: @classmethod @deprecated(reason="prefer .from_string") - def create_from_string(cls: Type[_UrnSelf], urn_str: str) -> "_UrnSelf": + def create_from_string(cls, urn_str: str) -> Self: return cls.from_string(urn_str) @deprecated(reason="prefer .entity_ids") @@ -270,5 +268,5 @@ def underlying_key_aspect_type(cls) -> Type: @classmethod @abstractmethod - def _parse_ids(cls: Type[_UrnSelf], entity_ids: List[str]) -> _UrnSelf: + def _parse_ids(cls, entity_ids: List[str]) -> Self: raise NotImplementedError() From 6b6d820eea3e7c1297381b2b9ad9b37e22cd9c5d Mon Sep 17 00:00:00 2001 From: deepgarg-visa <149145061+deepgarg-visa@users.noreply.github.com> Date: Sat, 28 Dec 2024 01:49:15 +0530 Subject: [PATCH 22/27] feat(businessAttribute): generate platform events on association/removal with schemaField (#12224) --- metadata-io/build.gradle | 2 +- ...hemaFieldBusinessAttributeChangeEvent.java | 38 ++++++ ...usinessAttributesChangeEventGenerator.java | 98 ++++++++++++++ ...essAttributesChangeEventGeneratorTest.java | 124 ++++++++++++++++++ .../event/EntityChangeEventGeneratorHook.java | 22 +++- .../src/main/resources/application.yaml | 1 + ...tyChangeEventGeneratorRegistryFactory.java | 2 + 7 files changed, 279 insertions(+), 8 deletions(-) create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/timeline/data/dataset/schema/SchemaFieldBusinessAttributeChangeEvent.java create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributesChangeEventGenerator.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributesChangeEventGeneratorTest.java diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index 516a77d59d50b..88bbfa2e10c4c 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -102,7 +102,7 @@ dependencies { testImplementation(testFixtures(project(":entity-registry"))) testAnnotationProcessor externalDependency.lombok - + testImplementation project(':mock-entity-registry') constraints { implementation(externalDependency.log4jCore) { because("previous versions are vulnerable to CVE-2021-45105") diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/dataset/schema/SchemaFieldBusinessAttributeChangeEvent.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/dataset/schema/SchemaFieldBusinessAttributeChangeEvent.java new file mode 100644 index 0000000000000..1f1252e208545 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/dataset/schema/SchemaFieldBusinessAttributeChangeEvent.java @@ -0,0 +1,38 @@ +package com.linkedin.metadata.timeline.data.dataset.schema; + +import com.google.common.collect.ImmutableMap; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.timeline.data.ChangeCategory; +import com.linkedin.metadata.timeline.data.ChangeEvent; +import com.linkedin.metadata.timeline.data.ChangeOperation; +import com.linkedin.metadata.timeline.data.SemanticChangeType; +import lombok.Builder; + +public class SchemaFieldBusinessAttributeChangeEvent extends ChangeEvent { + @Builder(builderMethodName = "schemaFieldBusinessAttributeChangeEventBuilder") + public SchemaFieldBusinessAttributeChangeEvent( + String entityUrn, + ChangeCategory category, + ChangeOperation operation, + String modifier, + AuditStamp auditStamp, + SemanticChangeType semVerChange, + String description, + Urn parentUrn, + Urn businessAttributeUrn, + Urn datasetUrn) { + super( + entityUrn, + category, + operation, + modifier, + ImmutableMap.of( + "parentUrn", parentUrn.toString(), + "businessAttributeUrn", businessAttributeUrn.toString(), + "datasetUrn", datasetUrn.toString()), + auditStamp, + semVerChange, + description); + } +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributesChangeEventGenerator.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributesChangeEventGenerator.java new file mode 100644 index 0000000000000..69d20f2f41bd5 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributesChangeEventGenerator.java @@ -0,0 +1,98 @@ +package com.linkedin.metadata.timeline.eventgenerator; + +import com.linkedin.businessattribute.BusinessAttributeAssociation; +import com.linkedin.businessattribute.BusinessAttributes; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.timeline.data.ChangeCategory; +import com.linkedin.metadata.timeline.data.ChangeEvent; +import com.linkedin.metadata.timeline.data.ChangeOperation; +import com.linkedin.metadata.timeline.data.SemanticChangeType; +import com.linkedin.metadata.timeline.data.dataset.schema.SchemaFieldBusinessAttributeChangeEvent; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class BusinessAttributesChangeEventGenerator + extends EntityChangeEventGenerator { + + private static final String BUSINESS_ATTRIBUTE_ADDED_FORMAT = + "BusinessAttribute '%s' added to entity '%s'."; + private static final String BUSINESS_ATTRIBUTE_REMOVED_FORMAT = + "BusinessAttribute '%s' removed from entity '%s'."; + + @Override + public List getChangeEvents( + @Nonnull Urn urn, + @Nonnull String entityName, + @Nonnull String aspectName, + @Nonnull Aspect from, + @Nonnull Aspect to, + @Nonnull AuditStamp auditStamp) { + log.debug( + "Calling BusinessAttributesChangeEventGenerator for entity {} and aspect {}", + entityName, + aspectName); + return computeDiff(urn, entityName, aspectName, from.getValue(), to.getValue(), auditStamp); + } + + private List computeDiff( + Urn urn, + String entityName, + String aspectName, + BusinessAttributes previousValue, + BusinessAttributes newValue, + AuditStamp auditStamp) { + List changeEvents = new ArrayList<>(); + + BusinessAttributeAssociation previousAssociation = + previousValue != null ? previousValue.getBusinessAttribute() : null; + BusinessAttributeAssociation newAssociation = + newValue != null ? newValue.getBusinessAttribute() : null; + + if (Objects.nonNull(previousAssociation) && Objects.isNull(newAssociation)) { + changeEvents.add( + createChangeEvent( + previousAssociation, + urn, + ChangeOperation.REMOVE, + BUSINESS_ATTRIBUTE_REMOVED_FORMAT, + auditStamp)); + + } else if (Objects.isNull(previousAssociation) && Objects.nonNull(newAssociation)) { + changeEvents.add( + createChangeEvent( + newAssociation, + urn, + ChangeOperation.ADD, + BUSINESS_ATTRIBUTE_ADDED_FORMAT, + auditStamp)); + } + return changeEvents; + } + + private ChangeEvent createChangeEvent( + BusinessAttributeAssociation businessAttributeAssociation, + Urn entityUrn, + ChangeOperation changeOperation, + String format, + AuditStamp auditStamp) { + return SchemaFieldBusinessAttributeChangeEvent.schemaFieldBusinessAttributeChangeEventBuilder() + .entityUrn(entityUrn.toString()) + .category(ChangeCategory.BUSINESS_ATTRIBUTE) + .operation(changeOperation) + .modifier(businessAttributeAssociation.getBusinessAttributeUrn().toString()) + .auditStamp(auditStamp) + .semVerChange(SemanticChangeType.MINOR) + .description( + String.format( + format, businessAttributeAssociation.getBusinessAttributeUrn().getId(), entityUrn)) + .parentUrn(entityUrn) + .businessAttributeUrn(businessAttributeAssociation.getBusinessAttributeUrn()) + .datasetUrn(entityUrn.getIdAsUrn()) + .build(); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributesChangeEventGeneratorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributesChangeEventGeneratorTest.java new file mode 100644 index 0000000000000..fb4c5ca3f9688 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributesChangeEventGeneratorTest.java @@ -0,0 +1,124 @@ +package com.linkedin.metadata.timeline.eventgenerator; + +import static org.testng.AssertJUnit.assertEquals; + +import com.linkedin.businessattribute.BusinessAttributeAssociation; +import com.linkedin.businessattribute.BusinessAttributes; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.BusinessAttributeUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.ByteString; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.timeline.data.ChangeEvent; +import com.linkedin.metadata.timeline.data.ChangeOperation; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.SystemMetadata; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import mock.MockEntitySpec; +import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.testng.annotations.Test; + +public class BusinessAttributesChangeEventGeneratorTest extends AbstractTestNGSpringContextTests { + + private static Urn getSchemaFieldUrn() throws URISyntaxException { + return Urn.createFromString( + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD),user_id)"); + } + + private static final String BUSINESS_ATTRIBUTE_URN = + "urn:li:businessAttribute:cypressTestAttribute"; + + private static AuditStamp getTestAuditStamp() throws URISyntaxException { + return new AuditStamp() + .setActor(Urn.createFromString("urn:li:corpuser:__datahub_system")) + .setTime(1683829509553L); + } + + private static Aspect getBusinessAttributes( + BusinessAttributeAssociation association) { + return new Aspect<>( + new BusinessAttributes().setBusinessAttribute(association), new SystemMetadata()); + } + + private static Aspect getNullBusinessAttributes() { + MockEntitySpec mockEntitySpec = new MockEntitySpec("schemaField"); + BusinessAttributes businessAttributes = new BusinessAttributes(); + final AspectSpec aspectSpec = + mockEntitySpec.createAspectSpec(businessAttributes, Constants.BUSINESS_ATTRIBUTE_ASPECT); + final RecordTemplate nullAspect = + GenericRecordUtils.deserializeAspect( + ByteString.copyString("{}", StandardCharsets.UTF_8), "application/json", aspectSpec); + return new Aspect(nullAspect, new SystemMetadata()); + } + + @Test + public void testBusinessAttributeAddition() throws Exception { + BusinessAttributesChangeEventGenerator businessAttributesChangeEventGenerator = + new BusinessAttributesChangeEventGenerator(); + + Urn urn = getSchemaFieldUrn(); + String entity = "schemaField"; + String aspect = "businessAttributes"; + AuditStamp auditStamp = getTestAuditStamp(); + + Aspect from = getNullBusinessAttributes(); + Aspect to = + getBusinessAttributes( + new BusinessAttributeAssociation() + .setBusinessAttributeUrn(new BusinessAttributeUrn(BUSINESS_ATTRIBUTE_URN))); + + List actual = + businessAttributesChangeEventGenerator.getChangeEvents( + urn, entity, aspect, from, to, auditStamp); + assertEquals(1, actual.size()); + assertEquals(ChangeOperation.ADD.name(), actual.get(0).getOperation().name()); + assertEquals(getSchemaFieldUrn(), Urn.createFromString(actual.get(0).getEntityUrn())); + } + + @Test + public void testBusinessAttributeRemoval() throws Exception { + BusinessAttributesChangeEventGenerator test = new BusinessAttributesChangeEventGenerator(); + + Urn urn = getSchemaFieldUrn(); + String entity = "schemaField"; + String aspect = "businessAttributes"; + AuditStamp auditStamp = getTestAuditStamp(); + + Aspect from = + getBusinessAttributes( + new BusinessAttributeAssociation() + .setBusinessAttributeUrn(new BusinessAttributeUrn(BUSINESS_ATTRIBUTE_URN))); + Aspect to = getNullBusinessAttributes(); + + List actual = test.getChangeEvents(urn, entity, aspect, from, to, auditStamp); + assertEquals(1, actual.size()); + assertEquals(ChangeOperation.REMOVE.name(), actual.get(0).getOperation().name()); + assertEquals(getSchemaFieldUrn(), Urn.createFromString(actual.get(0).getEntityUrn())); + } + + @Test + public void testNoChange() throws Exception { + BusinessAttributesChangeEventGenerator test = new BusinessAttributesChangeEventGenerator(); + + Urn urn = getSchemaFieldUrn(); + String entity = "schemaField"; + String aspect = "businessAttributes"; + AuditStamp auditStamp = getTestAuditStamp(); + + Aspect from = + getBusinessAttributes( + new BusinessAttributeAssociation() + .setBusinessAttributeUrn(new BusinessAttributeUrn(BUSINESS_ATTRIBUTE_URN))); + Aspect to = + getBusinessAttributes( + new BusinessAttributeAssociation() + .setBusinessAttributeUrn(new BusinessAttributeUrn(BUSINESS_ATTRIBUTE_URN))); + + List actual = test.getChangeEvents(urn, entity, aspect, from, to, auditStamp); + assertEquals(0, actual.size()); + } +} diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java index de570cc91b2fe..17e34f151ae01 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java @@ -1,7 +1,5 @@ package com.linkedin.metadata.kafka.hook.event; -import static com.linkedin.metadata.Constants.SCHEMA_FIELD_ENTITY_NAME; - import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; @@ -27,6 +25,7 @@ import com.linkedin.platform.event.v1.Parameters; import io.datahubproject.metadata.context.OperationContext; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Set; @@ -65,6 +64,7 @@ public class EntityChangeEventGeneratorHook implements MetadataChangeLogHook { Constants.ASSERTION_RUN_EVENT_ASPECT_NAME, Constants.DATA_PROCESS_INSTANCE_RUN_EVENT_ASPECT_NAME, Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + Constants.BUSINESS_ATTRIBUTE_ASPECT, // Entity Lifecycle Event Constants.DATASET_KEY_ASPECT_NAME, @@ -83,13 +83,12 @@ public class EntityChangeEventGeneratorHook implements MetadataChangeLogHook { private static final Set SUPPORTED_OPERATIONS = ImmutableSet.of("CREATE", "UPSERT", "DELETE"); - private static final Set ENTITY_EXCLUSIONS = ImmutableSet.of(SCHEMA_FIELD_ENTITY_NAME); - private final EntityChangeEventGeneratorRegistry entityChangeEventGeneratorRegistry; private final OperationContext systemOperationContext; private final SystemEntityClient systemEntityClient; private final Boolean isEnabled; @Getter private final String consumerGroupSuffix; + private final List entityExclusions; @Autowired public EntityChangeEventGeneratorHook( @@ -98,13 +97,16 @@ public EntityChangeEventGeneratorHook( final EntityChangeEventGeneratorRegistry entityChangeEventGeneratorRegistry, @Nonnull final SystemEntityClient entityClient, @Nonnull @Value("${entityChangeEvents.enabled:true}") Boolean isEnabled, - @Nonnull @Value("${entityChangeEvents.consumerGroupSuffix}") String consumerGroupSuffix) { + @Nonnull @Value("${entityChangeEvents.consumerGroupSuffix}") String consumerGroupSuffix, + @Nonnull @Value("#{'${entityChangeEvents.entityExclusions}'.split(',')}") + List entityExclusions) { this.systemOperationContext = systemOperationContext; this.entityChangeEventGeneratorRegistry = Objects.requireNonNull(entityChangeEventGeneratorRegistry); this.systemEntityClient = Objects.requireNonNull(entityClient); this.isEnabled = isEnabled; this.consumerGroupSuffix = consumerGroupSuffix; + this.entityExclusions = entityExclusions; } @VisibleForTesting @@ -113,7 +115,13 @@ public EntityChangeEventGeneratorHook( @Nonnull final EntityChangeEventGeneratorRegistry entityChangeEventGeneratorRegistry, @Nonnull final SystemEntityClient entityClient, @Nonnull Boolean isEnabled) { - this(systemOperationContext, entityChangeEventGeneratorRegistry, entityClient, isEnabled, ""); + this( + systemOperationContext, + entityChangeEventGeneratorRegistry, + entityClient, + isEnabled, + "", + Collections.emptyList()); } @Override @@ -202,7 +210,7 @@ private List generateChangeEvents( private boolean isEligibleForProcessing(final MetadataChangeLog log) { return SUPPORTED_OPERATIONS.contains(log.getChangeType().toString()) && SUPPORTED_ASPECT_NAMES.contains(log.getAspectName()) - && !ENTITY_EXCLUSIONS.contains(log.getEntityType()); + && !entityExclusions.contains(log.getEntityType()); } private void emitPlatformEvent( diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index b997bc108e4ba..f6fa4a37fdadb 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -467,6 +467,7 @@ featureFlags: entityChangeEvents: enabled: ${ENABLE_ENTITY_CHANGE_EVENTS_HOOK:true} consumerGroupSuffix: ${ECE_CONSUMER_GROUP_SUFFIX:} + entityExclusions: ${ECE_ENTITY_EXCLUSIONS:schemaField} # provides a comma separated list of entities to exclude from the ECE hook views: enabled: ${VIEWS_ENABLED:true} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java index cd8eb4f1218db..10770b83ad881 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java @@ -6,6 +6,7 @@ import com.linkedin.metadata.timeline.eventgenerator.AssertionRunEventChangeEventGenerator; import com.linkedin.metadata.timeline.eventgenerator.BusinessAttributeAssociationChangeEventGenerator; import com.linkedin.metadata.timeline.eventgenerator.BusinessAttributeInfoChangeEventGenerator; +import com.linkedin.metadata.timeline.eventgenerator.BusinessAttributesChangeEventGenerator; import com.linkedin.metadata.timeline.eventgenerator.DataProcessInstanceRunEventChangeEventGenerator; import com.linkedin.metadata.timeline.eventgenerator.DatasetPropertiesChangeEventGenerator; import com.linkedin.metadata.timeline.eventgenerator.DeprecationChangeEventGenerator; @@ -59,6 +60,7 @@ protected EntityChangeEventGeneratorRegistry entityChangeEventGeneratorRegistry( BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, new BusinessAttributeInfoChangeEventGenerator()); registry.register( BUSINESS_ATTRIBUTE_ASSOCIATION, new BusinessAttributeAssociationChangeEventGenerator()); + registry.register(BUSINESS_ATTRIBUTE_ASPECT, new BusinessAttributesChangeEventGenerator()); // Entity Lifecycle Differs registry.register(DATASET_KEY_ASPECT_NAME, new EntityKeyChangeEventGenerator<>()); From b79857fd948d29e41611b29678b8c66a91c6f62b Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware <159135491+sagar-salvi-apptware@users.noreply.github.com> Date: Sun, 29 Dec 2024 18:52:05 +0530 Subject: [PATCH 23/27] fix(ingest/sql-common): sql_common to use SqlParsingAggregator (#12220) --- .../src/datahub/ingestion/source/sql/hive.py | 15 + .../ingestion/source/sql/hive_metastore.py | 7 + .../ingestion/source/sql/mssql/source.py | 2 +- .../ingestion/source/sql/sql_common.py | 143 ++--- .../ingestion/source/sql/sql_report.py | 2 + .../hive_metastore_mces_golden_1.json | 120 ++-- .../hive_metastore_mces_golden_3.json | 120 ++-- .../hive_metastore_mces_golden_5.json | 120 ++-- .../hive/hive_mces_all_db_golden.json | 541 +++++++++++++++++- .../integration/hive/hive_mces_golden.json | 537 ++++++++++++++++- .../tests/integration/hive/hive_setup.sql | 2 + .../mysql/mysql_mces_no_db_golden.json | 144 ++++- .../golden_test_ingest_with_database.json | 318 ++++++++-- .../golden_test_ingest_with_out_database.json | 456 ++++++++++++--- .../postgres_all_db_mces_with_db_golden.json | 186 ++++-- .../postgres_mces_with_db_golden.json | 154 ++++- .../golden_mces_mssql_no_db_to_file.json | 232 +++++++- .../golden_mces_mssql_no_db_with_filter.json | 106 +++- .../golden_mces_mssql_to_file.json | 106 +++- ...golden_mces_mssql_with_lower_case_urn.json | 284 +++++++-- .../trino_hive_instance_mces_golden.json | 166 ++++-- .../trino/trino_hive_mces_golden.json | 166 ++++-- 22 files changed, 3254 insertions(+), 673 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py index fad54fda45378..6d67ab29b3a3d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py @@ -838,3 +838,18 @@ def _process_view( entityUrn=dataset_urn, aspect=view_properties_aspect, ).as_workunit() + + if view_definition and self.config.include_view_lineage: + default_db = None + default_schema = None + try: + default_db, default_schema = self.get_db_schema(dataset_name) + except ValueError: + logger.warning(f"Invalid view identifier: {dataset_name}") + + self.aggregator.add_view_definition( + view_urn=dataset_urn, + view_definition=view_definition, + default_db=default_db, + default_schema=default_schema, + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py index adb171d4ad54b..60ecbaf38838a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py @@ -123,6 +123,10 @@ class HiveMetastore(BasicSQLAlchemyConfig): description="Dataset Subtype name to be 'Table' or 'View' Valid options: ['True', 'False']", ) + include_view_lineage: bool = Field( + default=False, description="", hidden_from_docs=True + ) + include_catalog_name_in_ids: bool = Field( default=False, description="Add the Presto catalog name (e.g. hive) to the generated dataset urns. `urn:li:dataset:(urn:li:dataPlatform:hive,hive.user.logging_events,PROD)` versus `urn:li:dataset:(urn:li:dataPlatform:hive,user.logging_events,PROD)`", @@ -160,6 +164,9 @@ def get_sql_alchemy_url( @capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion") @capability(SourceCapability.DATA_PROFILING, "Not Supported", False) @capability(SourceCapability.CLASSIFICATION, "Not Supported", False) +@capability( + SourceCapability.LINEAGE_COARSE, "View lineage is not supported", supported=False +) class HiveMetastoreSource(SQLAlchemySource): """ This plugin extracts the following: diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py index 9d8b67041998c..a2338f14196d7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py @@ -724,7 +724,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ): yield from auto_workunit( generate_procedure_lineage( - schema_resolver=self.schema_resolver, + schema_resolver=self.get_schema_resolver(), procedure=procedure, procedure_job_urn=MSSQLDataJob(entity=procedure).urn, is_temp_table=self.is_temp_table, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 4e22930e7a2a0..a0bd9ce0760bd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -11,7 +11,6 @@ Dict, Iterable, List, - MutableMapping, Optional, Set, Tuple, @@ -36,7 +35,6 @@ make_tag_urn, ) from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.sql_parsing_builder import SqlParsingBuilder from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import capability from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage @@ -79,7 +77,6 @@ StatefulIngestionSourceBase, ) from datahub.metadata.com.linkedin.pegasus2avro.common import StatusClass -from datahub.metadata.com.linkedin.pegasus2avro.dataset import UpstreamLineage from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.com.linkedin.pegasus2avro.schema import ( @@ -106,17 +103,11 @@ GlobalTagsClass, SubTypesClass, TagAssociationClass, - UpstreamClass, ViewPropertiesClass, ) from datahub.sql_parsing.schema_resolver import SchemaResolver -from datahub.sql_parsing.sqlglot_lineage import ( - SqlParsingResult, - sqlglot_lineage, - view_definition_lineage_helper, -) +from datahub.sql_parsing.sql_parsing_aggregator import SqlParsingAggregator from datahub.telemetry import telemetry -from datahub.utilities.file_backed_collections import FileBackedDict from datahub.utilities.registries.domain_registry import DomainRegistry from datahub.utilities.sqlalchemy_type_converter import ( get_native_data_type_for_sqlalchemy_type, @@ -347,17 +338,19 @@ def __init__(self, config: SQLCommonConfig, ctx: PipelineContext, platform: str) ) self.views_failed_parsing: Set[str] = set() - self.schema_resolver: SchemaResolver = SchemaResolver( + + self.discovered_datasets: Set[str] = set() + self.aggregator = SqlParsingAggregator( platform=self.platform, platform_instance=self.config.platform_instance, env=self.config.env, + graph=self.ctx.graph, + generate_lineage=self.include_lineage, + generate_usage_statistics=False, + generate_operations=False, + eager_graph_load=False, ) - self.discovered_datasets: Set[str] = set() - self._view_definition_cache: MutableMapping[str, str] - if self.config.use_file_backed_cache: - self._view_definition_cache = FileBackedDict[str]() - else: - self._view_definition_cache = {} + self.report.sql_aggregator = self.aggregator.report @classmethod def test_connection(cls, config_dict: dict) -> TestConnectionReport: @@ -572,36 +565,9 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit profile_requests, profiler, platform=self.platform ) - if self.config.include_view_lineage: - yield from self.get_view_lineage() - - def get_view_lineage(self) -> Iterable[MetadataWorkUnit]: - builder = SqlParsingBuilder( - generate_lineage=True, - generate_usage_statistics=False, - generate_operations=False, - ) - for dataset_name in self._view_definition_cache.keys(): - # TODO: Ensure that the lineage generated from the view definition - # matches the dataset_name. - view_definition = self._view_definition_cache[dataset_name] - result = self._run_sql_parser( - dataset_name, - view_definition, - self.schema_resolver, - ) - if result and result.out_tables: - # This does not yield any workunits but we use - # yield here to execute this method - yield from builder.process_sql_parsing_result( - result=result, - query=view_definition, - is_view_ddl=True, - include_column_lineage=self.config.include_view_column_lineage, - ) - else: - self.views_failed_parsing.add(dataset_name) - yield from builder.gen_workunits() + # Generate workunit for aggregated SQL parsing results + for mcp in self.aggregator.gen_metadata(): + yield mcp.as_workunit() def get_identifier( self, *, schema: str, entity: str, inspector: Inspector, **kwargs: Any @@ -760,16 +726,6 @@ def _process_table( ) dataset_snapshot.aspects.append(dataset_properties) - if self.config.include_table_location_lineage and location_urn: - external_upstream_table = UpstreamClass( - dataset=location_urn, - type=DatasetLineageTypeClass.COPY, - ) - yield MetadataChangeProposalWrapper( - entityUrn=dataset_snapshot.urn, - aspect=UpstreamLineage(upstreams=[external_upstream_table]), - ).as_workunit() - extra_tags = self.get_extra_tags(inspector, schema, table) pk_constraints: dict = inspector.get_pk_constraint(table, schema) partitions: Optional[List[str]] = self.get_partitions(inspector, schema, table) @@ -795,7 +751,7 @@ def _process_table( dataset_snapshot.aspects.append(schema_metadata) if self._save_schema_to_resolver(): - self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata) + self.aggregator.register_schema(dataset_urn, schema_metadata) self.discovered_datasets.add(dataset_name) db_name = self.get_db_name(inspector) @@ -815,6 +771,13 @@ def _process_table( ), ) + if self.config.include_table_location_lineage and location_urn: + self.aggregator.add_known_lineage_mapping( + upstream_urn=location_urn, + downstream_urn=dataset_snapshot.urn, + lineage_type=DatasetLineageTypeClass.COPY, + ) + if self.config.domain: assert self.domain_registry yield from get_domain_wu( @@ -1089,6 +1052,7 @@ def _process_view( self.config.platform_instance, self.config.env, ) + try: columns = inspector.get_columns(view, schema) except KeyError: @@ -1108,7 +1072,7 @@ def _process_view( canonical_schema=schema_fields, ) if self._save_schema_to_resolver(): - self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata) + self.aggregator.register_schema(dataset_urn, schema_metadata) self.discovered_datasets.add(dataset_name) description, properties, _ = self.get_table_properties(inspector, schema, view) @@ -1117,7 +1081,18 @@ def _process_view( view_definition = self._get_view_definition(inspector, schema, view) properties["view_definition"] = view_definition if view_definition and self.config.include_view_lineage: - self._view_definition_cache[dataset_name] = view_definition + default_db = None + default_schema = None + try: + default_db, default_schema = self.get_db_schema(dataset_name) + except ValueError: + logger.warning(f"Invalid view identifier: {dataset_name}") + self.aggregator.add_view_definition( + view_urn=dataset_urn, + view_definition=view_definition, + default_db=default_db, + default_schema=default_schema, + ) dataset_snapshot = DatasetSnapshot( urn=dataset_urn, @@ -1169,48 +1144,9 @@ def _save_schema_to_resolver(self): hasattr(self.config, "include_lineage") and self.config.include_lineage ) - def _run_sql_parser( - self, view_identifier: str, query: str, schema_resolver: SchemaResolver - ) -> Optional[SqlParsingResult]: - try: - database, schema = self.get_db_schema(view_identifier) - except ValueError: - logger.warning(f"Invalid view identifier: {view_identifier}") - return None - raw_lineage = sqlglot_lineage( - query, - schema_resolver=schema_resolver, - default_db=database, - default_schema=schema, - ) - view_urn = make_dataset_urn_with_platform_instance( - self.platform, - view_identifier, - self.config.platform_instance, - self.config.env, - ) - - if raw_lineage.debug_info.table_error: - logger.debug( - f"Failed to parse lineage for view {view_identifier}: " - f"{raw_lineage.debug_info.table_error}" - ) - self.report.num_view_definitions_failed_parsing += 1 - self.report.view_definitions_parsing_failures.append( - f"Table-level sql parsing error for view {view_identifier}: {raw_lineage.debug_info.table_error}" - ) - return None - - elif raw_lineage.debug_info.column_error: - self.report.num_view_definitions_failed_column_parsing += 1 - self.report.view_definitions_parsing_failures.append( - f"Column-level sql parsing error for view {view_identifier}: {raw_lineage.debug_info.column_error}" - ) - else: - self.report.num_view_definitions_parsed += 1 - if raw_lineage.out_tables != [view_urn]: - self.report.num_view_definitions_view_urn_mismatch += 1 - return view_definition_lineage_helper(raw_lineage, view_urn) + @property + def include_lineage(self): + return self.config.include_view_lineage def get_db_schema(self, dataset_identifier: str) -> Tuple[Optional[str], str]: database, schema, _view = dataset_identifier.split(".", 2) @@ -1411,5 +1347,8 @@ def prepare_profiler_args( schema=schema, table=table, partition=partition, custom_sql=custom_sql ) + def get_schema_resolver(self) -> SchemaResolver: + return self.aggregator._schema_resolver + def get_report(self): return self.report diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py index c445ce44a9144..785972b88a49d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py @@ -5,6 +5,7 @@ from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalSourceReport, ) +from datahub.sql_parsing.sql_parsing_aggregator import SqlAggregatorReport from datahub.utilities.lossy_collections import LossyList from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport from datahub.utilities.stats_collections import TopKDict, int_top_k_dict @@ -52,6 +53,7 @@ class SQLSourceReport( num_view_definitions_failed_parsing: int = 0 num_view_definitions_failed_column_parsing: int = 0 view_definitions_parsing_failures: LossyList[str] = field(default_factory=LossyList) + sql_aggregator: Optional[SqlAggregatorReport] = None def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: """ diff --git a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_1.json b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_1.json index 3ba795a5d044a..8d2e29078880d 100644 --- a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_1.json +++ b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_1.json @@ -87,6 +87,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:1cfce89b5a05e1da5092d88ad9eb4589" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "hive-metastore-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", @@ -160,22 +176,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:1cfce89b5a05e1da5092d88ad9eb4589" - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "hive-metastore-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", @@ -238,7 +238,7 @@ { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258696" + "value": "1735298453" }, { "op": "add", @@ -268,7 +268,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -428,7 +428,7 @@ { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258696" + "value": "1735298453" }, { "op": "add", @@ -463,7 +463,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -672,10 +672,15 @@ "path": "/name", "value": "nested_struct_test" }, + { + "op": "add", + "path": "/customProperties/totalSize", + "value": "0" + }, { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258695" + "value": "1735298453" }, { "op": "add", @@ -697,11 +702,6 @@ "path": "/customProperties/numRows", "value": "0" }, - { - "op": "add", - "path": "/customProperties/totalSize", - "value": "0" - }, { "op": "add", "path": "/customProperties/table_type", @@ -715,7 +715,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -926,11 +926,6 @@ "path": "/customProperties/another.comment", "value": "This table has no partitions" }, - { - "op": "add", - "path": "/customProperties/numFiles", - "value": "1" - }, { "op": "add", "path": "/customProperties/numRows", @@ -943,13 +938,18 @@ }, { "op": "add", - "path": "/customProperties/transient_lastDdlTime", - "value": "1715258689" + "path": "/customProperties/totalSize", + "value": "33" }, { "op": "add", - "path": "/customProperties/totalSize", - "value": "33" + "path": "/customProperties/numFiles", + "value": "1" + }, + { + "op": "add", + "path": "/customProperties/transient_lastDdlTime", + "value": "1735298448" }, { "op": "add", @@ -974,7 +974,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -1164,6 +1164,11 @@ "path": "/customProperties/numRows", "value": "0" }, + { + "op": "add", + "path": "/customProperties/transient_lastDdlTime", + "value": "1735298442" + }, { "op": "add", "path": "/customProperties/numFiles", @@ -1174,11 +1179,6 @@ "path": "/customProperties/COLUMN_STATS_ACCURATE", "value": "{\"BASIC_STATS\":\"true\"}" }, - { - "op": "add", - "path": "/customProperties/transient_lastDdlTime", - "value": "1715258680" - }, { "op": "add", "path": "/customProperties/rawDataSize", @@ -1202,7 +1202,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -1386,6 +1386,11 @@ "path": "/customProperties/numRows", "value": "0" }, + { + "op": "add", + "path": "/customProperties/transient_lastDdlTime", + "value": "1735298441" + }, { "op": "add", "path": "/customProperties/numFiles", @@ -1396,11 +1401,6 @@ "path": "/customProperties/COLUMN_STATS_ACCURATE", "value": "{\"BASIC_STATS\":\"true\"}" }, - { - "op": "add", - "path": "/customProperties/transient_lastDdlTime", - "value": "1715258680" - }, { "op": "add", "path": "/customProperties/rawDataSize", @@ -1424,7 +1424,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -1576,7 +1576,7 @@ { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258672" + "value": "1735298433" }, { "op": "add", @@ -1591,7 +1591,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" }, { "op": "add", @@ -1637,31 +1637,31 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=string].baz", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "int", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].foo", + "fieldPath": "[version=2.0].[type=string].baz", "nullable": true, "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "int", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + "isPartitioningKey": true, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { "fieldPath": "[version=2.0].[type=string].bar", diff --git a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_3.json b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_3.json index a9bf2cb26da49..f408c6c064848 100644 --- a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_3.json +++ b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_3.json @@ -87,6 +87,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:1cfce89b5a05e1da5092d88ad9eb4589" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "hive-metastore-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", @@ -160,22 +176,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:1cfce89b5a05e1da5092d88ad9eb4589" - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "hive-metastore-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", @@ -238,7 +238,7 @@ { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258696" + "value": "1735298453" }, { "op": "add", @@ -268,7 +268,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -428,7 +428,7 @@ { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258696" + "value": "1735298453" }, { "op": "add", @@ -463,7 +463,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -672,10 +672,15 @@ "path": "/name", "value": "nested_struct_test" }, + { + "op": "add", + "path": "/customProperties/totalSize", + "value": "0" + }, { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258695" + "value": "1735298453" }, { "op": "add", @@ -697,11 +702,6 @@ "path": "/customProperties/numRows", "value": "0" }, - { - "op": "add", - "path": "/customProperties/totalSize", - "value": "0" - }, { "op": "add", "path": "/customProperties/table_type", @@ -715,7 +715,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -926,11 +926,6 @@ "path": "/customProperties/another.comment", "value": "This table has no partitions" }, - { - "op": "add", - "path": "/customProperties/numFiles", - "value": "1" - }, { "op": "add", "path": "/customProperties/numRows", @@ -943,13 +938,18 @@ }, { "op": "add", - "path": "/customProperties/transient_lastDdlTime", - "value": "1715258689" + "path": "/customProperties/totalSize", + "value": "33" }, { "op": "add", - "path": "/customProperties/totalSize", - "value": "33" + "path": "/customProperties/numFiles", + "value": "1" + }, + { + "op": "add", + "path": "/customProperties/transient_lastDdlTime", + "value": "1735298448" }, { "op": "add", @@ -974,7 +974,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -1164,6 +1164,11 @@ "path": "/customProperties/numRows", "value": "0" }, + { + "op": "add", + "path": "/customProperties/transient_lastDdlTime", + "value": "1735298442" + }, { "op": "add", "path": "/customProperties/numFiles", @@ -1174,11 +1179,6 @@ "path": "/customProperties/COLUMN_STATS_ACCURATE", "value": "{\"BASIC_STATS\":\"true\"}" }, - { - "op": "add", - "path": "/customProperties/transient_lastDdlTime", - "value": "1715258680" - }, { "op": "add", "path": "/customProperties/rawDataSize", @@ -1202,7 +1202,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -1386,6 +1386,11 @@ "path": "/customProperties/numRows", "value": "0" }, + { + "op": "add", + "path": "/customProperties/transient_lastDdlTime", + "value": "1735298441" + }, { "op": "add", "path": "/customProperties/numFiles", @@ -1396,11 +1401,6 @@ "path": "/customProperties/COLUMN_STATS_ACCURATE", "value": "{\"BASIC_STATS\":\"true\"}" }, - { - "op": "add", - "path": "/customProperties/transient_lastDdlTime", - "value": "1715258680" - }, { "op": "add", "path": "/customProperties/rawDataSize", @@ -1424,7 +1424,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -1576,7 +1576,7 @@ { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258672" + "value": "1735298433" }, { "op": "add", @@ -1591,7 +1591,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" }, { "op": "add", @@ -1637,31 +1637,31 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=string].baz", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "int", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].foo", + "fieldPath": "[version=2.0].[type=string].baz", "nullable": true, "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "int", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + "isPartitioningKey": true, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { "fieldPath": "[version=2.0].[type=string].bar", diff --git a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_5.json b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_5.json index 1937550e1bcbd..7604a96aef825 100644 --- a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_5.json +++ b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_5.json @@ -87,6 +87,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:1cfce89b5a05e1da5092d88ad9eb4589" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "hive-metastore-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", @@ -160,22 +176,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:1cfce89b5a05e1da5092d88ad9eb4589" - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "hive-metastore-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:9ba2e350c97c893a91bcaee4838cdcae", @@ -238,7 +238,7 @@ { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258696" + "value": "1735298453" }, { "op": "add", @@ -268,7 +268,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -428,7 +428,7 @@ { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258696" + "value": "1735298453" }, { "op": "add", @@ -463,7 +463,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -672,10 +672,15 @@ "path": "/name", "value": "nested_struct_test" }, + { + "op": "add", + "path": "/customProperties/totalSize", + "value": "0" + }, { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258695" + "value": "1735298453" }, { "op": "add", @@ -697,11 +702,6 @@ "path": "/customProperties/numRows", "value": "0" }, - { - "op": "add", - "path": "/customProperties/totalSize", - "value": "0" - }, { "op": "add", "path": "/customProperties/table_type", @@ -715,7 +715,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -926,11 +926,6 @@ "path": "/customProperties/another.comment", "value": "This table has no partitions" }, - { - "op": "add", - "path": "/customProperties/numFiles", - "value": "1" - }, { "op": "add", "path": "/customProperties/numRows", @@ -943,13 +938,18 @@ }, { "op": "add", - "path": "/customProperties/transient_lastDdlTime", - "value": "1715258689" + "path": "/customProperties/totalSize", + "value": "33" }, { "op": "add", - "path": "/customProperties/totalSize", - "value": "33" + "path": "/customProperties/numFiles", + "value": "1" + }, + { + "op": "add", + "path": "/customProperties/transient_lastDdlTime", + "value": "1735298448" }, { "op": "add", @@ -974,7 +974,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -1164,6 +1164,11 @@ "path": "/customProperties/numRows", "value": "0" }, + { + "op": "add", + "path": "/customProperties/transient_lastDdlTime", + "value": "1735298442" + }, { "op": "add", "path": "/customProperties/numFiles", @@ -1174,11 +1179,6 @@ "path": "/customProperties/COLUMN_STATS_ACCURATE", "value": "{\"BASIC_STATS\":\"true\"}" }, - { - "op": "add", - "path": "/customProperties/transient_lastDdlTime", - "value": "1715258680" - }, { "op": "add", "path": "/customProperties/rawDataSize", @@ -1202,7 +1202,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -1386,6 +1386,11 @@ "path": "/customProperties/numRows", "value": "0" }, + { + "op": "add", + "path": "/customProperties/transient_lastDdlTime", + "value": "1735298441" + }, { "op": "add", "path": "/customProperties/numFiles", @@ -1396,11 +1401,6 @@ "path": "/customProperties/COLUMN_STATS_ACCURATE", "value": "{\"BASIC_STATS\":\"true\"}" }, - { - "op": "add", - "path": "/customProperties/transient_lastDdlTime", - "value": "1715258680" - }, { "op": "add", "path": "/customProperties/rawDataSize", @@ -1424,7 +1424,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" } ] }, @@ -1576,7 +1576,7 @@ { "op": "add", "path": "/customProperties/transient_lastDdlTime", - "value": "1715258672" + "value": "1735298433" }, { "op": "add", @@ -1591,7 +1591,7 @@ { "op": "add", "path": "/customProperties/create_date", - "value": "2024-05-09" + "value": "2024-12-27" }, { "op": "add", @@ -1637,31 +1637,31 @@ }, "fields": [ { - "fieldPath": "baz", + "fieldPath": "foo", "nullable": true, "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "int", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "baz", "nullable": true, "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "int", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + "isPartitioningKey": true, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { "fieldPath": "bar", diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json index b3922f76d7b0c..a7716f7e10e55 100644 --- a/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json +++ b/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json @@ -118,7 +118,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", + "CreateTime:": "Thu Dec 26 13:11:56 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", @@ -128,7 +128,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166683", + "Table Parameters: transient_lastDdlTime": "1735218716", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -268,7 +268,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", + "CreateTime:": "Thu Dec 26 13:11:56 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", @@ -280,7 +280,7 @@ "Table Parameters: numRows": "1", "Table Parameters: rawDataSize": "32", "Table Parameters: totalSize": "33", - "Table Parameters: transient_lastDdlTime": "1724166687", + "Table Parameters: transient_lastDdlTime": "1735218720", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -458,11 +458,11 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", + "CreateTime:": "Thu Dec 26 13:12:03 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Table Type:": "VIRTUAL_VIEW", - "Table Parameters: transient_lastDdlTime": "1724166690", + "Table Parameters: transient_lastDdlTime": "1735218723", "SerDe Library:": "null", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -608,6 +608,187 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "Database:": "db1", + "Owner:": "root", + "CreateTime:": "Thu Dec 26 13:12:03 UTC 2024", + "LastAccessTime:": "UNKNOWN", + "Retention:": "0", + "Table Type:": "VIRTUAL_VIEW", + "Table Parameters: transient_lastDdlTime": "1735218723", + "SerDe Library:": "null", + "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", + "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "Compressed:": "No", + "Num Buckets:": "-1", + "Bucket Columns:": "[]", + "Sort Columns:": "[]", + "View Original Text:": "select * from db1.array_struct_test_view", + "View Expanded Text:": "select `array_struct_test_view`.`property_id`, `array_struct_test_view`.`service` from `db1`.`array_struct_test_view`", + "View Rewrite Enabled:": "No" + }, + "name": "array_struct_test_view_2", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test_view_2", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", @@ -639,7 +820,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:31 UTC 2024", + "CreateTime:": "Thu Dec 26 13:12:04 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", @@ -649,7 +830,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166691", + "Table Parameters: transient_lastDdlTime": "1735218724", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -793,7 +974,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", + "CreateTime:": "Thu Dec 26 13:12:03 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", @@ -803,7 +984,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166690", + "Table Parameters: transient_lastDdlTime": "1735218723", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -996,7 +1177,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:20 UTC 2024", + "CreateTime:": "Thu Dec 26 13:11:53 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", @@ -1006,7 +1187,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1724166680", + "Table Parameters: transient_lastDdlTime": "1735218713", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1158,7 +1339,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", + "CreateTime:": "Thu Dec 26 13:11:56 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", @@ -1168,7 +1349,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166683", + "Table Parameters: transient_lastDdlTime": "1735218716", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1339,14 +1520,14 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", + "CreateTime:": "Thu Dec 26 13:12:02 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test_view_materialized", "Table Type:": "MATERIALIZED_VIEW", "Table Parameters: numFiles": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166690", + "Table Parameters: transient_lastDdlTime": "1735218722", "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", @@ -1519,7 +1700,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", + "CreateTime:": "Thu Dec 26 13:12:03 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", @@ -1529,7 +1710,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166690", + "Table Parameters: transient_lastDdlTime": "1735218723", "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", @@ -1756,6 +1937,24 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW `db1.array_struct_test_view_2` AS select `array_struct_test_view`.`property_id`, `array_struct_test_view`.`service` from `db1`.`array_struct_test_view`", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:8cc876554899e33efe67c389aaf29c4b", @@ -1875,7 +2074,7 @@ "customProperties": { "Database:": "db2", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:22 UTC 2024", + "CreateTime:": "Thu Dec 26 13:11:55 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db2.db/pokes", @@ -1884,7 +2083,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1724166683", + "Table Parameters: transient_lastDdlTime": "1735218716", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -2080,5 +2279,307 @@ "runId": "hive-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD),property_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),property_id)" + ], + "confidenceScore": 0.35, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD),service)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),service)" + ], + "confidenceScore": 0.35, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW `db1.array_struct_test_view` AS\nSELECT\n `array_struct_test`.`property_id`,\n `array_struct_test`.`service`\nFROM `db1`.`array_struct_test`", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD),service)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),service)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:hive" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),property_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD),property_id)" + ], + "confidenceScore": 0.35, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),service)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD),service)" + ], + "confidenceScore": 0.35, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW `db1.array_struct_test_view_2` AS\nSELECT\n `array_struct_test_view`.`property_id`,\n `array_struct_test_view`.`service`\nFROM `db1`.`array_struct_test_view`", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),service)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD),service)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:hive" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json index 4a0a4886d606a..d24226e3f4544 100644 --- a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json @@ -118,7 +118,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", + "CreateTime:": "Thu Dec 26 13:11:56 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", @@ -128,7 +128,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166683", + "Table Parameters: transient_lastDdlTime": "1735218716", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -268,7 +268,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", + "CreateTime:": "Thu Dec 26 13:11:56 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", @@ -280,7 +280,7 @@ "Table Parameters: numRows": "1", "Table Parameters: rawDataSize": "32", "Table Parameters: totalSize": "33", - "Table Parameters: transient_lastDdlTime": "1724166687", + "Table Parameters: transient_lastDdlTime": "1735218720", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -458,11 +458,11 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", + "CreateTime:": "Thu Dec 26 13:12:03 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Table Type:": "VIRTUAL_VIEW", - "Table Parameters: transient_lastDdlTime": "1724166690", + "Table Parameters: transient_lastDdlTime": "1735218723", "SerDe Library:": "null", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -608,6 +608,187 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "Database:": "db1", + "Owner:": "root", + "CreateTime:": "Thu Dec 26 13:12:03 UTC 2024", + "LastAccessTime:": "UNKNOWN", + "Retention:": "0", + "Table Type:": "VIRTUAL_VIEW", + "Table Parameters: transient_lastDdlTime": "1735218723", + "SerDe Library:": "null", + "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", + "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "Compressed:": "No", + "Num Buckets:": "-1", + "Bucket Columns:": "[]", + "Sort Columns:": "[]", + "View Original Text:": "select * from db1.array_struct_test_view", + "View Expanded Text:": "select `array_struct_test_view`.`property_id`, `array_struct_test_view`.`service` from `db1`.`array_struct_test_view`", + "View Rewrite Enabled:": "No" + }, + "name": "array_struct_test_view_2", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test_view_2", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", @@ -639,7 +820,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:31 UTC 2024", + "CreateTime:": "Thu Dec 26 13:12:04 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", @@ -649,7 +830,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166691", + "Table Parameters: transient_lastDdlTime": "1735218724", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -793,7 +974,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", + "CreateTime:": "Thu Dec 26 13:12:03 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", @@ -803,7 +984,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166690", + "Table Parameters: transient_lastDdlTime": "1735218723", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -996,7 +1177,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:20 UTC 2024", + "CreateTime:": "Thu Dec 26 13:11:53 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", @@ -1006,7 +1187,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1724166680", + "Table Parameters: transient_lastDdlTime": "1735218713", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1158,7 +1339,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", + "CreateTime:": "Thu Dec 26 13:11:56 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", @@ -1168,7 +1349,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166683", + "Table Parameters: transient_lastDdlTime": "1735218716", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1339,14 +1520,14 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", + "CreateTime:": "Thu Dec 26 13:12:02 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test_view_materialized", "Table Type:": "MATERIALIZED_VIEW", "Table Parameters: numFiles": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166690", + "Table Parameters: transient_lastDdlTime": "1735218722", "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", @@ -1519,7 +1700,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", + "CreateTime:": "Thu Dec 26 13:12:03 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", @@ -1529,7 +1710,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1724166690", + "Table Parameters: transient_lastDdlTime": "1735218723", "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", @@ -1755,5 +1936,325 @@ "runId": "hive-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW `db1.array_struct_test_view_2` AS select `array_struct_test_view`.`property_id`, `array_struct_test_view`.`service` from `db1`.`array_struct_test_view`", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD),property_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),property_id)" + ], + "confidenceScore": 0.35, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD),service)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),service)" + ], + "confidenceScore": 0.35, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW `db1.array_struct_test_view` AS\nSELECT\n `array_struct_test`.`property_id`,\n `array_struct_test`.`service`\nFROM `db1`.`array_struct_test`", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD),service)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),service)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:hive" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),property_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD),property_id)" + ], + "confidenceScore": 0.35, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),service)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD),service)" + ], + "confidenceScore": 0.35, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW `db1.array_struct_test_view_2` AS\nSELECT\n `array_struct_test_view`.`property_id`,\n `array_struct_test_view`.`service`\nFROM `db1`.`array_struct_test_view`", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD),service)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view_2,PROD),service)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:hive" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cdb1.array_struct_test_view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/hive/hive_setup.sql b/metadata-ingestion/tests/integration/hive/hive_setup.sql index 323a78e24d10b..c027c174c9355 100644 --- a/metadata-ingestion/tests/integration/hive/hive_setup.sql +++ b/metadata-ingestion/tests/integration/hive/hive_setup.sql @@ -42,6 +42,8 @@ select * from test_data; CREATE MATERIALIZED VIEW db1.struct_test_view_materialized as select * from db1.struct_test; CREATE VIEW db1.array_struct_test_view as select * from db1.array_struct_test; +CREATE VIEW db1.array_struct_test_view_2 as select * from db1.array_struct_test_view; + CREATE TABLE IF NOT EXISTS db1.nested_struct_test ( property_id INT, diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json index 14b03619de4c1..974d10d535861 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json @@ -1550,8 +1550,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": "CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `metadata_index_view` AS select `metadata_index`.`id` AS `id`,`metadata_index`.`urn` AS `urn`,`metadata_index`.`path` AS `path`,`metadata_index`.`doubleVal` AS `doubleVal` from `metadata_index`", - "is_view": "True" + "is_view": "True", + "view_definition": "CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `metadata_index_view` AS select `metadata_index`.`id` AS `id`,`metadata_index`.`urn` AS `urn`,`metadata_index`.`path` AS `path`,`metadata_index`.`doubleVal` AS `doubleVal` from `metadata_index`" }, "name": "metadata_index_view", "tags": [] @@ -2701,35 +2701,42 @@ "upstreams": [ { "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amysql%2Cmetagalaxy.metadata_index_view%2CPROD%29" } ], "fineGrainedLineages": [ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),doubleVal)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),doubleVal)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amysql%2Cmetagalaxy.metadata_index_view%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amysql%2Cmetagalaxy.metadata_index_view%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -2740,18 +2747,95 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amysql%2Cmetagalaxy.metadata_index_view%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),doubleVal)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),doubleVal)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amysql%2Cmetagalaxy.metadata_index_view%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amysql%2Cmetagalaxy.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE ALGORITHM=UNDEFINED\nDEFINER=\"root\"@\"localhost\"\nSQL SECURITY DEFINER VIEW `metadata_index_view` AS\nSELECT\n `metadata_index`.`id` AS `id`,\n `metadata_index`.`urn` AS `urn`,\n `metadata_index`.`path` AS `path`,\n `metadata_index`.`doubleVal` AS `doubleVal`\nFROM `metadata_index`", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amysql%2Cmetagalaxy.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),doubleVal)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),path)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),doubleVal)" } ] } @@ -2762,6 +2846,38 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amysql%2Cmetagalaxy.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mysql" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amysql%2Cmetagalaxy.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "glossaryTerm", "entityUrn": "urn:li:glossaryTerm:Email_Address", diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json index abd9b2350638a..8cf69535f30f6 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json @@ -17,7 +17,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -33,7 +33,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -49,7 +49,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -67,7 +67,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -83,7 +83,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -99,7 +99,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -122,7 +122,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -138,7 +138,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -154,7 +154,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -172,7 +172,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -193,7 +193,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -209,7 +209,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -272,7 +272,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -290,7 +290,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -315,7 +315,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -331,7 +331,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -394,7 +394,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -412,7 +412,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -437,7 +437,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -453,7 +453,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -470,8 +470,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": "CREATE VIEW mock_view AS\n SELECT\n mock_column1,\n mock_column2\n FROM mock_table", - "is_view": "True" + "is_view": "True", + "view_definition": "CREATE VIEW mock_view AS\n SELECT\n mock_column1,\n mock_column2\n FROM mock_table" }, "name": "view1", "description": "Some mock comment here ...", @@ -519,7 +519,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -537,7 +537,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -555,7 +555,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -580,7 +580,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -596,7 +596,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -619,7 +619,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -635,7 +635,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -651,7 +651,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -669,7 +669,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -690,7 +690,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -706,7 +706,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -769,7 +769,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -787,7 +787,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -812,7 +812,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -828,7 +828,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -891,7 +891,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -909,7 +909,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -934,7 +934,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -950,7 +950,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -967,8 +967,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": "CREATE VIEW mock_view AS\n SELECT\n mock_column1,\n mock_column2\n FROM mock_table", - "is_view": "True" + "is_view": "True", + "view_definition": "CREATE VIEW mock_view AS\n SELECT\n mock_column1,\n mock_column2\n FROM mock_table" }, "name": "view1", "description": "Some mock comment here ...", @@ -1016,7 +1016,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -1034,7 +1034,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -1052,7 +1052,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -1077,7 +1077,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -1091,11 +1091,16 @@ "upstreams": [ { "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema1.view1%2CPROD%29" } ], "fineGrainedLineages": [ @@ -1108,7 +1113,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN1)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.2, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema1.view1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -1119,14 +1125,94 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN2)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.2, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema1.view1%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema1.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW mock_view AS\nSELECT\n mock_column1,\n mock_column2\nFROM mock_table", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema1.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD),MOCK_COLUMN1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_table,PROD),MOCK_COLUMN2)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN2)" } ] } }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema1.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:oracle" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } }, @@ -1140,11 +1226,16 @@ "upstreams": [ { "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema2.view1%2CPROD%29" } ], "fineGrainedLineages": [ @@ -1157,7 +1248,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN1)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.2, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema2.view1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -1168,14 +1260,126 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN2)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.2, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema2.view1%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema2.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW mock_view AS\nSELECT\n mock_column1,\n mock_column2\nFROM mock_table", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema2.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD),MOCK_COLUMN1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_table,PROD),MOCK_COLUMN2)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN2)" } ] } }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema2.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:oracle" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema1.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-38ppfw", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2COraDoc.schema2.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-38ppfw", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json index dc0208586d1a1..d57ea0e21479d 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json @@ -17,7 +17,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -33,7 +33,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -49,7 +49,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -67,7 +67,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -83,7 +83,23 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:937a38ee28b69ecae38665c5e842d0ad", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0e497517e191d344b0c403231bc708d0" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -106,7 +122,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -122,7 +138,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -138,7 +154,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -156,23 +172,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:937a38ee28b69ecae38665c5e842d0ad", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0e497517e191d344b0c403231bc708d0" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -193,7 +193,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -209,7 +209,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -272,7 +272,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -290,7 +290,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -315,7 +315,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -331,7 +331,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -394,7 +394,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -412,7 +412,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -437,7 +437,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -453,7 +453,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -470,8 +470,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": "CREATE VIEW mock_view AS\n SELECT\n mock_column1,\n mock_column2\n FROM mock_table", - "is_view": "True" + "is_view": "True", + "view_definition": "CREATE VIEW mock_view AS\n SELECT\n mock_column1,\n mock_column2\n FROM mock_table" }, "name": "view1", "description": "Some mock comment here ...", @@ -519,7 +519,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -537,7 +537,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -555,7 +555,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -580,7 +580,23 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1965527855ae77f259a8ddea2b8eed2f", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0e497517e191d344b0c403231bc708d0" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -603,7 +619,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -619,7 +635,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -635,7 +651,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -653,23 +669,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1965527855ae77f259a8ddea2b8eed2f", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0e497517e191d344b0c403231bc708d0" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -690,7 +690,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -706,7 +706,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -769,7 +769,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -787,7 +787,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -812,7 +812,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -828,7 +828,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -891,7 +891,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -909,7 +909,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -934,7 +934,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -950,7 +950,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -967,8 +967,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": "CREATE VIEW mock_view AS\n SELECT\n mock_column1,\n mock_column2\n FROM mock_table", - "is_view": "True" + "is_view": "True", + "view_definition": "CREATE VIEW mock_view AS\n SELECT\n mock_column1,\n mock_column2\n FROM mock_table" }, "name": "view1", "description": "Some mock comment here ...", @@ -1016,7 +1016,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -1034,7 +1034,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -1052,7 +1052,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } }, @@ -1077,7 +1077,309 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema1.view1%2CPROD%29" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),MOCK_COLUMN1)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema1.view1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),MOCK_COLUMN2)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema1.view1%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema1.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW mock_view AS\nSELECT\n mock_column1,\n mock_column2\nFROM mock_table", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema1.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN2)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),MOCK_COLUMN1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD),MOCK_COLUMN2)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema1.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:oracle" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema2.view1%2CPROD%29" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),MOCK_COLUMN1)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema2.view1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),MOCK_COLUMN2)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema2.view1%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema2.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW mock_view AS\nSELECT\n mock_column1,\n mock_column2\nFROM mock_table", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema2.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,mock_table,PROD),MOCK_COLUMN2)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),MOCK_COLUMN1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD),MOCK_COLUMN2)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema2.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:oracle" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema1.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aoracle%2Cschema2.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-ss8owb", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json index 21898ca246b65..dea5123e93b14 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json @@ -87,6 +87,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a208486b83be39fa411922e07701d984", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0202f800c992262c01ae6bbd5ee313f7" + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a208486b83be39fa411922e07701d984", @@ -160,22 +176,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:a208486b83be39fa411922e07701d984", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0202f800c992262c01ae6bbd5ee313f7" - } - }, - "systemMetadata": { - "lastObserved": 1646575200000, - "runId": "postgres-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:a208486b83be39fa411922e07701d984", @@ -285,6 +285,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a6097853edba03be190d99ece4b307ff" + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", @@ -358,22 +374,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:a6097853edba03be190d99ece4b307ff" - } - }, - "systemMetadata": { - "lastObserved": 1646575200000, - "runId": "postgres-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", @@ -640,8 +640,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": " SELECT metadata_aspect_v2.urn,\n metadata_aspect_v2.aspect\n FROM metadata_aspect_v2\n WHERE (metadata_aspect_v2.version = 0);", - "is_view": "True" + "is_view": "True", + "view_definition": " SELECT metadata_aspect_v2.urn,\n metadata_aspect_v2.aspect\n FROM metadata_aspect_v2\n WHERE (metadata_aspect_v2.version = 0);" }, "name": "metadata_aspect_view", "tags": [] @@ -856,35 +856,105 @@ "upstreams": [ { "auditStamp": { + "time": 1646575200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29" } ], "fineGrainedLineages": [ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n metadata_aspect_v2.urn,\n metadata_aspect_v2.aspect\nFROM metadata_aspect_v2\nWHERE\n (\n metadata_aspect_v2.version = 0\n )", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1646575200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)" } ] } @@ -894,5 +964,37 @@ "runId": "postgres-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:postgres" + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json index fc4a0affac561..e75ea679cecf2 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json @@ -87,6 +87,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a6097853edba03be190d99ece4b307ff" + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", @@ -160,22 +176,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:a6097853edba03be190d99ece4b307ff" - } - }, - "systemMetadata": { - "lastObserved": 1646575200000, - "runId": "postgres-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", @@ -464,8 +464,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": " SELECT metadata_aspect_v2.urn,\n metadata_aspect_v2.aspect\n FROM metadata_aspect_v2\n WHERE (metadata_aspect_v2.version = 0);", - "is_view": "True" + "is_view": "True", + "view_definition": " SELECT metadata_aspect_v2.urn,\n metadata_aspect_v2.aspect\n FROM metadata_aspect_v2\n WHERE (metadata_aspect_v2.version = 0);" }, "name": "metadata_aspect_view", "tags": [] @@ -622,35 +622,105 @@ "upstreams": [ { "auditStamp": { + "time": 1646575200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29" } ], "fineGrainedLineages": [ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n metadata_aspect_v2.urn,\n metadata_aspect_v2.aspect\nFROM metadata_aspect_v2\nWHERE\n (\n metadata_aspect_v2.version = 0\n )", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1646575200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)" } ] } @@ -661,6 +731,38 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:postgres" + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "glossaryTerm", "entityUrn": "urn:li:glossaryTerm:URN", diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 72dcda25c1296..0d9386dcda0cd 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "c2d77890-83ba-435f-879b-1c77fa38dd47", + "job_id": "a06cfdca-b65e-42de-8db2-8c21c183c5dd", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-12-05 16:44:43.910000", - "date_modified": "2024-12-05 16:44:44.043000", + "date_created": "2024-12-26 12:03:35.420000", + "date_modified": "2024-12-26 12:03:35.590000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -2103,8 +2103,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", - "is_view": "True" + "is_view": "True", + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n" }, "name": "PersonsView", "tags": [] @@ -2282,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-12-05 16:44:43.800000", - "date_modified": "2024-12-05 16:44:43.800000" + "date_created": "2024-12-26 12:03:35.230000", + "date_modified": "2024-12-26 12:03:35.230000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2310,8 +2310,8 @@ "depending_on_procedure": "{}", "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", "input parameters": "[]", - "date_created": "2024-12-05 16:44:43.803000", - "date_modified": "2024-12-05 16:44:43.803000" + "date_created": "2024-12-26 12:03:35.237000", + "date_modified": "2024-12-26 12:03:35.237000" }, "externalUrl": "", "name": "DemoData.Foo.NewProc", @@ -4427,8 +4427,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n", - "is_view": "True" + "is_view": "True", + "view_definition": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n" }, "name": "View1", "tags": [] @@ -4891,11 +4891,16 @@ "upstreams": [ { "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CDemoData.Foo.PersonsView%2CPROD%29" } ] } @@ -4906,6 +4911,73 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CDemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW Foo.PersonsView AS\nSELECT\n *\nFROM Foo.Persons", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1735214618898, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CDemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CDemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)", @@ -4916,35 +4988,105 @@ "upstreams": [ { "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CNewData.FooNew.View1%2CPROD%29" } ], "fineGrainedLineages": [ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),firstname)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),lastname)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD),firstname)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD),lastname)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.2, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CNewData.FooNew.View1%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),lastname)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),firstname)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD),lastname)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD),firstname)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.2, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CNewData.FooNew.View1%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CNewData.FooNew.View1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW FooNew.View1 AS\nSELECT\n LastName,\n FirstName\nFROM FooNew.PersonsNew\nWHERE\n Age > 18", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1735214618906, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CNewData.FooNew.View1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),firstname)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),lastname)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD),lastname)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD),firstname)" } ] } @@ -4955,6 +5097,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CNewData.FooNew.View1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -5034,5 +5192,37 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CDemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CNewData.FooNew.View1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 0df89ff1eb94d..07098f0161fc3 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "c2d77890-83ba-435f-879b-1c77fa38dd47", + "job_id": "a06cfdca-b65e-42de-8db2-8c21c183c5dd", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-12-05 16:44:43.910000", - "date_modified": "2024-12-05 16:44:44.043000", + "date_created": "2024-12-26 12:03:35.420000", + "date_modified": "2024-12-26 12:03:35.590000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -2103,8 +2103,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", - "is_view": "True" + "is_view": "True", + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n" }, "name": "PersonsView", "tags": [] @@ -2282,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-12-05 16:44:43.800000", - "date_modified": "2024-12-05 16:44:43.800000" + "date_created": "2024-12-26 12:03:35.230000", + "date_modified": "2024-12-26 12:03:35.230000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2638,11 +2638,16 @@ "upstreams": [ { "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CDemoData.Foo.PersonsView%2CPROD%29" } ] } @@ -2653,6 +2658,73 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CDemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW Foo.PersonsView AS\nSELECT\n *\nFROM Foo.Persons", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1735214621644, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CDemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CDemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -2716,5 +2788,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2CDemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index b36188405e7e1..bf30448469c30 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -137,11 +137,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "b8907be7-52f5-4df4-a870-f4fe0679ec45", + "job_id": "a06cfdca-b65e-42de-8db2-8c21c183c5dd", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-12-19 12:34:45.843000", - "date_modified": "2024-12-19 12:34:46.017000", + "date_created": "2024-12-26 12:03:35.420000", + "date_modified": "2024-12-26 12:03:35.590000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -2532,8 +2532,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-12-19 12:34:45.660000", - "date_modified": "2024-12-19 12:34:45.660000" + "date_created": "2024-12-26 12:03:35.230000", + "date_modified": "2024-12-26 12:03:35.230000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2577,8 +2577,8 @@ "depending_on_procedure": "{}", "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", "input parameters": "[]", - "date_created": "2024-12-19 12:34:45.667000", - "date_modified": "2024-12-19 12:34:45.667000" + "date_created": "2024-12-26 12:03:35.237000", + "date_modified": "2024-12-26 12:03:35.237000" }, "externalUrl": "", "name": "DemoData.Foo.NewProc", @@ -2968,11 +2968,67 @@ "upstreams": [ { "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.demodata.foo.persons,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cmy-instance.DemoData.Foo.PersonsView%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cmy-instance.DemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW Foo.PersonsView AS\nSELECT\n *\nFROM Foo.Persons", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1735214620908, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cmy-instance.DemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.demodata.foo.persons,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,my-instance.DemoData.Foo.PersonsView,PROD)" } ] } @@ -2983,6 +3039,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cmy-instance.DemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD)", @@ -3062,5 +3134,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cmy-instance.DemoData.Foo.PersonsView%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index ebcadcc11dcbf..ff27989d71de1 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "4130c37d-146c-43da-a671-dd9a413a44b3", + "job_id": "a06cfdca-b65e-42de-8db2-8c21c183c5dd", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-11-22 12:58:03.260000", - "date_modified": "2024-11-22 12:58:03.440000", + "date_created": "2024-12-26 12:03:35.420000", + "date_modified": "2024-12-26 12:03:35.590000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -2103,8 +2103,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", - "is_view": "True" + "is_view": "True", + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n" }, "name": "PersonsView", "tags": [] @@ -2282,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-11-22 12:58:03.137000", - "date_modified": "2024-11-22 12:58:03.137000" + "date_created": "2024-12-26 12:03:35.230000", + "date_modified": "2024-12-26 12:03:35.230000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2310,8 +2310,8 @@ "depending_on_procedure": "{}", "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", "input parameters": "[]", - "date_created": "2024-11-22 12:58:03.140000", - "date_modified": "2024-11-22 12:58:03.140000" + "date_created": "2024-12-26 12:03:35.237000", + "date_modified": "2024-12-26 12:03:35.237000" }, "externalUrl": "", "name": "DemoData.Foo.NewProc", @@ -4427,8 +4427,8 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "view_definition": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n", - "is_view": "True" + "is_view": "True", + "view_definition": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n" }, "name": "View1", "tags": [] @@ -4891,57 +4891,66 @@ "upstreams": [ { "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cdemodata.foo.personsview%2CPROD%29" } ], "fineGrainedLineages": [ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),Age)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),ID)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cdemodata.foo.personsview%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),FirstName)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),LastName)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),FirstName)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),LastName)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cdemodata.foo.personsview%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),FirstName)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),ID)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),FirstName)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cdemodata.foo.personsview%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),LastName)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),LastName)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),Age)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cdemodata.foo.personsview%2CPROD%29" } ] } @@ -4952,6 +4961,97 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cdemodata.foo.personsview%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW Foo.PersonsView AS\nSELECT\n *\nFROM Foo.Persons", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1735214622805, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cdemodata.foo.personsview%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),FirstName)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),LastName)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),ID)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),LastName)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),FirstName)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),Age)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cdemodata.foo.personsview%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", @@ -4962,35 +5062,105 @@ "upstreams": [ { "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cnewdata.foonew.view1%2CPROD%29" } ], "fineGrainedLineages": [ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),FirstName)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),LastName)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),FirstName)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),LastName)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cnewdata.foonew.view1%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),LastName)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),FirstName)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),LastName)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),FirstName)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cnewdata.foonew.view1%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cnewdata.foonew.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW FooNew.View1 AS\nSELECT\n LastName,\n FirstName\nFROM FooNew.PersonsNew\nWHERE\n Age > 18", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1735214622810, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cnewdata.foonew.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),FirstName)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),LastName)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),LastName)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),FirstName)" } ] } @@ -5001,6 +5171,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cnewdata.foonew.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", @@ -5151,5 +5337,37 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cdemodata.foo.personsview%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Amssql%2Cnewdata.foonew.view1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json index 6745268ea2c24..fe85b6b4396fb 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json @@ -94,6 +94,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46baa6eebd802861e5ee3d043456e171", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f311add3fdc7c16e8a50a63fe1dcce8b" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-instance-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:46baa6eebd802861e5ee3d043456e171", @@ -169,22 +185,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:46baa6eebd802861e5ee3d043456e171", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:f311add3fdc7c16e8a50a63fe1dcce8b" - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-hive-instance-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:46baa6eebd802861e5ee3d043456e171", @@ -246,7 +246,7 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1724180599" + "transient_lastddltime": "1735206396" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -507,7 +507,7 @@ "numrows": "3", "rawdatasize": "94", "totalsize": "97", - "transient_lastddltime": "1724180605" + "transient_lastddltime": "1735206403" }, "name": "classification_test", "tags": [] @@ -766,7 +766,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1724180602" + "transient_lastddltime": "1735206400" }, "name": "map_test", "tags": [] @@ -993,7 +993,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1724180602" + "transient_lastddltime": "1735206400" }, "name": "nested_struct_test", "tags": [] @@ -1264,7 +1264,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1724180591" + "transient_lastddltime": "1735206384" }, "name": "pokes", "tags": [] @@ -1499,7 +1499,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1724180595" + "transient_lastddltime": "1735206390" }, "name": "struct_test", "tags": [] @@ -1750,7 +1750,7 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1724180601" + "transient_lastddltime": "1735206399" }, "name": "struct_test_view_materialized", "tags": [] @@ -2004,7 +2004,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1724180595" + "transient_lastddltime": "1735206390" }, "name": "_test_table_underscore", "tags": [] @@ -2227,7 +2227,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1724180602" + "transient_lastddltime": "1735206400" }, "name": "union_test", "tags": [] @@ -2529,9 +2529,9 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1724180602", - "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", - "is_view": "True" + "transient_lastddltime": "1735206400", + "is_view": "True", + "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"" }, "name": "array_struct_test_view", "tags": [] @@ -2758,11 +2758,16 @@ "upstreams": [ { "auditStamp": { + "time": 1632398400000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29" } ], "fineGrainedLineages": [ @@ -2775,7 +2780,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test_view,PROD),property_id)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -2786,7 +2792,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test_view,PROD),service)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29" } ] } @@ -2797,6 +2804,85 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n \"property_id\",\n \"service\"\nFROM \"db1\".\"array_struct_test\"", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1632398400000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-instance-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test,PROD),service)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test_view,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test_view,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test_view,PROD),service)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-instance-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:trino" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-instance-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,local_server.db1._test_table_underscore,PROD)", @@ -2956,5 +3042,21 @@ "runId": "trino-hive-instance-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-instance-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json index 34acf6a6e369b..d68f014c93ac8 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json @@ -87,6 +87,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", @@ -160,22 +176,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-hive-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", @@ -233,7 +233,7 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1724180599" + "transient_lastddltime": "1735206396" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -473,7 +473,7 @@ "numrows": "3", "rawdatasize": "94", "totalsize": "97", - "transient_lastddltime": "1724180605" + "transient_lastddltime": "1735206403" }, "name": "classification_test", "tags": [] @@ -755,7 +755,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1724180602" + "transient_lastddltime": "1735206400" }, "name": "map_test", "tags": [] @@ -961,7 +961,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1724180602" + "transient_lastddltime": "1735206400" }, "name": "nested_struct_test", "tags": [] @@ -1211,7 +1211,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1724180591" + "transient_lastddltime": "1735206384" }, "name": "pokes", "tags": [] @@ -1425,7 +1425,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1724180595" + "transient_lastddltime": "1735206390" }, "name": "struct_test", "tags": [] @@ -1655,7 +1655,7 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1724180601" + "transient_lastddltime": "1735206399" }, "name": "struct_test_view_materialized", "tags": [] @@ -1888,7 +1888,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1724180595" + "transient_lastddltime": "1735206390" }, "name": "_test_table_underscore", "tags": [] @@ -2090,7 +2090,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1724180602" + "transient_lastddltime": "1735206400" }, "name": "union_test", "tags": [] @@ -2371,9 +2371,9 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1724180602", - "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", - "is_view": "True" + "transient_lastddltime": "1735206400", + "is_view": "True", + "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"" }, "name": "array_struct_test_view", "tags": [] @@ -2579,11 +2579,16 @@ "upstreams": [ { "auditStamp": { + "time": 1632398400000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Chivedb.db1.array_struct_test_view%2CPROD%29" } ], "fineGrainedLineages": [ @@ -2596,7 +2601,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD),property_id)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Chivedb.db1.array_struct_test_view%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -2607,7 +2613,71 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD),service)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Chivedb.db1.array_struct_test_view%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Chivedb.db1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n \"property_id\",\n \"service\"\nFROM \"db1\".\"array_struct_test\"", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1632398400000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Chivedb.db1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD),service)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD),property_id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD),service)" } ] } @@ -2618,6 +2688,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Chivedb.db1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:trino" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", @@ -2778,6 +2864,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Chivedb.db1.array_struct_test_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "glossaryTerm", "entityUrn": "urn:li:glossaryTerm:Age", From 3723a3e4bcad1fdf2ab8a812b60b52139da398f5 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Mon, 30 Dec 2024 21:06:48 +0530 Subject: [PATCH 24/27] fix(ingest/gc): reduce logging, remove unnecessary sleeps (#12238) --- .../source/gc/dataprocess_cleanup.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index 6d16aaab2d798..3f7a1fc453bcd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -170,6 +170,8 @@ class DataProcessCleanupReport(SourceReport): sample_removed_aspects_by_type: TopKDict[str, LossyList[str]] = field( default_factory=TopKDict ) + num_data_flows_found: int = 0 + num_data_jobs_found: int = 0 class DataProcessCleanup: @@ -265,13 +267,17 @@ def keep_last_n_dpi( self.report.report_failure( f"Exception while deleting DPI: {e}", exc=e ) - if deleted_count_last_n % self.config.batch_size == 0: + if ( + deleted_count_last_n % self.config.batch_size == 0 + and deleted_count_last_n > 0 + ): logger.info(f"Deleted {deleted_count_last_n} DPIs from {job.urn}") if self.config.delay: logger.info(f"Sleeping for {self.config.delay} seconds") time.sleep(self.config.delay) - logger.info(f"Deleted {deleted_count_last_n} DPIs from {job.urn}") + if deleted_count_last_n > 0: + logger.info(f"Deleted {deleted_count_last_n} DPIs from {job.urn}") def delete_entity(self, urn: str, type: str) -> None: assert self.ctx.graph @@ -351,7 +357,10 @@ def remove_old_dpis( except Exception as e: self.report.report_failure(f"Exception while deleting DPI: {e}", exc=e) - if deleted_count_retention % self.config.batch_size == 0: + if ( + deleted_count_retention % self.config.batch_size == 0 + and deleted_count_retention > 0 + ): logger.info( f"Deleted {deleted_count_retention} DPIs from {job.urn} due to retention" ) @@ -393,6 +402,7 @@ def get_data_flows(self) -> Iterable[DataFlowEntity]: scrollAcrossEntities = result.get("scrollAcrossEntities") if not scrollAcrossEntities: raise ValueError("Missing scrollAcrossEntities in response") + self.report.num_data_flows_found += scrollAcrossEntities.get("count") logger.info(f"Got {scrollAcrossEntities.get('count')} DataFlow entities") scroll_id = scrollAcrossEntities.get("nextScrollId") @@ -415,8 +425,9 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: assert self.ctx.graph dataFlows: Dict[str, DataFlowEntity] = {} - for flow in self.get_data_flows(): - dataFlows[flow.urn] = flow + if self.config.delete_empty_data_flows: + for flow in self.get_data_flows(): + dataFlows[flow.urn] = flow scroll_id: Optional[str] = None previous_scroll_id: Optional[str] = None @@ -443,6 +454,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: if not scrollAcrossEntities: raise ValueError("Missing scrollAcrossEntities in response") + self.report.num_data_jobs_found += scrollAcrossEntities.get("count") logger.info(f"Got {scrollAcrossEntities.get('count')} DataJob entities") scroll_id = scrollAcrossEntities.get("nextScrollId") @@ -481,7 +493,8 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: previous_scroll_id = scroll_id - logger.info(f"Deleted {deleted_jobs} DataJobs") + if deleted_jobs > 0: + logger.info(f"Deleted {deleted_jobs} DataJobs") # Delete empty dataflows if needed if self.config.delete_empty_data_flows: deleted_data_flows: int = 0 From 556e6cdb30d87905e7b5495d97eb6472ab5b3951 Mon Sep 17 00:00:00 2001 From: Jay <159848059+jayacryl@users.noreply.github.com> Date: Mon, 30 Dec 2024 12:57:25 -0500 Subject: [PATCH 25/27] fix(docs-site) mobile site and artwork polish (#12237) --- docs-website/src/styles/global.scss | 7 +++++-- .../static/img/solutions/observe-tile-7.png | Bin 84272 -> 87600 bytes 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs-website/src/styles/global.scss b/docs-website/src/styles/global.scss index 31261ea3d940d..855e75c69613f 100644 --- a/docs-website/src/styles/global.scss +++ b/docs-website/src/styles/global.scss @@ -109,7 +109,7 @@ div[class^="announcementBar"] { div { display: flex; align-items: center; - padding: 0 1rem; + padding: 0 .5rem; justify-content: space-between; font-size: 1rem; @@ -156,7 +156,10 @@ div[class^="announcementBar"] { a { color: #EFB300; text-decoration: none; - font-size: 1rem + font-size: 1rem; + padding-right: 0; + padding-left: 12px; + min-width: 108px; } } } diff --git a/docs-website/static/img/solutions/observe-tile-7.png b/docs-website/static/img/solutions/observe-tile-7.png index 2b882d54857ba190bca7aae9e50f8cc39cc880ef..00c1ded63c967eb37cae7fc83760bd3441528ad6 100644 GIT binary patch literal 87600 zcmbrl1ydZ+)&&X-8U{~rmq5@kI1Enkpur&ocY-^EyAAG^Ai>>Tf(9Gh-Q67?_ulWl zKk!ynS683Yr)zibV{7f*VM+>881IPR!NI{{$Vf}5z`-F1!NI|Ipd!DOXmT&hzn#$R zq_rI3;L!2@bHKx;We~mNbA~d{oI4u<4E~JPH!P>I&B_YCuY3; zf}+;IifzF-ib>f7T}`(FIx(S>jU`%%*mv~*ZJLY05C-fzp9=G&EsA}4*Pk`D#Se|e z&89RQ3wmAZPgSn&^`z&I>;1pG5x$3ug<;ME^La639qf+t1+Herl-3lM#yd9t1M8b@iUD53G1M=jelbx_nRD^D^RKD_>&;U zGSSyaxhy&1g2d7~Y*2iy(El|hPqUV;hF!=vPZ&$Sh#`hZd=zn$48)^Uj=F8XZPVlb zDj7?jY5O}gRz~{2;AN*85I6WtN{!P>0d3@UKoB!TG>DyjX^rT|p3kKawbl;BKD7eT zq9uMb`6%LrV7%WbRek>7#=PrPXfX|mu`b|X7Xi1MmX603z9WwxftxliBD@*On9-g0 zyV=>BOSlOeVbbv6p2j}*D>Ctf?olucr zcj1Hq>Z66Z<0jegKP!3tsAiVeorjMY`Dx(MN`G&z$;hF9I;ZV-yNGvpWsyLfWox^N zOGl2e@;Qml0`uMSf^>;06g3ojELJ!m!Kt+0smD(Kn8l*OT$ie|Y5!mIDb>K*)YN9G zIJANhc?U#E zv{l5KmTx0X1k6Z!8<+jR?uUnzT0aq6lb1kc7vWc3+myQ;T|eJ7hMQ>n@(=2gFW`jL zG26PsrMktHZPihGA3J*?H1?ajawe)*3MnfDAB?jkmDZ^mL_dw5+Y)adY{Sv*on?v61D=x{aX2bK1B)a`j4<~W)`S0?m7SYtO-GqKnds|Gj# zraw$espRLuBT$D%0@O5aAgpLHbbNiXZc8|w_gw)f5xbLxifo}MC&O0z6e)a;rqOOk z&O6!L()}{@nF%y)zVrrsFa>dor9yiy87M9Ii)fKrp-kXE_UQljyPr@M{EE?3OZBvn zvA`0!2ynzjf|AN=Q?AF}aKnY7hGGt)nwrJV-Oj+PBt04b3B)N9G~p4gzwfJ zW#e0W2b5&DCKD!Dr5E7b#rJhdz;c5zw1M5Dg>wi!q?^+J8+_}Hf!ccW%dnbk@cv+3 zF8~)$_A8t$gke6`^5aTPU{6tQAR>yy-%L%1jlRX2QGTYL7>STwUJuv#T=5XlY5Q4> z=N-EOE)|WsG4j5_k~RX}Hbt7v6fHI4sz%pRqU4uI?s zNAhW6jYIhGQ*tpYU-~;=PN~HMrEA2xczPikk?xBrRYkwG_qXoeTLZDBEiD4OZ6?D> zO;*R>A8YzEdP61N&8i3nJDrp6*YDe05C_HtV#cx9YzA7jMYLw^J2TsQrP2Oxu=kU^ z`-CbJO+FQsY^m!!twxFx;TQ*B94g{wE(f>}0m1JBZJb0r_F^PZ`kx5D4lQnAXQ?KI zd0+>AuX213uqhP(v;2Q14qn?F4YuG1Q}fBo3>SVD0GEbl0S49}+>!B*_dqnPd@v<$ zH*0?rN*A=4F^(LYoK(~jPR?+NFIuHr9Ap%;)3lpqH5lQXoe6o^Z`99Q{BHKr3?eZOpx0y+7qtZkK(Q&uTLhy&e-D(K{NqkVe#W(BrtSW$wqXe$Ghr@^9;x81dfv! zIqWR&uZ#g@7nO*FUG1#{lpun%*Yt8{gpz%9B>Q$~;3WI|=)En4}cgzW@4 z(SuF1g!85Qo&-FvK2(_$$4#3YY*%Gg%VtmV#UZ0=s-ouqGnW_i%SP3_h5Dw< z-y5sJ$K0+nS#?sziB=14|GckXp*dhT;l$TTgwaSwVbs&-g5C#o3zK8hGYIPyU~ba# z%l_JJt%}EO%i(zgbtHMQmGcsTkT2AjjbUZKIW=VE`NZ`nbPENSO0Gb=TSVI&c&ug+ z6h0ctLUs5R5@$>%RK-*c3V%9-Xk^ zkg^t@dT|fVH39=l=JcHkeTPB17@LJ!i;;yoD`9K2B@i~{6Qza!R+uIC-8Dz&(wN&d zsltyGjBB!Q`kLsWj6$R`2-7e6O9wru>sqlAK1Q&T_W#egAVOnyt6}edB{l@SszoLR zN$U0nImw4>(sNxMqSH8Q7?I0-Xk%@+E02;Sqx2#Q|CD{{1hM9?d~8QE`1FIcF%G!W zM@f4lC8r_b;5?15*=#+jKdL^RIZT`BMtoItth%Ei;BkV0?^A1wDTC zZat0u=9rp`!MmJPU(@aJK0$qC%WdRfET(R7{^Fda#|wkJ7N z;XwOw+13@;KtkbpR5jLqnR&W|I!$uo$werQeH2GZL=7L8Csi(&hmqAZ6Qe{V+0cBZ zqbo0)ILj{Qr8)n%OrGPPFNrssRdQlq_f`>RiTth>jd$G!79~eenU)n%YNrqrEK|PX zhV7o`e{C z7@d|cdk{*eliD5Zv~gl=w6hnl6x`^Crc)DaGfhg7Wfm z(s~g}8(PsjDj#oe@2|uHhu)oVb1wsJL$vUqpdv-|_0%ccY@?yRkf0J2riw)qxnF8N z_-FB9$db~xL=}J8hNVvA7dQHN#&vrKofE~bwFkUJlYHv?_NGqRxa=ZAldsK5*quu9T=S;^9^;{>-xoI)8_DU?XIQceZ&JZK|GzLWjsZ)5DLdb6vE}wrUjr?C3ZGXec zHwj~T>@!hTTFPT`L6%hSCF_W2poKR+V)Z@+D80d!%nEJz$3123Kp3EV)0NxRSGgw9 zyfNtI=-F#9j&Ae@shEP0)$xFD3Fb>!Q(P*0__cC1RlxN`|Ennspb5GRLuYMVOX{ca zc_N5)+3Kl4m-ua#ES!LKi}kxVVsaUN?ny+u&j!3koz$+sGl4`CZO{k+&yoI+ z_yuTdeo`fqEZ~=JUU!@o@W6UF>=*ddF0akc9&GZ>ESqUxLI}oCimHKC+P=xW!~y20 z2EZ(!QTo9~Q_eVz%z)*l0Zg8VN;j>Yi+X*L()`P-^Fu$@g!ZIbB$&;VWhsk(qelmk z7S^}#;hZ3$zU|Az2(Rr8vf8Dwgp}plcJya?q1kK68R3W6)9QS|J1=|#!~?HN+riqc zGKtcB>u<|Hh>StZsXWS%RC53eV%)I|58^ZA%TqN=;muE#$9@`iY=JuTQMQ&NP-*{U z<4&4Yb0g8-#rALneCF<%=ZgDoQ(|8@ZW6=i@9XLx!!RVt6q|whl-}X zoaJjEjP=G0ATu=Da1pWmTWi5etKJ74lz|Yl@P3Y9F}1a z--!LaF4ky!cf%a@HJ!U@PsD|iOsmT93qa);m(@(^!y1ZkBQi(vKlXRnl>ZPH-xiz- zs-Kn0+8q$@9z047Tf^1z?!DBOe{!8K zWM<#1vo$+fY&aF(?I$XgJ!P8J_jxWFVDqV$FWBtA5aG*hy%X2^6=^SVzoAP$0;SB) zUG&LP6bc(N7PUKLkBy#70KzpL=w6CQX-(g6$3}DFCu0iZMh^lowEA&~D~tJez~fXw z7eBwzTXmB`CkzhkN{0BcZAC4eh1B@cHAzkEi)vGq?)!#%B>2z3SXf6@PC+mg$zCzY zrHB~?|LiG~~cAH$qH^%8S@GxkU;aPvI? zU?(KuMNF7+ei>MZE_FB>Oa3I{|GAMwcsyXw_f8W{kz#Nxx&11C5*Zn}J&vM~kd$=k z2c6dL0*Lb5-sDHj>TA6i2DAa$-m8~B z$>$jXGeck{%R%KOE7@qkz^Yvn6GD&OTsNZtZF5Sdh~Yn7sFaza-oRnL8f5XS_Q$`0HN5bw0<@09)2Pp&R$)F;^ZsK@ENH>-V^n89F?w<&7Kf=m(+3 zA#vo7tpqCM(Cn6z=0k8PG=pLn?%w_BykFy8Q4AB_C$vl4YOw=xP!XL_u19jn;(ST`P2e%6zk?t>6aAXprw|C}yU_-~UhOUy zWSC{NzT>CBag`;uZSXCGLe^*ghtf=Je>ZG9e^u=$bC3Twi3fR62UUmKHXh^exkq4| z+=uoF=ju|R(f;Hu^|tj8jEp9-=(+Glzwr6dGe`%sIpGC*W#_9NaQzK)hf;*suo7k7 zFA$|aUQsAD&rBTRpEGZLf^aYTA_F=4$)7Kr5J#j4vAcq0vlh2UCO2=pf1mmJpYsni$SMr5C((*qJ(;Hf3ccY62q3S8ruFdY~>t&LxQX-+mNy<}KEXf~z}$oZ(>{C%*wd=oY%el4ip zXSzjMIL+5}Tf8X6AfzWe)ndKZTs#@O@M^#Kl4%?wwL-8}m@Z|JAeT;#O(E^^V@e)o zwCN!H_Tr}_O9_8Y;ysiG1cX9j2Ud>MFBbQz<+$?(;Ac!cf+8mvB zvsQ;su!>GJwkW^kc}lKBn|7DZ%wwV4CE|BRT+@PLhtBaVMI{>sx~~0Kf3LcfjZYiH z>R5`4=>F>#9ii#{DLV4##>9E)O9We%Rp0%uO2)+Z&kY(kVo$K?Z@=R}bYJ#03R;-4 za-9OR*fBY#*ik0Q8T*ym^IDV^oEjH8AQi_ud`Y$mJQ~qa`7tN^Nv9Of%H=W2N!b%y z?7>Px##*H}o)@9}gDlSKGis?m7&~MAs#5_<+D=5{Y?tpD&AjE0U+*kmxiQ?59fq8% zh&c*sd=6BzEq@^GB!BlP`lMmh$1nJJb39zi-_If)T6B<7AN^@L0-vSZgWJ}iVkoaS0U@^)oPs+%MNmhb>++oe1WcCBGXxvEQg^3b<)Paw>!HZ zANn!-TZkqoP5OJB(bbcI5*KNylgxH?2c1x4(pXa>EP*s7dYV}!O|rdzCd5b1;t!LV zS!dbo7+x9=;HgjB;cdY~UrHaL`e0MwCCV>%n=^hf1}FOvj3Tbit^68E(e*WD6BYBJdHuw7}Q_*za-u zFtm7o%{_T4oqc(4xvF4t=cNc*230u4FT7T*zPLw!L31Ddf?oBR>OT@p*8`YAj#(iT z35~*S%@0_@ZWOfh^78UemQB~~jlR#X_k`#z)&Lp>w5hM8==t~ID`&8k|Wy5KR6}axy2?4%)ZG3zm*4rqG)4rqFPaV zvUVNN9N2x0o0T{ANNgt9hoZl@zOSy@&^`eV|9-OnVc6e&wiS^?W>hMQm#dI{|3iu) zd+_siGRLrz(~2ZSupB=C8O}#teks8s4$w@>Aa*|N1tUZLtJE~8@ZNsNY2c{>6DZAW zm8u|;#J(DkkTBo{Yx@^6iudgJ8S3m8=sW9Cr}C_D<@sa&kn@b#q}x&N#G!NAB6+r& zwMJfABE?0^w6ucR2aJWoq1ER^e;C%ff;Z-JvXuI^=VIx{QXvPf%!T_e za=^{ik?QbTFE>wvpRXpfd6Nh}9&uJ;gm@RuisG71`^_b;B(mJFa)|i@VEa6-&grQR z6l@}`I&nUHFJ`7&{=f0UqpMg~V~kZWwtv<(#+G=6MmdGc~gx&=4c~* z>9{EB0x#L@PUwV@;1P;?`KI*hpO*LrV!pY;zdWCdt3F&l2*i$qTO&P9 z!9^!(|27-g9XrP0ezrd%u7B|2UEfPZhwXMUjrtAn;Y18Tcj&lB1`>1_hn~_V5bWqL z#UEGZe+RKOV$QdG2C@*r0MiHIU1-o_>BkQ?j8~=k;Jk0)5R;7J=a!4vV+vcR#~7Ja zl}A^|N3};YGHeQDTP#*<;jRZb{{Zhip{rc14}kL+pQp`#acRhT;Jf36x)MLV0a1^f z4gceJ#~LeF1w7tkw216ReUUM%PLp5f7SWT58tTR?Pk%%E=3OULP~3r$uVi@5k;q2T zV1k*2xW+T~xh78YQ2-OKB|;v?r?BZ*Oih=R>dP*6+TdA;H`vy04cS_oQRJ3XB2Pe2 z8g$y+lVl&*eY2n6f-mD6sTzy}h%#{oqf%I0>i-#K7+-Zf-`MNGj_^vM@{CM6Gg&bq z-&0Jcc@C|Hp8lb;E|5tIkZ)k7|Fm+u!UwHO2s%I2Om1M+PmNI`H*}xfpKulnVL2Ui z&EHHO3o^Ep_@3~XvtfvVVNBvMDH5qmPyg98B&`N-dGZvm>^sE2I75tP zn{5hjgxwcAfPetT=}*uLUDsu2x0MV5S6jeVh_4M+bfruemShuQO4TQNV2Y?0 zR*;z6>+9!wzIT1~iqEv7YJv}|sp@dH-f8_suM63t;m~Xxkm0W<*d47&Q z?-VGgg}ZWiKU{5Oiftp|hTrz?%Kpki`N%=`VNkIkRE`se-PPSKjkvu!5+rHsS2=9o zrKGmq!^RI3tSrOV4+hE=Gv0lnx?a+LD*oOVDhem{StIx-31*Z6IvK}F7luCQRCS4- zpJ%pcC|bOETW}v`jHsQNWBVd=o|w}in!K$s7nO0* z_znC~FqR>uwso&OP>e5q3YRF9tlF# zM~dzV+t+`>2bbxqPo}l6!DT&PNj_f81PQ$ylvHp-&;Fc|-TQsT8o(Si80)#5uQB5h zx@{U`x!=!z*+)~e*TRM8N~Zx+!Haw-of?wQ`-^hTx#+!1>E-~W6soyfh8T$GSO?u?qW1~I{V0D_t%|gx%85ow`A{07D6%MZd(~D z8c{kame+|Y){|`42r$w(u-7n!)p3`SZJ#>T^5CVMIPaSMD4y&_ojAaK7!4qKv7MOp z&t8juoP@~P>%-yuddE>{sXhrLJ$-yxU2NTPCfyEQ-oVxr`>xs=nr*I894uJ>XDy)^ z!5dBF4&G|VAOSK00gEKE;Z*lcf%EhC_a;rBh30)@g|ZeavgDJpBeOBGPquxT4-ube zYzvUKyPYn`hFb?8s1GsVsA*O)hEM@SEA*Sj+Db01{mnEzc{=yxd(rmnSik^?@p3O0 zjA~diIEn)0+ZDj6|BSGmRKmoAlM0GlAsxb2K@DjqnDGgY(zkq#Z+Nb;$r)L;?c(OnrHQNLT)ofMcu>z6%ge0VERS znWvDU2v8*O9Q~oR4~XU9g^RwTSKB3hts6O=Ko|_u4GutKmR5*B*_IS^jmw%A4>qHu z{K_`S-rC{e7DYzT%Bk5W_;}4P*2*H0S9xl_54ASSlJoEB8(@>U4;o?n&M5)@{k}`s z^>!T9g1ImsURr?U!}4&2_49VPBd3#v7k$qMGPPlXRL@1*+sXNRh_t!-oj3z=Un8%Ws!RivrQ@GJk9a$GbAy|N8LD)%N_uBGPf3M!05q#jrkMMW)!e*|v7_ z3%dA@o;K~0{}byL0e;YIci5J#@p*kgrWN`X{q&1Y+{dDih?0@}#6Jgmy%o6Dl|r5sUI!MK*u1mhCG+AT;K|EdFf)_~G&`S5#)LZ^te znZ9N4=2xC&<5BSCKhm*rx3$p&-(_z;xX!Y&@p3>HAMvNG^|af2=Qi(0mt@R4@sta` zCyy$stPh#EOP`NAr=HjO zU)vw4!afj}^&kTLQ&b2P5*T!6_p*-VGuI}S06%H8`g3BQM*gnHaqIp~(-c^-iluj| z*7)vjq8#idj7nhNEx5GiJ=dqbm3o`{J%Qwe>MLr0-|hj@sB*a*gdPNKsPPiuTbnCsMR@+jR`rybIOr{9y3d?1!mGfv)ZfusHXLCvgC!0j6Oi}Fvt zf%WH_f7SxOhAJ7xP$;~xRsGRr5Dx$g2$NwKL7qvc#Z&<*m-KJpDpq~F5Qp>ok(Ia1 zhwadiYXXzdn=ugLS^GNvzJ%6LV+wg5;QHM$v3s{;!Mt1qa8pt!mMuYWGvS!RAh#?> z?~U?X*ZSlw#t}n@kZIrDuKEbi?1;8FN8q(zB))MJvLsf$G^1tm^chxZ^__pIlOlRG zmiduJqVef}d(4cG_f3qTLnc!*(u@s$u2N?b}Pm2@m66|o`- zCCTYK9g5eRH|an~0>NyMMjGp&O^#qj3IRu4(5y|=yDw$*+{gKF3%Q>?8o8NaKX%LX z`ixpNl~#zBKO873k>-gTuYh)79FwOR4!bwf%H8bJpycNR zs-U7|UJZZ00r5!x=J?BZ5gLuT-8D+OON-qa;_UY0PUhW3k01Uz``BHT z(8jY-P9K*0ny1M9`m!S&U6ts(p#eL%C&yxV5W}IArnCh$J0S0M!v%mlh}>;R7NP@$ z{6pW^dNh5e4>KTwTIWRjf1^D0qGz^hHEZ1c0rU^JNq_pLjeK}T3Be}P>G-=oH9(fB4Kf0P`z=leDI zqR_6>Yqy$ZH5#5M!cb-_lfYDI`gPjtNn5&weOl}1`Z2$$bAnokrl3ZS3!e^5B|y0! z!uEbkvbcI2US!!M*c6G^VavwP{7#+(l`jf5h-9(4Y2yJrja}PIrJTa4VfScrqdJfT zNYDp5l4^VsY(_j59w!pr8m{%Vy;_L%K<*6Ak(6 zsAK}wH#XmIXi86`S2k$cuNpvkb~pS_ibnFj9t9hcm`c!>GW zE+=KOk2Z?xXp2OGjzf#H@9Qbu#zF~Baven7?8FCw!%;T)&v26?4qK1wf_lkCxCZW3 zwr@W5s`V*WtmHy4b@00qDMR8Su$HfE2clv&*eKn1O!M+I?H4xu#0l-f#Pf5KO)4`aG!igih~^RX`YC4p^l z*VFAa0m3f0EZivVJy!1MjWN@r>~KPt|2Qy8ga%Des0G}eP8Jk=_j8sJPpga!`fFw9 zfp;I+Glg_rd{Gx&;J%o>bKN7gr2v_aY;mmJ=&Bg)5oM6V?pug9=c z{LNkEZ#f|YtBwr7>?2xi=uKxMQNWeP+cZw6 zZh0L_gklZ@B#udZ6)x-zQQate@LW2AeC~ORHECsYWi@_A%UBP(9c%V=WscvU9FqB3 z-d0AQwvnZH_3c@ngzHh_?*LZxR~k)nT3%Y-*22)YgtorX&k`Tc{e*Sj!mCVZ} zC`k}=I+TnHsl%Y8sQdU*Htz?Jhm(UOgM&w+Pv{H~hdBrOnDjvc5Fx@lz z@m>Z_!x1gro}`N}NYAO~T0m35V?5phmf6Crj~QMadDVH$BiLL0h#{X#r!`@_kGV^W zlR!LprlhE}ngX#w^n4uu^@C+Q?=UjhYxRy_Qm}QIWZRPF1iK4n&?%T@Ad*kTI~d4n zm5Pb`5vQHQCFxYIou>S?>x>X{>5%`|Vu^eVEh-h@d#13Kl*${qqP2YeGWer0DtN2=M_)HHfSn_n)BH``Ya&xE&jgtav87;cn#Yl&K`Dyoa%KV~ z{!Df@3z=ou_bu9C0?ZQ1^k)kxMcY?D4bU7JONgRVA-AnBFB7V?bn-ivc`O4~hCpDu@vT zIUOG-=FWh(pOx(GuI=mO}8~xzQNpbK1;8HvhZeu5z7TL3}%MUH!Q&x=^?_3WB zG6-zck18CQ4fwDz9q{>N*zq2CO1!JGK|MLrC8khl?7mI}L+E%KtbAUxN>Ok8l6wBU zPJW08iz1?fXtyx2mD6;ewF>G`b#?OfNRdgpF}oI!-FJCj0vaNuurUABcs1AsjAolS z#LZQpT7>w3b+~Hd7!O=VD^q=PCGB3ee?(kS|8xtkJ=c?s6Lqa#vNui_CsxA0nF{Ix z*N~&e#j{g1#(X}fVNWm)wdiw9BAS#$K!?#C0(?{+0xqPFeJ<*Kpya+JhT0z@IKBr5 zqSR9>+4})u1&DcOJ;4}65K2C__Pbw~t8cj8idPidnA|%!@c2!HB9HvG`Eeum6}@wl zcWqejm?*|uWiCf=(IeN=fWJZ7ji|fXa^N;J3n`UtnUXPfF08WhF?XZ<)QBBe_*XW) zm%g%uf2^w5akAv^QY8FRuZZ7cD{^2gr#1c$iDl@6`zLtnHS%J(`@fPo?Bbjuqwp9B z)N1oM)keK=Jwa)ELRX1BqROlXy=mayHr-9p<6m&E$5WC$4 z%Ha%m)=!F(C+HDv*|A-^XtE_ynlQ)}z(&zkwG48spPYK3w|S(8AF`MgpH33=zY-v! zkarB3@+)S^8w}30pLKq0&#XKBUuM!NMnRA7R}51-G8>q)N>WjOUaPwJ&Z(Q4 zF`>tfFeHr2>gN_aus$;-^N8LSD;A!X3kxm-G=Za?42T_efl`k*xZ#N#`qu?-C49^{Yo8*sS^NRXwpaf-OA*2}Tw z_-d^R?)L^l81Nc=zI_sc`cLm6{Ne4ArKKIRD$e0MXLly!5MyO=v`X68FUAo6Fwe7e8Pd{kB7 z+dh8txCBxj${KUYZw#5voEu^+BeTboRlj4VMqF-##Y%y=N;#^)+WACN_U<`%G4wGt zb(ZaQXZeRsI>B=Lv|OZc5%am&Ez3vu{)4CPz|m+aSy$$?jFB%gm;{_XT<@@?M&GR; zV3PcC#1lG9R^>uK0F~^hw-JCbZy=qut#e5RRFG0l=n>^M6djP z1tDf8xf9R2`x0`smS$lR)F)4dnRNF{@&N~bDE&6prIpELONBJ1!aV_F`U{W*VzG6A zCZ!xLjkRRBmRsAOQah4;R5S-IByw z!uCtB32MK$Yvk}7OpA!lEYSLL&APGwwrk~?$~=Vb|WJPGm;5H`rwN%w%HQmMJyF(0BSuu$ZwR79Bqrh3t>OvbVcZ5lBY;ifO zw1rb(25i8LR^$dE9$E-09|@@Q+(RLw0#i=f4Vys>ME_uF&oudBAk5f_AW71P3F<*HO>aum$&Xnxzza2&NKqi!cI zlb+J`*v}{RTKXk7=Zn$lfx{iy_m@E5n%CS=@Ez04nBD2(>Wj+`9pcWsS@ww1ybb9c zLNKJxemB!~hufMS6Zx=ye`pO;pF$oTqUnVB0qM52nbv0`jGp)=;%}J@{H?U`Mn50S zw*vjH?JcWoN#tLp7kx0@1c@Du8td6G@}Sp3MK0-mrw57Mn8R6wr@;?LGWv2GzFkxT zi)ww1DEG@96D*-S%Ju=hEF6NN{oD|8OUkG_?H&)3PzqJ@b*yucbp{PfJqPmKK~)swUUtk*C66wcmo z1g%kOrD=e?8wrZs%8}vN{L%L=79VWXq$yl3Pi5FIP-m$Y@OOUPZ<7?j4eN6aa>K#w z774)=Kuq}1{Y8s9`wzz-tAt-lbiP-eY(vCyGXTv)M1x;7T`i)5`GM&B-B{~XUKupP z0~{MBZM__=jtRm`Ys)zbMsLw>dtViT$Lr(rSp6yY1Ohc9yNGHL?Kjb6b%*BlKbrSZ zU4=JGq+$znB{_HoZRqLZ))rp8)oRK;*zb1zINHHl)t9$27=1F(;f#r-1thLz!16hP zV+hN>a)e5sx8O$jeo^?rR+{lM2O9e5#tIUFuJ47tRiD8K&mWCSXf8{Hs7^cUY2{&+ z(H8=4svx}>Z;Ga}T@1OvYT@TeVLijuW6McVX$x8UZ@RBuw|F}oKG#?~sn=|2v-)1; zL`))Hjqr?ckJ$>{xFdDhk4wAcVs~@VJ=B}cLgt%<@e>4w?(iMZ-1LD!F7HN`#9ibd zvPwahq};?DHyzjvJ!e#=x6p=;zjD`N?^9Jn$vj;4 z>(egEGh>xXhEAyZJ=dROtlICS7!^iE3g!3N3O#o$kK1q=kN+qYjvNDptV{eE8BJKMZwvMNZk0_rC z@a#9y(QIQ`m&Bz2x0<%3>mXZ;LSHu4(DY|wVpa%n&Oy6~S-c=(zwvrTCjv|(IKXzy z_y+_v0d$~J01%$2M;Pb@_JICQG%R+*;A(3=5)}RL0RlP+U^XJc$@XL74IxYj>14}s z2g}0mPo_zTr}S^2c!z87-%pJ^t6q+yw}f&5%vfaH88Gm=>(t8`YCX;Juhs(9C#tVwp!F*wC?YV$yoZ^|0)=`J{Q%x3+eufR?ErJ} z7qfsCh00_0(lJG+KrlQW_KK;M*;o{HF=O5uldI?(#-$6^b{k z?;)f04Xgf{Smo#CM~{Q^6qe~6KhsnQd52T=O#P2NeepL`H&}TrdnTrPbp8?a4LRTw ztSt{@E^t^>md@;UInEO1*?%;1P(DJ>1Ob!sMb`vY??;=GmRh^jyYp|pFq_=nfyO3(UN zwt8Q7o37wnT&_@YY&5(^C9qOn8;E-#CH4KOUC%A!vSCQkf}n-_eQ(I46M599V=ZYI zUiCeij0YU6_Mg)TGljapBa$-FL--7NSx_rKJF@o{8h)#jio$Ky`>=e9st>Qpr8RuH zz+(R>vF6PtFh0`9rN9#lALpk_A&s@tcjAwmHN{T77kVGm=bf2Zs(9bxNV@%Gmv~H* zn(jf}#KV`(+rNgvN#CErJNn6qH>zqrhcmfK#KjvE79|%14kTiGWC?AaWc5Ir9qt|a zy4T;cFWZ7-^n+hn)t`{iHa5JOg=K_t!w)L-2i{M66?dD%e574!^?0`kn9_rs{3_Z`BP$K;UPt9NrbLpF*zpwbN> zw%oJ2*YP%Jy3)&KT$g&1&z@ew^sZ)8?}D89kJ)XU3p~qyPz6Ysy`#z_uSfpBywIJM z&H&u+$ac+mLL>j(3U^LM#2eAcUtY8upnH8OUO)TO`;zJ3#w0#3ni4%QGgT6rt&|#r zUoUS~p;cpAlz8(#=rCV!bU-UD3H!6=?#ws8TabNee`s4Xh+R(*Pj&86T}E-#QPEeD zSv!TfzhVZHr`_NjPOvXGX33BDm5JlBdbv0NU*gd_iWF`V-uwrmG_Ql#(jYnRI z$p6huVFJ)G|2%Dhnb?L|5L`W-_iJ&1xZq;a&?XmK7*#S2A{@R?J zj&y1I;kKfhkqt^7_p1ud!uVc=apqI&kqMg)d>9swbZ{~H87@T=oauQr`>0WC$k&QC zC&^JYX9!qH+O4`XIM1ss%cJObk3>G`z~4hHBk?d4KW zCnDse33@%JId$mWjnIE=SNquplkqBlPEY@*NDk?HN?!HoXdy?>NfU2!S{OCEJz1Xp zm&&or#H(L#trfkEXg7N8#heByEBHly_xFeUcX^xJV@NlnwJW)jV-RVv8h(W51u+p^ zKzvO@qbe6Fw*PJM=kJazJ!^ozaOH zq!>Apk;l1*7;EpYkJx9qI;O3{rJJcl%IKbWW;PQ-?>r3VrmRcPs+wA5omie;qQ84L&clU z|FlYhXz`hukGvaSfdiNt+BXpt+&MGabT=h_j1iu7X9J1)O_b|z8+@U`V9H#9VwyRD zzXt}x{PWZq;yPa;ZmXe{V(TY@t;0Ydrq*BxajJYeCVj$5wUdRg%2G-8U;Zggx2mfH z&OR7vcnf2eCjanIkpd}ma`Dr?n~}x13E8kK^GNPh-&22iNq6M^^!_{15#C}oNt09d zEupJY<}7|^twMTr4MSV;bG5%(Nvl&Y{T0suZ4a8|p-0`4U@1M*Oyt2H%}^}q(Wr)} z2pYaOhxwZQbiZUGy?2Mb^dWHRR2o%^p%!L)o;2+)X?rZyBMg(@sV(PI(rDj{Th5&a z$KOd;$2Zt&tpEHnt+y0WayA+z-9}Z@f;RmDpY6)?f>79Mm|d)V>QH0GSpwx`i&#)| zw+kefcbWVZ%|-C3a{jy%P~dJpz+k=bbib33{qSj0&?q?8-9aB~>^Xq_u*3F(hEkB* z(~<4-{e{oZe^)`E({h~UR~Xj6--|xh$Z05gzmmt?>)fwpesXaXAkxS230JXk&c6vz z-Hgkw%N3I;#8fWA38qElXkka<7Kw>k!DY+A-5B@ie+fL))r3seL!N@40*Q-9br>Me zy8UIF7~2BxV_Yi6N?;UBw@PV-28UpcZTc(uf)WLiSc9axGMB}#f1>*~FqRdMeXfp$ zw?3Lyl3In_IWN=~s@}aEWAR&Hnc649*EG*)ei|p}xB--0cx3M%6DOfV zQGN1i4AW-SmfL$*U}vCRJxrnf#ltA;L5GQWp`SM0tH0-?=eXrW^by+>ogMi(e_7o1nVaG7P^snTsQ}fz%$YBlAgHbQE)`F%sM`1bjpUO>86CC7 z;&6?X7@Pr3@c#o@L8iX3ICVU!{g%<_W>f%5m+1LmbmpHbpz{pLa)bC7RDL5$4~@gByk5NHj>2tzaO;qy$2k(6tvxpTU$oDzFYfI< zywg0g)YRhEhD z#6syUi%Xs(tkC)%G52L&wxo~7C@^em*-i-jM{V>@|>`KfcR)rD1Q z3bm24G-|gx%&Pc8_e1I{o`>pJ)w|Z=#_C7avsju!c>#{)_AjqP>b+3@Wc#Pv@fz%7 zlDxX$Os#iZ@w%4#RBf%ixK;H9a^J#s zc?p=&r6J5`CREo&RaZ*|%fvMHz`1jV%h;lYAVFNT5F#!ZFmp;9)LtwAxsjqJJPo;g zJfC41`g`uV2m1mo)eV^G?%%S-9Nx2MM9B@M)JlG~Z8JXoCg1mrd(g?C8N`A{68uh5uSzN%bf?_c^4g%RsmV{z*A$m>wqn;Lo1 z{>$D)Ed83J|pD3>TxdBg@0 zeJ-De;?ZtG=?~>W`whjR-&LJ*k)2y8jrw^kO)~yiUPIyPutNSr>YG?zq<>vLp}bV3 zDVEpJb7r=?M_O3EHW+inMlGza5q~=W_%*@P=dsZ)~U>B;{$1r-*wlKCGNrwk^||E)bVk~ z7S6f5u{}2BPRf=wHv9Veik+}kI*N0obHOE-6t+APQ)bRI^IrYxVLx5HmeIV=n5Vu_ z02tS-hg6lZQ`PMvGcn*FK~X$qxHmZ$8vv|H&X)+tF* zt|=*xSSP0M#l9|Mel^PS68qY#N<&tne$fcs)8$+yZII!F>U$_%Wdo9S6B|rr8RHic z-|M;(E6-5S;jIyb)*@3u&b7o z2~+z|?Z008550?6eG8>Y*OyLU%9JTZys{0I0n!}I&LZJ~x+{jCJp63*+3BY(7+USw zs?1)0@yktJuzZ;IPGRrfy&G{{zkWRoo10v@8RIo0>K%Q!?~cm|?_)g^6VInqOk z56-zLlI)ay+FH(ME?@2v1|#Qz9XlL(nZ9U|ImO1;)z{bR=d>%0y=n3d-xxA>?(Q?$ z3@hn*`A4kyXrYmK^}@F}k^}OycVL&9+L&%GnEwWI{`}XQ5bAUmp}|n*ubs|AcLXsv zb;67J2SWa2C(P%(7wddOX^+Jp>x4t;45hbjd4|efBbL_C7plAtkr4Xhb3K@a;$+w| zy)sT2k0dQ2^1^$X+{nC?4Tx0(74LM?c!+x;`3?xYKby?SJyLe~D0@LWG(}S%2tm#*7&q z9rUp?J3iFg*`0mu7#AUwfaIs{nm$ksVpCP*f7Yy7rjt1Bw9}kKVd>JP`IaqP+_ZMA zw`WOC6GqMuU~oM?2s>ccOA{Dj-y=cA^ovLr$PH3N`6oPHq)qmO+RBR^P8AuS6p#LcIKI9n(MB+&V>ik zgV|t+2dhF_M(L6$(^1uEV-0z<{$lRR2I1KAvN-Cvk6r6dgwN#Qq|;SJ5M)QvPTE3* zAcW96EsIBn8_G{v7*%1*FslX*4J~ESoiXq8OA+VT_XiykIwxLB{~Y*n&Xs93$pAPdd*s?ed!CB-5=W!TNpGX)B@URY>i> zIxS`HLic4np)||1Nkiw5Ul|75S*I{{>QqO3WE-f$@-_{UI|cSBmXw%I!j}6(JYTbB zO^z`d_PE$kT$3IEc0PVt8?NP(`F*-1!J+1;xMxf&c~Ru1AVYa%LRN@00o@_S!~li= zf}KfAY`F6a6-pB1c3j1*E|8jQuDPax>FVCgAQmu$WCtrWg9ToF_0=$HZdz7e?D0ii zpkbehY;3WZtbCX`oMcCk)QrzJ!{ zLh;WOUnHJ(X#`p2~EuoyoH51-5T%^#o1 zH1fVIADupiN4F$aycdc?KhtsQaCv0d^v68uwk5-1dUbio9|k^62CCX!=v}AwZ`yJ* z9_^q0c)t!Ilx>E7DfQ?isN#|z6O#?hHzZV@p3&9c-~U5wKV{WuuJ9@Gfwad8G2GHL zfDn2(h}m3)=14rwRGrA^PWtiB_O`n9449 zW%ooV=W^B9eVt*;g$fA}9!_09_uO;y#~**(o#UQmkLI8L>7QWZ^9g1;23g_DzJ{b* z9j5+*k|KY-QFrEmlyqpxg_e-u8GeTkA1!~91KgRTb2p^u&hFEwKNDvnSYsfJ`26)P#w@n?Y}(TlcoJ*(AHAS7~!Ws zDGkLDb-R;59KF50?woaUGJ*LW4`~n7Tp#$r2Yw>&_4V~NVVg^#21pN9ZYJr0tq;kL zHg5K|!R|U!TV-WJ3zd5bPBy%QL?+(9tWcGWb3{NR9H<#xfSkQ07b2pa501QtOyfv( z7XT83f?*+1+Oy1x%K)%Poa4A?(V}e4nl%uat(ZT5{t8>@8Ay;k+Lw$iEpq=-GBgrg z`^xA@8eYDQr{OC>;TF#sj!^|hvgMx@_wYq(<7`=tTG>@{$dK_xz zvFXly=btg|&{`BCy(6H%jF?}#N4cQ9V&R1RGkx$U^GUyD=@BT(7inN#jd(x4Oeo&W zr8`j$NJCS^9qn{fUS;@3)C2vlTD}}rT8q4(zM?+i!k#yL85 zgDaDx91)g3X13GHeW!}^rc5jD!JoWFnD$=Ne(CW;qUoimu0HYIa#KNOCS(A~ck0PT z@t)yE1-a;rI0PG-bbFrNd3WYl+nU zC$9agwp5g_50is)kypec`)Ei}&>rj<)WjeK_ck1uXYe}1tY52k$ynx02ipwV0>V4x zlv7?mZ{EE3$#JYiQZPdD6q8i#+qcicmku?7HDsp`A2lk*SR_H=Rm<7TB{+*u@kLj-qvR)b_X7B#bL;dGE2E z?`{q_?ASR-mkmB4r1n4Y?Y|BX4@n^6!??irn-59*+%@jQt!$Hk_$Y97!%Aot$LdBF zQOo6dSVBx7>^*z-xchZfUa>G^j{mZ>GF>}%?0El$7hZS+ z-pk1+pPX&qzP*VL6WHitR@dd1U+!jZb1ff?|Jb9G>Mk$w0iSS7;fVH^8=<-yE{?c4 z4$G@<`i&<;a*o>wW|WWImAzP~5QGP;%`-Q;!&G)ig4_y?u<(Y6%aR+Y!}6=IzB-37 zGf5Auz85ZBh}E54IQQIhPeDyBlgwc_#jjVJraAVv)I?CHv!k1;%0CuV#nEm!Rrkk| zm#VPJ!sujK)p1jnwmNQN;ns;?HY|Ka#8NCTD4ZSV)qic>ZufaaDulQpRj~tnF{2Ra zs3I=0@I&QVMV5G^8Xd3_&=I63l%_J06(WC7bJ?-~Mi3rLfO6RQP*Q9O;Gh*=xMgYV zxZecn!R&26{x>lL9Fv~7%LBAU;1QniApAm#%DAnYMkZ!@h0-U>L^~mi63>RDlqwKP zcuH%v--@D1m6kMWM#doV0h5F64`z6Q1%mAC+_}@a=av+32MchV!_Eg%9xv^2sxC~0 zv#Kt)KvE4=NP#JJ3**MX59vzoZHZ{3Cc_L#lw^D|%uqb?JVag)C-VoDTWI=v9#CtA?(1i=-?Q=D ziTjrHK$Tpm>AmXT&Ca%Cy<5MBGrKVUAAb1ZZWN5w88$7+>4G6MR)2({6gEJ!jM7jSZW0oXA~fOQc^Y7k z4)|5CXMyoqdN;=&$_8h?Z{82RZ+Y(3VMSOpke0J3WgMaSW!&;y69SM8gnjVfLE~+a zTK4tQ>C$Nq#X~oe*oXXr@Gzf{@{oju%26Wr!62n(I(>KFefNKddhxmEo@?0g+1U`< z@tn4VZl?tZIj|2W_yLZb*@boe;70HTENbjDM}QzSAUW=Yiyh`e7uDqVU)=YjI?@9%fShhO&cD<4S2 z=h#4qP{?NzQjMj0Nwz9go-YfCwZ?CScEi` zCP{iA<>5@TsyNE>7K6>(*Dkr_l4~H00BkunF+SrQc0M>_JePi;83Or%osYlf(a<(n z@74~;d!gsj9i=^bTD-b0)Lu9btz+F{{hX^g^KhV~JWZ^|eA{iexqUbw&cal7==ZQU z7bHIL1HuD<^g!YRcbK}qa^*@V<-s$q1>kjt`TY6wonNTClEf$BXd>1bOH!0Du}CV% z;{)*Br{wgJkqln$N=HJPFG$2;M6S}q>sg=-Kt0vVS!KGNVh z^Oo3{h21P5-Z;?i|Nu1GHHjwVVv zPA&P3Jwse}yesT%kQS)GAPpi}2}!GBgiFp&1Cc^l3X557&fn2v!=0s|Nf6B zRae5%fbJZd&DL!x(D~Her8%K0q6HVnzA{3T3IRH>7Irdq+9?M_N(Bnbj= zEw?}6J*ggx&BCloPno(XBqgEU4-Jo*5BZaRW!ccbUc3=NU58pLwE1W#9cA}JdDFP- zuDdR{^wLW=VvGmjfusjExLhxZwmxu+b1Xo-apT68z_@6!U&f|6ckbNA&Ye4*AMytf z9S|by(+L6u;2zKAwHLq?6Q$J75rzFRRb_Hz=x$Moq0>=Y=kZifpS_qY<3{w zu_~|TWtUxsHF$DMe2N8(K!C7lA;=Hb>)E$&UjzDiutaYdmHWk}oRkOa{-AI{gseKO znN(c~M+>rJ)t%aC#NqIR&QR_{MKuk?r*!HOM=BTGy|(g>vSVsR6{}HL&N9T^!h5hA`l&rwh-Y7Ns(ljGLK~{ zElD!*-1b%f`(5vP*T!?sImg9oRbCnM4>J@yPG^4GiVTyKtYD)h9gzCq$aVWL>&y5V z5vnlS2E$awOM?n~Ab{iu&+NI8&pg4&cr6!EmZXQ=CA0)|9=eD5^pY__cL%~F zS6xJAkZ1s6E!z7aQ!sGi`aKYR!JwI8Vc{d_1pQ{DPOXIk#mbz}DK1#BAiMkSyFcF7 z*Z12dAz^~S4kES{1&B&ri4QtI{Ag#O4IW8`LMyH&w1N^JN_*rHND`mPiK>qUJuqw1 z0`Q*vAmMwL8;5E_Bh^MA5L^qWDos^{M3RV5c+x+oo@;4NC_JQ(yTOB~Wm#4`J)!G5 zBu~1r>D&0@AOHC6?|ILAHbM9RE@0JIZdG(lX{Wspr?vA4;RcQnc3|Ac5;leG*)zE`M>|G^H4wSu{n0-s0kK8XztGDQ0t2tROwA7Y+;)Uu= zC>^nOF3D8cY%{H*UZS+i!j>p64gxO*UI+BjNEXe7}K{j|TP+RH5T z3_nY5{>YB?8|TfNhjG|Fw<@d##$=Aw1h$v5>FzrF?6aLYixqsh@t*7U0l22g3qaCC zlNfo#53|AjJxYTL2@xPAMuG%6UgJ%1f-g;fQs&w4nm!~yxh`;ufwF_h-m zxh82$r z0s@2|36S3uog~Sr#qjfn&zYDZHe{?U3L}dZWayl_GOD`3vgA&zoYKw}o2{EB3hUNe zZ*{ef`*JZpOMDu%>){M8EL><=By4^#_5HV#adX1af?mHzs-^HX7FsD)`o(63+IDBx z4{tUHo_e^rUU0?x&D2wtwD`;XRMmu$psGu&x#Zqm36lpg^^1O{{e&baIxH#8k&-{H zCX-}DF8ar{c68c8HZUPKA>yE)$@Im_Bc?WsrGt=*GFB1)SUP2V*qY+|-~awQfBBbx zdA%LKVVmQYl-a#|cavHnxw$O%NwfXbAPA4Q_i<$C!V52SM96Xj_kuX)gCxoLvAvb$ z2*LE?8SH|%6b|mma}p!DA*!$!D(r-$p>x^iXQA$rg~_?}oG*_6Vc-mDkDr(C7BjSw zJK`fr4;UXZ4C#*@2r^oKdxn^!0yc+sSI=NE4rvfXY7nR86R;^>LP7|-Gu6+kz|Axz zw%`19v;L1-_uSon%h%00pZRV4t?Y{qqX7;P!7`%UE?{3q#e1++rG2854 z_fXJX2=Etje`w1_7cQ~zXMZ6WZ{OJ~tlJIfP0ci$m+qOf&3GYX^T4m@gvo=Dq(k1< zZh;Uvib+o>HPHspAT-(@h@gEB!;{-#gw)U)@}8E`l(~zADf3pxe<+?<7$LW1&#bih z`jsnJzW#|Po_Iai?6DtsE_1l{WCtI8IqZFK#C@E@ji2sbtoeg`UZ9b~Eyj4iiZ;S( zQR_1DxG%A01a$hxPP#xy(FrOtcOU^Ik>0C|xrE=YPL=~?!%zp&RIF#|7b zG*ABeyUclidz*7c|fueS^({t|M5}lXTu1BvhMeP)hv9+FPQ14E-_ng{Hi(m z9lvOH-}l4X#Ak5d?x6dFmdqeseakP5`UPo%|NUzpGJWT~!W^;Z<4Js~)DTqzK*GeK z>|KTIv$X0pu z>x~7Wk>Co>lKg<=G{izeZfki+eC#P}Bz9;wa`8ft9;+8@Y~8xGxpCu0w;?Hl1M$HW zIOqtWud{^5&EUcsJ}{iJKWE?5sj2LPwjsTAuxVa8)O6>a$28M7aKOwtw9E7yK4`{@ zuAW{q#SR+dIdCMDqm1zAuh%lj&RMFrfT@dy)mtDx2cCMw%zWh=3V*ScU(fu_Pg#<4 zP0Kw;I4m(CX&I8fY&3hUBxd7Z|Dn6Lm7I8VD_yYO4n&0_^>;| z-*$Wd)c1e52p@HE`l6-9^XISsoV$nfvw!EW%%1!1G&8J}2{^cUgN^?}v&qJb5&|hY z*%GqunR5z%2W*@#|C8&@SP)VVBusv2!V{7Pg^qxn;gpb21IsK?{=zrD@r_^KvuDrG zb5@t}$DujyS<6)Smd(E);o&uM4*O5GnkC1M41kWJr=8VC#)va|8;+>G`jJ&YWsV62d*QZ&wjNG3d_^+auh7 zP;I~CM)RWetNo1@F&m0BC{7;4>TgV?Q&!(0;Yp+?VYIOI<`uJN&;9{N=3J`0JhP#C zNgF+9LdE6IVSGeD(t{%nn=zH$UN)_?#~Czp#uw~+Kzgu61+KaJBR3_5uFgt+vK1>< zb6`r3IepJ&)7>1KP7PnmO-q8NpSHv_y6cJm62^{@bVpvt zrmt#Lb<(?jcvJKl!f$6)Xh?Xr-+rT6sP0b_QYCrsBuHRp%rP&0@2f4(pwc?LclQWC zP@z$pwC{g@v>;g_vPI};kn{}hwS*^-rgotXL?uy7c0Q9Iu{IV`d&$+g6B5RNWet)W z9BD@*E1T|E0FWI|dYr_Edure(?R*GYU1MH944bi^2X=AA6n3b&xabkU^?P8@jMblU zj_Grw#nB)=1SCGs%{tXIv$1!;koX*Yexu2oW49qns0aN8Dn;^AsvwOdL%?%sn}FF{ z)5Pp8Ig@L?eMU7G0DqzIL53lDfr@FfmAnuj6|-OSMrZ$n-xmLj>QWw|4ukY&^VhF+ z*Rx(d5qt2VQSkr=rPzjJD_O}vo&I0sO5aVi-g zNf1l8z%nva!$(_S2OlLQZ9ep|M=K88Z>94Ei|jT25z&EzjCtf0sFA zj|521vX6bXxMqA<)a-Zy$Qn8ol=Bn6^#SwpKOUWpkDSgI&KwjiB8j#9Bx=z5FWzMop<&<+@`{>=L1!j+oFQI z%li5J*@g`pTsW}r!4&od3l_N5Awhni_8QHsu8sRA$I7?6rq6cdXRInOz|ESmbUeu0$d+`qle88QQq zj`W^&L5n|3gU8CP#~YB>9e3Vfwr<{N7XRDMEX}2jn-l4o9MBS*P&+E~Cnb34eI|4Y zSw3=ooDp_Ck$Z)`j|5jplnWW!^9Hx4#}P|b-*eAB4Y|M(R(FK7he2aT7k*s5x!nZ} zCyTU)nAgA4{W^guv*(#<3))lv_4gk%4?g@v@YyRae7WiA87Z~6>tBA-ay@U}9JBna zW#(9iCOzF=Ei-&Qdj$Cb2|*{!#S1}X+SyDLey|M%cM`7x;)C0?cwrn`;v=O!v0bkd zCLU!HA33v&*CApAH+82sCnQuMM0{8g3)>$h4GIz-N`L&Wsx5%JiyM<-tsczi@;kB@ zM?fuJn2kl%g;`wBADBFQ9YNZ&Y}ZE931EGnzB87$GkH_@4CWV{-%}YSjHok^yr27X20f5EmOlm zeqQr`9xLoqKzR7WpxTbXF#(AWwxMw0`%OwiZZM$Nl_s6Y2&uZ{lK)9caP(27=F$X6 z+QLbDI<&bu+r1jl7z~_qO@1ip5wpMCbJ+J-iH?g) z5+JPDIA_ir><{90e;v(UUHcA?-mvlJEr-mVTMwBJp4n^8nK{(h{_CTA-SIcinPGan z+wV+a=nNz0Mh6x+gn=`Ro!gn=HQ$n#-}vyqapzyY{{L7av$MD^!zHhKmAT}iSGn_> z{_!^VBk`dWiIO0Yqiyylr9x|;T5lFxQU!?;&fy2w@FSmZdj19X4ALh6VI8p7BxG>6 z!1|H?bXaw9@P%pu3YFrKy}McrbJidlTv@=`aO9`Wpo!XZpM+bx?t;rnO$@TX%Ba4mwRzwCYL2Q zjs<`Kf#57(zPt%(52QYLj?K~X;oaif!r&O%$(pSP%mynhy6we7?s`-Ipm}W15%*g^ z1}Rb}W_J}h3Ojd0YSiluc^`b_ar58*^jfp_>1WJWtt9DtmJn^XWa=~5e!-HGXPuje ztQ%Z|z73(oJ2HoR|rcX*n& z3-{@vaN!2-5!R>eeUPMy#8^9FsK`Ko@PjG~KkU?uYwX&b2JrfzG$171Nz$I958YN` zQn-*!Pb`dtgt4MKFpDcVg&p7&cAn#S)m2xyRhT)O3lbkQZ19ZDON|NG2SMe9A7*x~ zS+gd`s?BTHu5EzyV53sl_xSxc8u>Wy)AiKeL9=UM(7bm3kj=)xVb}(hOH6$riSQ(h7m8k2tF^T5Rdyg;z|h$2oNMcFl@#)@!V=&0yDWR z;c2X2zrLkrb88CB`m#i*iNy`M;c3?4y}Bg+V%j=+ zYzYnkwb-}*{${h%Uc)Bnmp<^D=K0M-YcqY%N@7#2nhZp}59^e7(z86l>l9FnGpKf^7iA$JWi-`_d$gAIc;MNpf~{q1Cvd z=OKv@?R-Md+rj>QgXZ>IcDm!LpP6eGo^o_PQWGX0y0x?i@MOnHZfN98+aDqIaeipu zgGCHEqYD5Df~^l`vH-C2;WO{fZCe4Uk0m+TbI(0jY+C^W1k!`mo|d-KXdB6H2Z`G3s3xpU0vr!5*GX?ewk=Q`r^Kd=2?PWl7CBr_mIkOTp^cc6mg z(4SOg&7V7DbMp=>Eo#@chdBFpKHoB{>)@7+?ni*|T=17S2lq&mQlCVAjxJ)VExCeo zlK325#MbL6gEp-gEdlz^zp~B~o+;*g-`ZyW-(Ne`BqRXc(uPKniy=mowd~bpygOGJ z#mp{8dT8SV6&IvEe26uCAi2TO5+uy-GTe!sfU3(3+ZYwwU~r>SGmiK#=g%~Eyfk3G zv#H;apTk*KrJwU>cbltU)@P+e?SEB*@J#*Ze{zJUxcBJ2lalyIfc!wU1R?^$bKMuN zH_tu4#Ss+ze*eGxrXxx)e*T^_f_}a82LolO#6IK65d2?e-K?-ADF#T4K5f-uu;)S8 zlr)`fiPK6;pkUhr^7Xt`d%gQ#UEx$^b%CuKF0{mkLU zAnn09?R>mlkMqw33_*DCg-lS|<9BE;++zVl*!g(EW5&8qR~i1%R(4F#S~-8JdGkqA z&3UuB%`H}ib@P^?boO-(MLoSSoP?Lp>@x3J((Bv;*!c9}-G|NPCr_&+LEV!%T_!tFCMzJ%nrnG^pjeJ|OaG>iN-ul2;>7K1^_7{dS^5&aanHOv+5!9t=m_a|Ml6;j{NMiZa{c0x^$K!HD9{%eDkSK zt~bxx@IahafBI#nZ$_8-*Y6nmNXPyE{9l(kl^VztNzcXBf5PMfB|dW99xdT1wycok zhsMn$Jt0_CSI*5!CF$W>K7=JdIi|Bix4U`sX17Nd$d6TXWtK>p@xa!DP4lH^54iJx zc>bVy_vzEkn-)$pzqG8!{LG?h=Dr<=&6_M4!s^eb_jZ|ozr4?&SXM3=(i87HlAEt? zIAHGFI@FN%9?s3LysUgR<*94_F-UrjoPUw~O%x3%52GPG5+FZIDpn5GB6l9y8FTB3go%eFqWazNJmKg=+xo;*S9(hO zSOEg^QApZhmU-5i6Cc{t}nAJ;Wz1$=P?f zX+FJnzj@E%9&^XmL#EfNz5e?vtjO~xd(87=MLyXZM}N!qV#f{?4_^!1OMm~z_$?Ou2Ax-+k{c1g%@+2A4lXvBztybe9n z>Cove`-%_6t-}l5kEuAIVkF_Aii`e|s%!EfM0)hKZZFauk9f!JY!P#`0X5e*zxbm0 zu$A_t*<_Os-MLt`*@L8K1W0&DdUzj3&Bkw6f!SQruo+SxPIvbP%T7Xs>Fijf5WBSd zIE|m#Rm8^`T|Rx|GM#;ymFAq}XLS+V>^Y=4;4;f+beo^-95B26O3=9|8q5Yet*23Y zCFkbsxh6lbr{&(@Nol6RWJin)=)sk0FG-3dLIfp745Xh>I;zqd3L_MD8QBWOTSl}< zRPqWV)fh>Oxz*3*eq378lL*h`L6vGtw-H^x_3>E3JwMvxW|1Y*GdVFr;-gR)GV9LH zPiuE!$er0idZ6Yath9#`AxMXy-pa6!4`Rqohvy+TF=)w2)nHL4oOZ*my4GPbZxewK(InxalubWR zH{NZgd?Rsox~Qat|IAy8tV$Hvby4wX@wmRR%V95Gwgeo z_H~)TW^UH+A2iGC`LCYUXWnI>{n6w5oRKuh5Y%P3UkA-Z#0 zUW5jG*-m(lfmo`zB+ArRTIJIY!O(v7GkG3!tFK8cs?=qndu9G&X^-6tNpU0r3L#b& z2@{LFLiMTZqWpl!BKAQE37vqm#>~|wrGbv4u=SB@F28nz_i+Z}u+bq%9InXq}u&lS3ijFDn-?AhIsxAMb`Av;KQ23Ee!G#4&1)Bpa9 z#eLZP969R(Gr0I=CSmd;G!V-U8$c&6iGU_HuQ<*eK zs=IXFV*W!au#ohoO8u1v$;pMPdZ|@e?bhRjgwY}sq$l^4t}twt4r$-xJDZkgko<4~ zL+M}7?D7&H=a$piy^Rl7Z_X&~fvU^T?8+7_Sddw%&~J~2aWjGSctCibu>J^4XNMFA zo0NX^C&RWm(;At1$EnlI&z;_58f|=7E_-pU>7LSUrrq{kOLl(P@~nB%Y36`cdDTTy zbsZgq25qXhVs;GDZPizB=vgf6I{wQrTH+K7r(L)0`jba_U*{u9dnOOG%1XX-`An;b zdE^f>CZ|jp-OSm91Px7mbdM2SxtZjLD>qZsMMygzEM|!Px;*)D9%1Jr!vp#8s;=Tb zq&~QY6lgqWbphYl*l+&+`H|Zv#E1_8Dbas8XNFUM)xok|8%=j}h*(j@)&1x_?t0+$ zZ?n>%`qSD^B;xbpOFPZn*)yz;fAV#FWn@PV!XXJy*&rQq-)_1>5|j`Q4Gfs=JNBAW zPnlnvIE;p1)bXRsEG9LQNQ7tdqN;s_?0e+7lp-b@pM*}KW#JJ?eoDT^oEvsBBtLqu zE)pK7xcu`BQ`vF8d-rbV&ad3eDeZ2Db{IJ$dMsiHi4Uf=k2h}dwmw*o2h-Enym-KD zIu!0ifAwX3P965_{v*z|=b>GPYZD&{7&T)Wdjo6u96D$vKI1>TYjCh>wptQ_jXrS> z^z=+KC!aJYN-Fm5J7E3nD%_6;rcUj4gYIaEO(;)uX7}0OjB(FX?pErMK$~f{E`9FW z5sq>Y7!rz(l(=L?nDXB6BuhRY)&$Ein)z^gkoQHlOsDxWC?jE6*1F=cRLe<~9k65P zUbAbrRr|V+irg(ixzN_1X`-xUp7!rQXtwRxZ3YGoi}KL@O4d@U%=6Hh&ok3dq`eU= z|8_8c-Yi#7y1LS4rsDzqo;Aq{Jqt+$6RAz;G$iri_7slTC=opjm>W=a;gU1DaLvUH zxrPrQW_7thpzea55B$Q$#}b}mY>?Ykko4dmx2=$AFyn+@J-g4W-*?1ZdFs%5Ja=t@ zbSFG$E|}gh|82$4-d-Cm`MLb$v3-Sr08JF0d}@>F??2RX4=ROh7v}sdXY_6^C!a{&aWqYd1$>O=fjK_P)|S*L-iz~ z^9Yd|eT)$x5b%M40h5r>2^7Rfk{(=V!@IM)OG|d5QXs!WJJ<7R0Z7lxnKNPB?0Pxu z(hgM@S8Rr~2L{fV-NkJy04z`jKxNA)xOnLP@Zlo`>Cs`fUXLVXA@}2jP(4ZL477xi_p};IlN>goI8&Zc;je^!RgE@gUXll@kjXwoGg1k#;^LK$QFl183*QZ*fudY#=>2 z`c<3VJxY7HS!uiI8biPNrqgDbudeSm=g%G@LvyAyidkQy*^QlOVJjX8WIES={ZaI~mv$`nhfpIe=KLA&5wgkw<4dOEziBFGJS?wNW-~0AO)7_7N z`s*{Vo^8g0?!hA+BR&}xaO-YwAgdDPcTW;ge>>@@%9C8&?nFj+Du+^xr~O*ONJw%M znh_b25G5O*gifN7wa(^}mXs})m8^W;&L}IO+mM}WBqqfv`W?D^_=i_GLnQ#K4K}ioTJh{nK zMRnvy-tz{{(oL~bJ4g@q?9vfob{A)L$?5F&+}(qbGq$Yo2F`9RAI$7R*j9SfcCDY$ z-EhCjf_eQrO{dU3v;6@ao%=lB!RP7+I7s(88=B?Kb4U?0)RIzQ)Y1GtY?4 zY?C{rjm%Rhu9-7>%)*oAj>yyLR-Gr^8B;I)?UsQVhw$482Y`BO>5{s$BFmc?2vn>- z;N2}!HZmydWlK&eO|Oi3EM;Bx+?e<~!4GvP5ufpaCM}x0#MbVS$CzDdLc&;3tlXT5 z1q_8~F((-U^t2clVa06Wls(24i)B5@t- z_)oUP0blZ%E>8DANPIUI`5GiWATXm*$skV{l>2?20Hir44I0kVE^`Dsd{=}&n!Js+ zJVO3P8y3jy{F-H2cUXbF6bL13rJBAz^p(6jl=B7Wo#Bko2L|fZe_6*yTlVx{h1oWJ zb7v>Jk)wgoj7F{ElEf$>p_7pIJVQ4G0SZ~$Du{%ul<|V(!Z7Y&&59b>4s7^w+7&Wnomk zo6)qjy6t+hw116U88*j^n|YMyu-8iq7$$TI6zde{#e?CQU2?XUllu6rE2Lrbhz~51 z9|^frJBg2<*6z-^dNb^Nuzdx0Xa}Bo=9!Q8^z?juL|$H2sYJ#>=vz@T~h zB1?Lv)nDhP?Ks9TShPa`s^TqsUKU@Sd+iL|b@LB|1{&V4LY+Y=d(f#1iZ%SGiHG}k6DUG`asgFMvt2o2fXWF!B?zvTQWh+;%EF?WkmoCjA z?SWfJeDH&+i$>4wfDLX*U$;4V`ozv`nRet*=SYv0^z@v*%ydofiMz?NQ9Ua|=!;B- zCp)y*7o{zbx$0JS#(|+CM=8BcDssBmJ4B7I>f8mU zcgdNK{IoM@JO5c+G_B9;NE&2Zp*TYwx;%!?Lw-VW={RB?ZJFOXsPa=az_cR?sS|g* zq%T$n%F~Gx-ptIoRCn3mX+diCEbzNd8@jsuTU*Ksc;5~0DlPJOD1 zES-fz6z%)<=>>^JP&%cgyBkDG0g>+R?hZ)>De08%?xm!=yPIX{?s(_v)%)uCVOhLS`5y_xEgNwVuo*pTvRg4+Afv)7**{l8jyC>Bb!OkZ;V zi-iTd9agoMq6e-Qz@*x=ZserLvrM{^it3;rA0bjGsxdg(i z*3)Hk`(m+O?8TFMCe6uLzwMs0HmcFhLS?IZs@77>>Cyi*a234Lds2Vt!v#=mjh6i1 zpTCKv#Ss>s6(;1RbHPyv6b3s|hx8?SpTG$0Wj*9dV2IHp|KD*2Y`=ajj1H+pY zTg~%O-d4(v`@OVFfGpkh)p-@-YWKEVDgw_`qaS4b@UO`e@XaK5WeWet-j$ zH{N=t^MM+I$!*{(N)1UmksNo~t{`$T3es!|-<9sm&q~@YKgt|a+}A8ZobfXbZ>wB* zEPo^YFb%D)R7WGV*QC?u{Yn|1(86L4Ac<6y42!1{mi_#B!ijLrDY|TS4NyBZ^|H4rt|brY_l$=s z5{8zyrss2x+KCFJY8iaSmPz3u>V>T(Q)PlD0-h|As_jE-hMFs?mOEJ6QyEv

9V$ z<>go@+fq%rHgwrlkmz>FURAqzyOE*we>NIxe;f*-Y@R-6*t#DD-A_e{{(qXfQBQY6 z(Mx?q0|}Vu5qPZUgYeB>wOLbZRlx;C_m!Tb)KGoj*kGIX<@;pQjw4q_v`Er#wXEUK1i$2StR{Yjlur|)m!1d{`jz2M zS%UMa;-NKItz#R2We#oy54vm=$o7oq`xUj4>ftvzZjqv@vooQTx)IV`1%T0m#s_+4;}J=fa7 zVWRw$ir9YfuN?11(dlp?CiH30+oiJOi-|^dB_3hu37cexQWK?rHlLB5sfekTD`dC1 zweY!TNcEA9%dv#eEjGQ$|mogw^dq)wgH@)h)pP!#~)gJz^)&fef zz%l{Gem~tv#0Y{+AYY!JhhbMhgt80wKxj!S!(n&}*xdP!s{V|Te39@Q#(o?^|2bLu zG{hA;Rxe7J6~&ksrNN)%D;~O45&GVK5ozYUHf}*9gu|gXAf)U?|G(>3*Edbx?(3J~njTX^(x1eP z{GKIGYsU*(F44!mQmiB$0L$e4PNx>218$W&h=*!0(8fgc_xZoyima`OK zfOCKC@1Ect`e@E1Tz?kCmF_bo^oQ-`uOlZGHC~yiIAM{^J0mmvXiNeQV^J1Jm@c@W zM$$HfTxJrGit8>Y*&1$25b)S9)*$B<2EvY5DCAwLJSYq5D>bTfWE2TJ19niyRi1=IE#viibTMwo0P$H}#^N zE9q1RGJZ~!Fq~+kWz2RjS_P6_n*iPvrNHXe+se_YRBZd+%|({6sQzqWb~r|Er{ zaW#VT)NFFdOjt;V4S@5cC_YkrJwwkyumlG{AF=9i-U)>X%0!CgjOjp%D6}cWVtAN! z*eX#$(lI3XNngpg3hO=^8euQz#UE6zV3fA4FmqC`I_{wMNPF1+%M8<29k69|7^6S# z0n!k4!9O*1V*nhQ^Ub0;F@aFf(?p*+5Q!dxl0ui|bAb!^S$p+#Rf8;T0^&0i_R$L! zpt#QX2O2{tLrVR)-gpO1b;}w{E=&33Z6-dN1l2D4-U7mu!BZrM$*xb7qNcFu1a(yr zNyE?s@sCdv-qH8qUR_muS0ZCm0y*`Mq#Ug=X^`3#2^dB*7zlqmlh|H1mU>bmMWZ#bHX5{@-@6}Sq{6Z@Y#^RN+~egbeG5&PvT z*BbXn70`Uj+M&&d#14KQxvF0#RKesiHJo+CCDPfccLR`Qc$ra+k)KpTWN)j-MCLq` z-o_?t`-t$rA7^A^KOka<0k5?5gw`r(_Bj zQu!SEbm(3D+!!i*n_=Z7!F=+hbaP(K> z9~*vc^9UPdTwG>OgY#R-F`(QF^g_=A%}%XzMx0~vd6(@Q`SB)K30m-o%lUL+x> zQ&D?WCOA(BK##%@V4FuQWebs-x+ z#`mRfBq5l^2zN|K!jH7?)NFX7gep_3aW5FNK@~MkMw*c$bK*7S_Rp)lRgu%$G?V8jKeDjE69%8DsIpQ^cf^B@$Gk`Wo z*PF4#tAGER(SbhPAo2p0n>DLUGT9Z^fuH64c`0#&pBj(MptySfTLEw{Ye$X*sbvcJ zv3?=@T+05~XDAgHMSL-qYmsuPiZL6(K^$6B zv+cRsNtp*;lB}f~OWLyM{=9m0GK4+U(w{rMqH(lZ9Vzn?SW{hz&lG26-{O6E!@dL0@ol&A$dgGpn@c?exTuAtj>L`L1G6IA z-iKh|lKz#ijOg=BHAu^8IjM;(-2^;o&?^EUN(amfV_C)Sk_MvxVlzb^CZl-_PZ4pD zw#~|*h-sO<5%~BK_$=BD7>CyeX%Ty)yz4pyyv}C$4dtlZTp#A(`vsC}q9j9$dwVs% zqJ=7Gk0_RXpV05mhBZUJvpil*pC`HQ2qhM?y7-lD<)>+Qm7K`FYJS?HoGud_)$o#h zaUnBZ5fC|UVDQi8n9!f2^k1?sQm}T1=Mk@y7OMq7Z^Cvip%_cPP@@TQPrNMrEjf8-}!r(KYT%iyH z@C^}!T^oe+iPVNDDO8q;1ctGa(kpl@Iu;Fft8a)u(3$sYiSY>G-=_0B*N_UjHKmxd z{i9v%=JF!;GX@qtv-t|$3XHw+tgLS4=S_=zbq$|M8F%Ukms)4&SYl1jWGG^{ePVuh ztLJ6p*CcPO4q%zE{D28L+Qc7sjAFwUiN*lKK6$*5HrG+v5RR2;*8Wwn;ZT4j-kU%i zew#NiZLU?>U>ABpnXP^YTX z;z4(s%K$zbk(DFy6mTBseos)`&t}zvO|Jm|Jm-o3PmE`k>&Iolio&vuW2)Esy$89Y zi2SD2W1UG`M0!c{1w(#L>dw$FmM2$?7Zv2av>DVXC!h9FJCXDd^mM74gApSdiume5 z=P@P_;AXTQ*wTMhvqC_-LaB4u=%LOr%E$3V(_d3TCPLM*r49y(au0Gc&~ja!xhArd z&>VjB!9VN135WE0b-G7!NDIff{x<#J3;G&IBY(E$)Fj>_PTI}iC9Nw(W<#WtYBrJ# z-=Y8o8!o$ZSsXex&SE6Dw~NeYIAtv$hzf4YDER(@aCB4X9Y4yAkO6hs+??MmV2RVX|m2GH+1?K{)$BOc@)P`fYQ} zKo{l#&mhq--mT`V!%hp5*xH?+r%siD_?S>9<`9bM=o6(-@Zq7u7!M$E^Qp=FP~J|0 z`CJ=C@dOwh3uSTsITaG`Xx<2-qFY|uR^#x z^!96x`q!IfQ5o=bn3wDinYc)S9(SjWe}M(Q;~%GOUSFPN)RSlwg1+GM)HLi2>=Nhw zZ>9Ytf<`JijBi&kcV>u40b?(4O~hBxiFpWs+oTAHeogRQwT9g&*c36ubjO`uA`RVcjtiG~g9y~Z@ednB|2ZJ< z>Y1arMy!RdP&j5+tUztL>^`{J21mHGI$7=^smJt9(Jx@y2m9 zDrkIHz2HSFMAXga$tx)cL*OfG23vgJ{w?b;{%+4$gfgf0n(ks1vj$}wh2>-%*w3}b zIYIwG@69)CiULGCrnhzmVmsVXT22xO68kkkyr7S5;Zn%m8L~a-w-H6GGU1*q7RKEo zd^H92(}&OPbx2}z1xjuyHMxJ{*b@;^@D^Im+ZJvns>o4-l!<5$iHC@T_`A0g zv)#Kn3{5P>&~x8KCfqVZ(~~g1(H}ZQ?`S zctLuu%l10LcxZ(RROl7YxoaO*%c4hEx5OSll)dXfKPHD38Ka`_^zR{>fRNe_XTB@; zY5*qhaNN_*hCn;POr2TTpW`oxVI%U(?#n)|nTmiiae5?k)*6SFl}O74Z2)B>dTI~! zewp1Nj%?v^Vh$w;3+c_o!iHHG($0gW- zb(Zr;X9MqLa^RanYvL|#^gi~T>d~}Tc=ua60h@GV1M~i|D-PYKEoqRMU?fXuh{FQm z#UU8+&C|I&8T6=%FzWQk#Oqh3dmhM-zln-ePhz46F5)c49pi2vb+NgN6{J`YK^+5TDE?U&w`azLTj{9x<)qRKp z8X79NCP(s5es>ib>dA_G{qklXX$E9&9KeQu(%CU)?B#md z_Ah-mR}KE5$`U!{FZF>LSmp$S!51Nc1h+C?1Slujeik`lbw9mQhTR9DGGOY)vmm$q zpVi*(1Qc&shTiRpZ%GZJRXWB82?RK7D$ZFl_ew(YuOg4hcjNz8TqON#xlt@8tauBu z6y6v9fL@G(~baM!oDylJ*&D>|T-TM`2NDa50dwsGn& zc1>8i{vlR#<5+P}wW;kVLxC1BXex_k;e7r7@V60=zfznzY0004*H9+!odUt z1s=4cp*OnV1P09*Sc$JcDVUlAK*q$_T#YsK`(aLZS~X^yl&c?xHx- z_=X6vGI0~uH!K}KX_5;@wZs&a*SoQj&PoZ4cqT=m+@(=y?nEW0AJ#7)(W4?l*6m;h zRH(#|Quwncv!GxKx!ajT5d6&HJo>pP3Zu^Ua5@7O<*em5LZ#p02`IBP9$07*NCKLE z?{kIN_t{ANesX|SQvNZ6*!^rDoCw zu5zXRotmf|Ch?;)yH>$l5emz=oj@20dmiLl2~AsFMUg&at)`7#v;h}M= z+=~D7##QQpW-&HS_rdfz=76do?3BCyb+XZ)6?sGhUuj$kdhRFg)=IQu@S8`*Exh=9 zAr=Q2$+PlD3wxzNL*NN*&5fugm22rT$#e>gMa%BWgw;dvJ8g>|e| zq`EB~|d7cry_JOFmLXq=+>;@DqLC8FdFd{*za8{V%pqA#l%?3xjX1I?Ka zh=YN7;XNQsgc24vrY0cZ)OAixUJhNrxiHUyoF+^J>KP>Wq}#vCDeue+32OF@Ha)#qhQ#+2QY8i9j?*(w znkF&2#&i8O(aV#`ih&itS2PL+SQXQUqqJ|JTgJV+9`;D4T}ri3#iU^%k*TLFN-}vD z#5IdtvtkJf4oqpl58$#%WQbQ1syW&bjP2`AQQoxV4pS4%_9^O`RMsoVg2kKA-C9v& zgKb+=ik`Z&DY|F)bm{!3Gi87bS55CGhjj@uasDB-+ z^Otj;d?{X%(6h|LQpTbOZrmY3S8yu&d}4YK_8?c+r<|&`9)w0kq4S={BZ70nX>k%E zJ_A*|pn~cB5`MJdhKBlFa2}#QOz)lOt-B7)<}`D1j0WuSjB_qXsI`y@&2wwS-@S4nL<_~F0E(i~BOcZ-Tqp; zOWA>R8LmgB-}fZfKhM%JwQKNI#v6yQvTk>SlfM=eLb2116R|8M-aXpZpi(Ek|h}MiyTzcGq z+?3>x95jkcxgWz$@eRy%#BmLZUZLsKVJS+oKwS63%_)rq3r2eG6Hq+b&4-FrSmZ*@ zJvw8WEjXzo=g9G@Y&B*R{{(cx5v3xCA~G#k8FoLGbwmMLK3sgw3YV!S!kEm7Z;nCB zYS_tZEQK}0#l2iM4dZFiA`cdb?}71({aE(Fk2(-%3&k`1eW3R;+{!x-0#3x_Ilq2V zcb}%l#@lWI#B3~%acLs!unNJD^EbC1;&elF365PRQYC4-2KPIX);QmkPct3oG(=k4 z$HTdo9qN$m8dTj!IndBZ#i~uEWR{FWJNzQ#EN0a)8FwxBx|s%q`*VW-)z9mpyW*oN zR5Ly<)muBuc(EaqiXKgdFf;ay9Vx6Li}jj<*$l_J%i0GI+Kly$HWqUAKBLo zNVu@yX8iLA@oxQ#yZdK|&#>mh>}c!&B07r%a`2-HvT2|crE|Slp<7Z6o0`io`t_`G z3|Te3G&cIPm`XAxJJMv_w{z|?P5Ho(F0*@d3K$z2<58C&*2>L3G9v5t#zD)*z z71j}pv1G8qu?KLdMbgK{stkKj5Rh^uo0gj#sUnwF%5a6kcEUav100WVDfC^UkN^K1 zcoMXzDN9L=(MPWdF%z-_np~7-8IFj`a3-M%4nXK<-=C~5Jk-NS13Z9eu~5@BzHhbW z(2c39F;WHOu%Ukndvpe9&(QQCtoQh+*hvwEb)Zt-)Af$m{~C|vy#G7NA2Q{s+{(+i zQ8UQ~5kREC<^bHn3^Bv)KaC2S_V6>B$+4!P;mRz>-a_n7IAn)drTi_A6v8oA4#en1 zkg{K;{JsxYA9Uo+*nF#hYU;4CtIZ-KAaw;Hm!hDcq~`(hssFo{gQ8-w&?~Gf)14v= zjagAjN+fJ^7UD_f0;AA^0O(qKSbas2uO9NAik@FVQVTc%Uu^^FJTP^*a^oj*arzKC z^rm?^0pMtsX*Z6QuVGk%pSGKP?l1O$KPIuxA!RWH2RNd9gHa$>1Ybi z)@hA_JFhNb3CdWkwz>p{wnQ2&AZk!B==118@;m69En-@n(Y5N)&-YioRh#dsW{HEdW#Yu}lOWs7rK%eTCLbu$Bgt969$1S?~52w?z z^@}mxd~C1g6HqmJu=AGmptz>ch}uy~t|X(X&fg%iCSwX_u@SooZ;mZ@&YZx^b|WH2 zE$P};uC#t6#l*xoe2`R3j-8(0wL3J8AfTm=1m$R}{pI*^ zFLR!Z+0k%g%6TDd20iM$PCND;9XGhJB!u9URfE9!W52g|(GPksze~{EipvS|~{Le~TU+!d2ZB zK{eX(GO|Nu-$X6odHidtmc-#QSIPtJA~kbpb}nkS7)P8!9K=$yEC}R$b`TJ}J%j-B zBBI0j%6=x6!h@|NmM5_vz_5*4EGQ^QW0Ak(a!jzl)#)0r?AgOl8-fdgz#iF!hCqUVKJ*i z!FnP+2$lf@+xVjm)Ty9rAF%+FWD4&82-_wg!8+%23BXg=ZFY3jZrB}q)c$zl4D7(z zGFBtsZcBLuc(TsE62bhB*lS1S{8K@KSy8}Ag9ovg(c&hivbcsB4TUCRte69K`sKY> z`tpG8Zn$ciVO1p6u{WD3DFZW!xJOe|G>!NoAl677{wh~3ihpz1Rp*LL2H5;?GJ$6A zo|3Dke=kEiU+5fhNrXZyS=n+Bc|nCd6Vo^65Vk7M+QpZI%H;pfii@0EzYcH`vZn`z zP(#wY$bO{vMOovJ#Zbt3KY590tnzm}9F#^_9)pF>0<>#EB`kNHWWcOxBONCbz#@pP z?%Mc$ZV8~Y@e=d8AzhkH6pcyCuhq9x$fzGl=Wqs0(Kr;}mYK7=yhQ0|2?Xypz4}dK8Jtev9;m0Ce_kWu3cEdF4+_Id>3n&nB`4uRqhlfZ zUa++OZhCgw9OCW0>IxW+v&~Vhop6;+Lr&bicK2FNdd-%an|JE*RUpsHtm_6Z9s2E4 z)Fh(&Smg^CF-WmjcitK^6BB?zg@o}u@YnF-8%J9FBOilYXJB>MqZVhgLb(t09S6FH zRSJqI^k7-=iiT`-;4k#P5kQ~Fv)o`i46rR1DDExsHF66{r!Ehs)l$&$^rrAODXWw= zh2)K@?7%i2V!;Onx0t+qLC|v!H#;&iHc(@mNz~D@@t~qNLv!#p{%H3af&Z=2W3s?< zi34ymm|Egp(70yy}U*K*;-*5V9sKc)ic9>z;&|WvK^IHCD?f1N{ zJG-^_@5PU5SJC0}=z)0;@zk@lh=ead%?>GoFT%e^1K2;zH$Vd5+%%*tt48Lnm9C> z!-J!pudMcG6=VHKkJoy=fm8$}+Ka8H;jvvBT2$gD{#c;tt3aIB6`0rVy2-t0xp5^l72ErJD^f|*Ed>Ce;o5*V3wrOPlv0m zP5|icpZ^sRmaqgox)|ECoDBEUA9=6<){3h0&d>s2^3Vf9LcLw82B6x|L$DuYid1$S z!?bBdb|&HJi{RxK=c$h8Mzb;s7`+N18d|Q{+&+hIL=w%cdRNigxSgJY!X(>o;>C(| zwmyF$UOUx=tNh_u0f!N+L^b!GZqH`U|I=-h$D0p~rkp@38|h?yqY#mfy@&p#=sD#o z3zy2{kOM&_Ys_X&+!8w=B%rW~x!4E;IH~5{A;CpF=gpUkr>XVJe#)?ua`U!zrnE;V z&b5Cfqps8fbpAV3W<$k=sgoaKGlLZncFg-~IR7j$8mfFun|)ui{_}Vd*(LQZ9$uN0 zKy$-qJ)>&5oxu&j2B zVMd$HE-?=zjZFr3z&Uo*RZU8qbuE}xe zO4Z7ppc+*ZapYyo-(~zLt~-~6uRvWvgt}=wn670MB7k3vNQ6Q#d9=4qY!=9^xLqzl z04gfOms3b(62wxy*Y#{8AVtL2YWNym+dUajRI|H&liro;-k*ePAbe&OvInG}zhUV3 zDVEOAl;B`DAPhJMa74?YIwl+czq* zbXq1?&T}cbmx92erUyoS`AKSWzHG?tCtM!CC6}dncLC|vo3I93*QqSvgHJ)*`E&6` z>Ee5XwYNl<1$ODzl*)j+B-zh)BWJYvPG8Ao{0z1cUGWFOV^TsaEJYB9ypo7(yMcv1^Ju|I%k zbd&wp)x?@K4p~>_*3nb!*&XayBMXF1Akb|Fr>#nXWup#{5FCPzHnN}djkCgV!DmXA zE=7QVKuAN@YwtR9YL1DecnblGjEJqp{g`w5=(*+8cSPrD>GF%NL3S6z7U#x1w-pGn zWqCPD(r7*ioYIN-Om%Y0Br=L&vup_Q@*Wx4HBUQ7*vj&}S!BtCEVQ^~!0JCWlZv{s zEv(z$h%}Flc<#R8)FN4GU%j&T@>4Oi(RXf4x%~B>>LTQ6i}W{*u5oXyuQ%?wtOfBy zZZk%9y>H#SqBl3W_eF~+G++EQFsCP~M`HDh`L~yZ*=%kSIH!9+ePaRfHyhGa=1fKd z_5RpgxBFd2p@o@VR)}BL#c(&HC0)g&8kOe730DEw&+=(K zg=3NBRB60vZh_1nKS#g|Dj6&F{_QyI3>$rt*MpuYW{8eoX-f#WzT9ECJs$lPaa|Ld zUiS9NU#0?KB||%|Nt4DnEAJDcusLFWuT3k~ezO1g;;UFr%-Km`OYk^X?%X47Z19M6 zoB0`c5B?F$UE8U7-+bM!?}zx~CW0%znHLT9Ob9I{Ww$ARi#)5|$LE>ftZ%b|8RBsO zHGyN(FCBIO~Mpc_dc%+{K6n>K4#x!Nl2&r{|X< zXykD0f8trYPuh)0ErSNv6=yor}%R9lgRWglk@!r#qpq1|KkclR+bCVUtj{W!ER6a#`O#nlVJ zUGqwd*qf>qqx(PSR7^e0!>PMyXz#M3@hUPgNqBUp|9kS})yQbv%|FX80mqW;CvSEf zA`)hpBcsr|V?Sw1(vHbc6d<#0I!v>J(5H#|iwyj3>ErK_2O_x*8XPty=&XJSPehRl zjsW9k9ozMKYxM^6ak9o;WNMKK=*`WI0Qd5L-@&?@^}FBe9=WsU?UV5v_lSq1&Mm{3 zQRBj}dMY<9*6#F0&XM6HR(A6Y&(xI3`E6A2nBjWRp_}8`qWNen4hING8_qgJFxfcE zoGgWy@KL~Ds_W^jAf7K?Pjv51M6jkJ9?qwgH?HXk>EDXJuE>9IOFnkTEVgi4^X^!5 zdFCB?aaF{$4X^}ln2Wsfp7_lB-U>GQnloAdyw!a4*}JG6c}x8I!0I!&lh||I;3y{{ zW)RcA@!Lg00+(pO{dtG-GCkot7fWFUS}z=P~Jx<3nUW?7z;Vt$)Zm`AR z{yN3s6i#k5+xspvy@ESVR9w-S(L8qkbR3=pv5DpPap6+9UosJj6N!HcsK_Lf_hM-X zK4GDI8?>|MdYs$Wol>-#>|6e7RwV)XW;{!>+FN zL}Gh}%)vY8G61XGA{TO)U~1o6Wf{9?WOY0ZHf`%J@Q1O6O((~&=@n7+PV$IRs{ThD zdP^hX%8-%tW;Y&7b2fgt?A;E?`hJ7#=D3UB-R8CyqO+khfDL7H=&LxYJMUCDO@!mG zsk2??ac(s@%n9|hI;hTNNfxlzD4AO2q~7e>@Erg{h$ucGg(Ky0s-y%ih{fS5uLpSi zI|jnl=k3lC8LAqlh5Ri>34vBErSkEVP%dWL-w^xe5h2@Yge}Bk%y2{+9gg+y`0hc zWfHLVy+YzT;7WlHUqHPDk9f zhYNg!YQIGE0ZlhEk3x|TR2(1}%&AtaqNxQ(lvulh@(6zS0`V%0=6{(qVzR0U#>Mq= z?3#FvaiNp-!Rf;l-=gAWGDRL~%B#(S?au0q2}k=~o4G4YNe%*gZwit0Av+pTt)*^4 zy_EH=(%vTi%4*Zzx(cf;#I5S09ez9-q`*#z5Ga%WmR}(g$xB=xC&ed+joZ=pDC5aa z`k`*TUq&HIQz&joO8I5aDw$^YsW}DUObKGR*nY(Y4y=rTSm$n8F-3H{)$Xny16k3OZFhbQa6vyD1v5q zq?D#R@t`)9T0c7Y8ChHyS{E7BlKs-KipuhIRRnhnr$1y-fj!0+-8jcZhWk5m!u<{U zqbTcjWS<6bSNaQv8iTlU8MAd&2wD44qtll%^F+`xK3CBp4xW|r2zrynWG>saj0z9? zFe4QQYN2yBh z<#M})jE9(~eFGJHtnqZ=I2??}qtgLHuJzaN;B@g@d!W~z!Or_%pNV7SRJiw;N)2Cg z&H*Qo{dCe4z4A8F)a;ip7)j-;())9HawuC(FA)3+nE|3E`**^klNT#>Hc>PP?zaXy z-y4pKyb3vcoW=;oykq7zNLI0)gd^IF{$jM(6;KY-E|re)vr&iPxEDX=i0cLkAGWCYg) zJR&EeR_mBPdGy`E)C=J^$Ar9_z12UaO(JZ55rvrmJ4<}kU@0P&{wLXsKapv;eQ)Nn zEk+}YS}@RJfu7nw!g29{jgUCN2JV1Z`)?=S%|`84Zo*4bE|9TZ!a~5ST^wC|9t&Fr zSl_!tqZD}h?XS^8=}pKO{x47Z+^Tnvr_TdVrCDt=9ZRVYkWvlshhOt`4E&MbVmff=JZC?AQ-)s{4X~shuBV8P8FYr9{Ii#5Nn|m zNNja~{hED&u-U0tv4Y{zZDzjStFI4aM0KpV+gFk+>MoEC%qBYtDwk=6uBIP;(a*q39yeG5)s_k$wt zUGTO;n!*l=U(d5fg>QxR?mFA9iMtq&f~Yt7yA$TAX?WLa3eGB58}sa6m2|?1p8oc>$3vvh4No~&?NtMjF|(>lSY$k#0xbz|FpoG6t$SJwUI+_x|J{7R z{DQU}QjsZA+IRH7ClFn>MADLoL|514TsS!|xcU}Z#zTyn%+sii|FqR~@2&i2R2|m3 zsg!*AFzV+=@Ksubo_ED1ewVFi>$-5qIr)4O8@cb6X6?b(N~aU=$CuZeHKg{q()LSX zbt~EOeEj0s$Mz+=(*+b8o2suGD8zjV4s@+myI;u+H%7f$SAqY^}A-t|~VsCeBRxLcLG!)w8rqO1x=SOXn_LC7N%(yH=K{ ziayVO=Ea-K4MEo)-HbBcqhvJVYo{-!O@Dr~(AcNFmShQ>@MK&@_mg3(D+SUyGd|n1=pQZ{XcUn%=kJr>;wMc1caU3TSzK zVfsHaBcHymqZ_G;tT?wxHHNR#%C3&|xpd{96=mYwDPap%Z~m(5)hznlr-`y@+qKT2 z{ZSGcc=&Ld8h6bV`1YhJT~*wc_UHUURPr4HN2M=|ybzI#$Q>EVTwa}6Z?T>MJo5bO&{ zPm9>5sg>^wZ-JoNeZFcUW*Twc)$+l{x~8dV!KR2lZtMT@PQx2pJ70tI)rM0IRm9eU zwKAnNCtkk$hg51`g-Wo~vKqB*rJ9-gcg`HMcFUR%NbHo@!$HVWP2G@|zs;U*lR{W{ zWKpJ~+sqddg)NO|(yYtaoEiaDsGApC3)GamNJ+vSg_Qsdl$7-}x=g?tH_jrQSk5_+ zzL#3$>q}t7iuP1`ylI)(Im__}i{uEnA8h&0JA;4hys;N-x=xPTRi#wQ3C|@3S|Wi_ zMO0kpC_>8jUDV%)V_ub-0z+%M#(@E52x;=?+W0;eGq5Aa2draW#QrEEDMHq!CM~sc z_Fo)oHB2I!og?S*H$O4#$iUl}eq^jYRNTDWv|2&&IGv+EP=SQXpEfLLPIyU4E~cRM z_75f7LHp+?EK^IX>@z3TwXIF{7OI_YKhk;+IJJ1j+p5a8Pkhd^(zy##g&IyXYd(SL zkG9_nPc&H+7u(x%{^B6R79M3(Ba|wBC(AC9&%|%8+!GmMt>g-+riU!UCX$H#t&O~7c!$)ePs2W9$OR>edahn^7q3W^4OH|*Q(Fo z`s7-bo%z6xbf#`!XLCYk<-TWG-LfspmamXNS`hTD-=#-#f3#5|0svla@2{X+9xFN` zeRM>)c~Q}*=Rb zo?j+Ri6l5bZp{s-V^gwU{Bl3$J4n4tVw%Tb$u|n-_L3OF^vbltBR92Dn#*!IpOGjN zj$5ReBdgk?V|DfU;U(gfj_!cs=nS6|cEqxxvvnZxv~gf$MEq55c=P}V8^^>h0dWgD zXWp(98vuRocTe@NHcnvlY8sc*@|N(3tuI?NNL8JI+{fl^r^5_2Q>J6Nzcx+l0+L91NbAS>ow2bCOXS>uZTsu^CmV%4IF*s+A`-$7E{==!ZN8%C`Tg zuz#;Q@6N|{lbDY4mfDS@@47o5JFGT8q~47#&0#Y^C<$Z zD!rt>mB^*9D%j%Z-P*+v!OhUsh2-cF#>P z{V@`6t>TIfhnb?{cor8o_o_`F95aIqJ?s0^@4p^2Yjr_)?I*^2Dyw%d>=l;6%TG(8 z>m0vwK2FDaq${*yN(uQ2rw`YBsWlt@r5szEJmBJHDGt9|P!-*%j$iL%q1kz#Afrk5 z+|~%qYd-m^-sB~rtZxBnX#1ElmPzaN-^#Ynt|umoN&Z$Cm5laB`f>d2>LN`a-8rmh ziE21*OoS`~Y*5~Oa&()>WTWvb@H^con)O?v%Khwb2iEf|2JfW46>!R?=w@%o%3CUy zoIBITyy%YK@!-{rrRV%`3=|XZs=7@i8NbQX)1_4LEReXAY-iP&YE#Y|6jf9v*xMm9F^3!_!J4P8#U$V zin86G!I|1uOL1zrW%89QmLv>(lNSp-?oW;OZNs+Pd{NNj_ouV(L6d=1*{t*3Xj@^I zmcG?yv&!xw{m!JYYpLFSXWmLj)uLvWtzX`G9Indt)pyV22g&@ZS+&86Sm@Cooqip{+xPPSI*oQio(MG|lH(C&79ZYzyjHp4|S90db{r=w~RHuwIYr9 z)dW$_ips5=F0S=7W1_w-@Jr1Q49}O9Hi9obGP1Y=oeP*mntoyve{dPA&H1sW>0$|f zbft_+B|^s|ba*T$=)?)a?utV+tgPS*&n>O=3h~^ffhG?h#edT22}XD`489d##$b?f z_Ww%Q){8BZJ>huBNgX9rtFE3pg@=wkjI$7VR&7Lw-%tvtKQQs0{R(z8CP7`n@^6c3 z3&a@xUW}!Quxxl()A#dqX{pY@#>3`!efEs|0~Ncz8t1hw^|VX8Hm!1@UpXTmr*yQ> zeKq07;^ypLu`^vdN2lRG;Yi36)%o4XzaA~=1!HO#;$C2wnz^!r)8a#A9BKoS&(*SW z*zdv4*>$cqmF#(ZXa<=RndywSC#y*+dxT1D#)Rq#gCuE9-Mq#*u^ zTIKBHFOF$h20a3>0~TjT+`)DDSqj5fxxx<`f79#fOnn8uDc^~G^y`_WfgMo-!bI5o zd!CQ^9)=!1ii(QAd+s#mlbjoW_JTHzeFOvz&JEXASACu9gQEkG!+#B(6B1|F2&zNJw7L)eTqiS zEOO#pR!rx6tLam;WP?1ywk2JG4{7dt#Gd><2nS^f@AbSc;cYQho{zpFWebZwVp~Pj zEueF+DZ)~cfA*O+=*5Jewnan6dJp^;IApVN&XY5p5R&QxvXw2TQsX}lUnm*Pb{-fr zi_p3_hEN1hQ)nW^G(pV9Gc)|JCZ(OHYJS)@LCVq?6(ghHv0iEFx>!bYK>E^#MDWs) z&Bus%>E+%cdZSOf7^|f{he0|&%5jGmmYbtxI*6XI82P-*SI(<1i%ERPy|oqBY~n>3 zECEm5Ia;GYBp8AYn;ENdv52NBK5Ey zRwAdo5{+#i$~U ztG+E30PI&Ll6yQ>@ln;-AS)$thL!V|GJVY_c`fn*7%KfegoO<*ZgdwwJm{^3ySO zaq`#FC?rw&3joKs!gHaodxc9%G`|b03 z`)vEwwevcT?|GbQJyOD}I7|(Tt5Mg;a$#&GUqR$}i38uGXJogFg2I?1kmcXV``$Lf zM9CoCAcDd5?ploN0NN#2M}X7c5=czpLoSYwOQWh-G$ z?RzjKF6j*PuJf-Zt+2IeXf`)rqtW{}9!JVzf?bQcQ}(oli{ZR`-)MFDmJ)k)xSBR* zTShm>oJMzNH1fUg-JZ+{WjIoNy~_Wyg!|Lk&Wic@CNtU(rtN679uUySCu%-*6k~rr z*$-XjB--eGS1DB*{DIS$ilQv!cpj4d%|Zrc7!GqZv`Lv2blVy++_SsC#iChdn}FzR%2z(FbQbaUYEbVS6t2uu}ZYIgZA$hd_@-Y~F?$|IlDiE@s! z>dCs8xH#v|-nm~w%l8Aihn;|G^_Ovdf z^9={Ogleabi`DTTliAwGkr;8MxR(@fu~u3Qabd za1~BX5FkZ4p%@2&oDP}XPD(8!l$>NN4tl$lC8EF&|Ay>xwBf;rf2k$Ava$0Wp#gzo z_I?gqIn&kLjKG0Al=3e9eHPW3sKOxs<9`}G(UHLzLSkf>6*BNs2L{J&FbJXcin&PGc|IAj;im*> zwOHpN5mB$+MV%@kEUHpinbLZ7x`DHOKe`{6X?%Ksh!Qm%Fb$uK}5KbR4*H zIK5*EPk=w2Yir)H^gZyn0pWAd=1opSLY0jov0)^36Wx0&>4;}ahL&t^P&O|ZZEf}v z8#6$Mott4XWBzcH+ER;%pSC4%rux#uf1KIbf5uKw3MXnCwr5u7uP<>bh5k;b-*0WT zP9MLYlxp%Wm2zJVDhk2-G^|*cfATcH2@or-J5lsFJftQ}s>W*^mq4!Co7Ui(%J}(f znBZKdq(C#AJonT|Kf~K8<89NY)uts)jQcpT@B-d%ZJm2`Bh}Qp|LrIO8gx*#(C#`2 zstUGtufylEJWtIvaH4NWxWXu(G#<~*rTxkOGWLc}%~6xu{2_S+%--=Tjt=ow1sc4Z zS@C#Ax~;(tVj8&7$01$Aac^bSM{%NF-WV5BP+`TUa2T#&8O~pFPUrGSfnvn9p?zYk z`fq`|I?|{_!A7@dI3LwlIkt5^Vspc5%)qbB_^2b1)0!&Q5dKfbin$b2IC#$mMlv#) z$61D)W{)H3Ei%yZSub5KB@xfRCe}DK)r@O!y0;_zy6m`dbn7?2U*TiZL12 zX5b=e77YsxK)a~m3WFLp#2|Z|1}_!(+gEN@G9Ohw$>Bz+Bm3`UR=(`$Z#~v54kSCx z8cns?SWm;b7s4K3-34?T)7*Ny{$t!j_Ps(^qSSYTt<8 z&`KIgP`eXKp6xgW#^3!BHcAzK^62tWr0P(a4yjtpJ!+Yzwde=@FFE|&yw42*4a_6j z0-aC3D~;S^K=s^JqI?be(PdLED?uzt`S9T^)pVTjW2YbQb)>GRa-X@#d+(*~v}4W0 zuyesn*;4WkkdZs_#*(U!e3WYlk~} zE>{8@8lBuApj^CP$MRRgSa6p%q>yLVW_O$GYTXj}1akdrr+&^R7nC^ucM`sh-gsbO zjwWWr1}-s293sFb<1oc7ioVEX0! zHD>j+^y&&>JPJWi;}>9LgGW(gq!Cw(08-TskW0!>*hr@+GW2P0ne~(@hYPd zdUup&lc>y;U|riIbh=|K%b-Wthl>j*gF;CKWs$G=^J~6zL=qqP-^aof2%X$|eXA>I zx1JiR=mz!ml&Yo~M=RiN(F0^gG#F(ngitF#HVcHZvuy-`WNZSjXhoE6ImBr{H}t1^ z>2Lx(Gc@WZbpemmZO$I7Zh%aplKW!mU%EdcO{KBsJHxJ_I~c(K3V1Pa;5%|~-kF=B9dwCZPJ#|p!Q>;D}Ke)kq!`dRMaGmA}we`#Ahu>)Aa`9IRWuK#{!p9A6YA2fNs04G*eL83a zHvY=W*pbQM!myGrv5Va8lYsb$`HxE^PaQQI+R!a!-uGVj1;KAdztI6z?qnwap<1GnhuF+dq3jwHh$Z`xLT4D* z+sBnpDw*2c{9{PM_NI`KnMnvfQ~FFSBF4b*fvK7(=n!&UN^@6EF5(q1;RXr0E`I1Et1ZXf}+g(Q-qVmIPWM(Sxg1)0sWBuU2d_=cW zDJ>C0^*vV0p+*W9zh7*kS(ITk=yQ=$qKoN04XC2L$|f7 z4o_W9om;4Q8s*s^;UD;TQ`9OJ_Qiy6VNKvOFG3&D1C;vDg3i@IoK4MCFEIk;^!WD= znI9m0%UEBhak2^3h)XV}{q3|ZjOo3Tl6d`FpSs{JXQ3q+bW`?sk4LVVvq~yu7M)<% zMJgc^bx^IqBb-lqTM0xt(%9TOGQxbIyO=#sE%v*)Y4Y{S+VJk>HCA;`k@ow71Y_03 zTIiz&-A7Suz2vh=cMP>=E4)IHG`Sa{O_#|GO^r{AHaCwDDvg_@H@HNtSFfemi7^oo zJ9k2G>VelNjycAqtPS3%*KdfvVEcqu5#iKEc2TTE)ndv9(J^BO0gBn@I0LPWrxj&T zBAx}yWs}{G^jVEW}Gk6i!KllBwMy{hlGC8@Cf65!=>izAKOW;GiUnkc-eQ=xuR*4QJ-T3 zmaiN;x+Q;e4kyeQ&YmGK&bTjz?s0~JggKT6yT|iQPI?#r1!&pJHV5I)Kqe-cJ9~TL zf`Wn)hG~0JMI|Le*#pf%I45aoX%E7r%;OKIr>6!_Y=}=w^`*Xx*>~7gUy7HRI?=bH zOl=CGJ%8vlMu#^v-UbC_d)_v^${2y4rKCKwYkF;O!DZwCp}|h%w2=~+UVmgJ{F(x` zWbL!$d?&0pYa)V$Fl}RTdwRnJ$jp>B&6ry8QzSc$oG)0h3E@R+}XV* z&3T^rBFQdWzv#u${le>^G+Ny-9Qj=JnZ+pvF<1zVWqqV{`L2h14~7<0Zfx)%exW;{ zI=R3gY_=D}pfjNAi1YIGU4_Hpm(;(kLl2*5g2t_Hl~Pnl^pXh)5J9Nr+}Cx5bX=pV z(U%K%T&_xU@dZy920~musQbs?DW;9zTIevl*8atl{7mo{fAHHzIkO3)vzh7XdHLa8 zP2Mu@kJD(;dq}yH!55y(nUx~LCq)$%?*ah?^XdFyR+fcvm!P%lkYKfBm}BP8xAM>VB{jxa;AWG#{AF7lU&i(Z#`>hD zkGh5x<0x;#fb9bh)J``mDulS}Is3WGkQ|6e982!PhUM4a$Y1WjsnpO?l(FQ8P>f-$ zPI`r0^MC)X?3ih<#~J}G`FcOfxcL|=3t5i|S@@oCeZmRtKgl162F#Yg-?~97Hwb&{ zW%BkQWi6e>u9lX6*B-Mor_|owymoO3uIUcsT*q~Oc{*HgnpzZm^}$kE22(R64VNRP zwj-4Wo0P&fN9+|9da>H=hbOd!P(@i7Y1A=WvYdeR;|83?e%|{MI(h$Ih^x^ToO{W! zPV2WlA=I^pJ{ue~ee^y_Obag)m(R07bMIS0(piC(^^%f}H{q5^0Ia%J_&eJev+?ML z^*#ppA1cuN3NbwXm@AZSh3VSg=+ibM%0lw)Q#IhIR`1MCA`=QfknHSvuplDuizvUR zwb`Ynw0KCqD(3LSVkVsBoj85Dk_NfW?mK<`RA1)}u36RH^TFVu-EIWU*)9>_kg2I` zaF4xFYG{UQ;M;{u1XZ=%d*hbn$_gQZn#rs5Bbgh{P`RJ@Z`ZPhC3H~pd!M1H>G0@J zJ^fS~raTjk3!iF%eD12VMk^{gVs<3rs9NsP0kS{I>x{K(flkDld4v*OVbeZYWfnwvq zTk{|H6|c|Zg8TA(4^~TMb#<>6bcTQFG3V+Od}<8ua2t@j3DI*{=FJj z$`2RPd{bz^AOQy3`Sem6nN}od`@w2{mOK*$wi6U8t&BMa1bCTv$sR}UOFNwIDOy*x5`=T8=Ui}0`Bawg8Kdu>gn{WNv(uO{uD}NadU6EZk zAI&~v^&e+RGguSTOG zHfV?^QsmlM9#_($dNv~GerX59*5xzEELp=38JWhh9yf{*hLru6V3q1e80MTrADGQv zx3B1t3}WRx__6BQ@i7nBPM+EQp*#b@?6vZIfc}meKD9AQm*PxT8j^mopN4x$GsH{) z20%VLQr}t6uOTc^vpf7%wx>Fhf}ocny-jUrtMI9Dw1`2+8Dh}!sH5Zj4%FxLk?+HN zQg>*7MaX%k@4Tq-SpJ#5CtAwsDy`fFUYxs}?&liU^gF0sCKnIxH;Y9YY3G+TO)TT5 z&YBJ5tN5H(TKmhiq^|P@>eCI{5lmvt>H-#z8>*rR1*#RduiAW*DW4<~{ll{t5tbR- z9Z|Dv9FYGKKukqL4|v`T#wesN(4l4u5nJ~$u*?cyPqX<>3eEc4!l3HfPC9}j;nIjU z##(xtE?o6$q`K^6X<$##ITYoG=f)FtRH`4%1*<=STn3U74#`-nRk*sYh8~Oa##0pH zB-#%*Ov>u)34tZb+_rx&h2&ky)^2Z~ zr96^i`R>~#^)$AgjnBnzqz>J>3dQmvzM_ugWHd&0Lq9}pmSKFDL0d2Q4PWMrk8fE& z0pbW6Z9KC_pAExBJ)p8af0M#?d6!P;0Z}X~O*fD1K#-XX3(}=QJ~!-4jPK2lK5pJZ zIk@l=VkW0_XIJ5`xFk3BrBal*-Sai&BvEpXX80$IeTez2jym;&%o!mugQpFUU*!x z1p)}L!;4dyw1k4D=A&S~nv<%8Obw9f+bFhb+-G8m#95)PF+C^ST*Cg;z<`V%;O?!i zlG-BG63@#bQ}FhF>t95;5QSMn-ZpQRu6;{WhZfO!;F@bqTPBOcF zDgC+A0>0SPt(;#tl!Oce0FEAl3l;ez{035_hLnA6qE zQ>SbyWH9c-)8MblCqRZq_Ru+1chvMlgi`Ri3D2k%K1Pl%T5#b>_jUK?ra+1r~>?U&B|A={F zOCqAJ^zPVyr7nq>o{<5GK!ScBF>T%8|GKtFrO=nXB$~|}5n2Gza?s>J2G2WIu}_t) z`VQ`Vq+{v15+TQ2TW)1|{R39Kh&azLtttIrC906NuGO>uBKx=I67>sqoAV5(3U_)I zkfA(&VLU&`(2t6zv6+xr#yEp{JuQ3Kd8fk2!NkNQ;c@Qf=H_6x*U9z-oDjK~z1B4& z;kwfL?$d^`v2p%sQBe^*5_pnVx{E?pK7TXQ89;Nrfrjg?QAaH3Q zTLmb1QGS#n8<{pYH`jCpnMd9wCC%P;$d(uOFf#SQ*mA$B2p%gu%kr6e5mK=Pr@9uZoLk?vb!hjDy%kZ5*rBzbZ;y$_+&h-?0CUwvKBDjUe$9i z?$_onzp!^j-bJ%Ce{GIPs+m`XTu3?8xUL>1rmj(0&{T~;sq1HRc3I!>yeNFE5mLD^Lch2r z)#S{TlDPR?uq(Fg<+n7zvukJDcsaUPmrT-=4#rz!qEom1zrOLYQ#}cH=4&0{_wbHz z2ToQfgk};j3X7}nL|;9={B({G=i%z%XP_1F5(O^SSO$D%Js%op@e<`Qq`|SpI;p-- z+EF%^g^06iLa#ZUkr9Q0{7D#{n$LN>#3HIo-7f)Ox+6xp8W&ocnW6w7Gcp!+yJ8y> zAt8%|ZE9bJuCXW`q+!ZNdx^uK2@}| zm|9i2CLkFa*NefgXmOQFB*)F>$)A=KQezK>8TZtI}t5hP4 zU%lB5o?7ypEL_mFiW={cOTqgWF>Vk`%7muSHCf0JlpQKrR^|K3dumw_g85z4Xd;jh z#lk14!3?N7dn=#cuG2=d+QYlY^@8Qk!Vp-%Oziw5-2I$Qp`S_uD6 zIo;WAe!V7$^_`MPNrOS&f&OcrpqofaPWrb6q2hEofZ^?IWv6?)H)W=Ez06NnD8t(t z3Lp%g3jorI9=mVdF|`Y675UtpgtIH;Hza)%ztQ) zsQOH;M+Xk_;bRo8)O=HVk4Wi4MP0tw@%W4UML{XuaqrZ4i$X9Bc~Dhz3m&u-IdS$4 zkyGgNPKVp`p3oa}Xf$iEJS~@A($FheC;oQfEUSrVn)VP_(ye?+hPL{GWTp91{uQ$6 z=#+u_CS*sDQ{Ec)fV-@#NPvyHa797r=ctF_56ZcIFh+V@8#@x0^PON2eic2qyCYtkxxs6{H#jKsrZMY}9gNsevqv;W0buEt zr!hB%7!J##ZE}^vcb(sE*|~dCZy{Y^4Q20Q$9q51shDkkl!^9>JajXfE!6dE<@Ysi zINVHmW?Jly@- zs+^?y&aV&e90)OrxkQj3C?Xu8u)Bdr?iu|0 zFX&hb4#Hy(B!86m0cX?g_1_#q#d!)7wC(J;yT+QDJhw910xJbzqx`mG~FA}^Xd;!PTn?g!GDK-em8SI4m@vwNq1L& z`jtS1<+>d2xJfB}cDcV4yvN4?g*Z{(zu)XQ0LP<)NSC){7QMTfN7`0y25Gdsz*8(5 zT$c6jrhLp(Yl#QnJiEt3%218xf5&n?MzHe-nFmbmmyiaP?sul!HNGK{3w5Z^GPYA+lPfZggD8O;jF5`>y5gVG z&y#bgMHv>HxVrq0w;CE9C)?Uc<{od|BH9X;IYVljpM%aO+QWS#NE9-3+;7)qRIjj>Q?7RrRzLHKhDY&o`TH z^Exue&FU&u{upqncWlSvV+ zfXob%^Q}^7d^=z*VW87gE^LvWL(05vZA9<*2sFK5o|Ly=4y||^aP3uy!h#RAX%19j z8qEOZbhVl?sc*?>memjfS2anHb4<8SckfwQBJjxDgHah}j~UMfTK+_~9{uplw<(j! z^X6F@;BQSWz|*@$t3#@oeWXCS71mA79+*YB$#3EeYFLrMCr$}^pXhxVFu=V;FP~nB zA(*p*bc6~mq;87k-S?L-%A8*I-sCxP;1jO{_ekj5*EUFfnL@}^M>4nDJQem)x-EQG z=CaQAk<{&p*xp{ehQL^8S1ce_+mBLxc_DGF6585u&dw=>MXD8_dmJ>`x3s+=y3bFO zYYsBT9D{DEBl>bn{&t?81j9o&bYOD)!%E-iG86T`O9fx1)5?k6R(h<@@#X99^$Rbq z(gbZYmY0A54vZXZjIO?v?7~YqRQ1Ap{o;o)km_vr(K^T9noud1imtA%! zRMvW}l&I(%j7G?29|*i^)NwgsZXi6rrzElW2Rbz>x;Ck~gSQ{C+)JBy6-&|Fr6}u` z8dNbA;F;GVHMe%%9z4RV^*b70P7D7P7yk-1)MyaSkSi!;jZgzkM(B$lW&2>003r#W zJ--Z}fr+Yjz5!?P{4XNhb{VK#f`MUk)5Ffn?fkWf^u7>RNlcJ0C0s@R(D+=6UN^y|q zZG!|wT{EdkT+BLclEtLBitR&I_N#j_49!Vr1TopJ@4J^zfH%qV!mr1GCSgL+7U|<` zlt&=wyrbsAD3Rql)^k5_|H9_ONzi!@$d^G69y~z5 zv;>dts1&~SSE5v71}IU|vngrM-&K51fa5NEO>{Zw`X9`V2!8s_gXAjz%qHgl=9nz= zT{Y=xV7}Cc%xKKV^m&$&#kfN5PkMZ;ex1d2x*gn*x&dvp;cE7BZ9~J;|J8Hf@wrI_ zzs*>3`V-5td#fo!Dut$}Z8LD-K+qq?kN~$GUPtE)#LRXqoLdp@{*fY3BLkW@UYO;| zKvyX4SZx(rq_g*Q&uujfY(9E`4j$DD6_3>h4(>9pA1UBIE!6a7$4W%4*)Hn)VUPmy zjji*-MyBnKp4O+-WNt<3y!yE~O#(9=Cp5OJLpZL*q|6OE+wc!l-pUglRbydg8n<6L z2vWmJf@tK;W|4yYzti5{yI{<<1r`^TsHGh~w?l9pBs3^IqZM119$*!)&fZvRLC5w| zEIkP^GVOZD%C3>H)r`R`^h-H}ugVR4#SVjJZO+e-*mTtZ)ffPpYac9S27EeD1F1GN zdr>NP0TXhTUeXJ#H8X7rclP(EsyyX+kN5Eixtq90Q$sSwp84kx%(-gy) zOw={@wy}8xe@c+#Z|40xp!+*b^8!Cw zTnRpY=qtO$U03EN*JhcVR_WlcTnlzUF1S4U;$iuFF6%=F-y^}ON3$LNdjndJtFpZX z*z6?Ao@=1!8;@-S&cyDzCF~(V{hkB&PgM{ct*M*}d0!-=ksZE>mF;UoI~mJZ{DNM` z^G9ZEAR4E5us&{d-L>g>Jfm$m$Oo+sL}e`&Z9}Wg`@KTxm2T8 zTu^xcDB0*~HuX`yUtT&wfXhD;x?cPqZ4t1pe@YO@v1knYIPpRVAtFs^zMA0kl^NH? zFuqrLpERUu_)}!4Jb5-nVYbI#;wKm3p#QFGK{xuHew|SoI`qoXvA5L|E5&DdHX;q* z{~zTQM|Y=nTDhtU;!RAcaO-2|b3FAOvOCnb|G$uzmA|{yzYKUAaJ6G8K6Z_IWVC%T zL1~=>`v|=>UG=82{_)n4A->833{>ABY@TApV{cA5UiCfTg* z2kqvDm??WRY_ro?vp0(1vLwcei_)y-KKnd*k6AzF|G2HPh;pH4k@xy(kAEo>qjaV# zrDm})-e*=Ujp#80C1;)h@NNAwD=9~KG;bgVn>i8x{-d~@ zJC-l{skcUeXiw+H8EBbH&2 z5sjE9kC0XoTD7L3nNpgB<`u-$)RZoVi_p#v7b4Pm`W)jz=y`#8`}&3`O)c25oNc+7 z1?+IyVb1r=gBGI^P*}&ji~L6)xnq9ascB0>Mwuqe_%}lJ{WD6Vay~>gf!5PJc{}6f zLrjCNcB*WBSSRrd@n?xz2&MJROtE!)ms2|@0n`dT5B zxVdX-&??qq7^p=6eVHNKI|l~?!->UTDVL~TMX}s2g=pI`V0sF^jD08wvG_}Ob>|*V zVtS5?<&HZ3$IFRjRPcL=X$F?+AkqXvo`{OM5(Fo_KA3lB^E-ErIB5d2q)CIbSox{M z=4!`&IlAdzlY^6!*zwjlz#ZVJGI8`#AnvS%*Z1x9m#l5H7{%fyqjLJ6@RS@8FE(Bq zIo?74=j7=Vp%i|#EURaAVAiyu}ln^{{)Q^zYyAyFtFwyyIlDL z+Kuh@2HzRFXo_knqF6Q?n|l!)hoQ&$1H?j=zs*y1 zhv|7WAHQ zJ1*bJ{fqyjQp)Vb@@;qI-#q421Nd%%9%-5A2)%GsO1cV4WmoKhH*{2Y9PL(4Rb`!3 zM{wbvb7zr|d?((@g~Hl@Vr5E+_30A)9hZDsf+w@Kck1wS^rk)gs2#Db+_qU+*MY`= z-I`kO_N#>bVetfo-aJJ%Fo+vJnw&}#h2!_+3mLw@AkQ@CJZ}$cSqt5+R9ijcq(R^Q zD4AOSaOF|$Pw$HKrO9pJfexN|uf{TLye)bh*>MMmnmPl)796mi2m9qiDUaJZh&e$X zuk$`Azf3Uf`r)?fd>DBByKPQ2$8W_Mk+JR$D$dJ;37FP)?yGWQJlAA4R+>mFnrvr& zOb32QismfW*Ix8dF1ahQ!IU&@r5u`RlOXeE7kA6^c}sx@G+XsfROZM4w$XW|kUE-) z*D{JMG0CSeKo&dcf(_juX|vt1l-w!|`a&m+f5_rKOZv?0!bx99G<7jmMA(h^ekeH# z_WjpD_HP38H|AqvGJ(5urx>nrqa*W7*g1GGE_OQ_z#rAY^6~g+q-AXHtI_72n|0I8 z>%arCj?)C+H2It3oAt6UhTya^Wzl1?hmCl7`-&z6JL)Tc6lHHur^PyyxnTr8uig?j z*m(23{A^`0`!H}NSbm^94QVFaWbw=Yy6@y!Az4D z|E~qeA}+Q6a&ZGlv4qEyE?>zz2|AVu+h-$z6mzdFY6MXkgi7nOq<~H27`A4g}l7gH38pnED42qB`tn=ejgvCu+WI}^v%$n?dbd&H$oXm-wwm= zG_Xf7IO&YO=^FOzlcnU6E2#^Uv-z3C7UWv_MsOloZ^9eA{D(|GeI^-Mb8IBk65SHsCOz@8iS zE0?2|XN`a3%FUi5pD{XuKOD~rY+U=s)t4%Hl3Cy5dab~S?`MD; z+`upIXiV3lqOHN>xv)M`3yya&CJ!X7y{2p?~iorVRG%;~RWN0y?>Ql^*}o|cV7?33{& z@RR*~9eCY3jn9yD&5?3j`%LqKVzErd&TZa{d0!+oD`bMDO zIK4<2>?FmwM{cr{C(GP}0dr`+7$U^TgceL5fe zRP^4;vN9M_IxT=8&@JT~+*_5gfg>}A)kMGV`lkBW<(gU0`JY5HWSI!W+T{k#wtia9 z3%N1}I!f0A?KX|yyASaTzc@Bo8Vg40NE5hJ*Yk2g^!!}M^=VOc^~9(WXJEY4mkkGf zapiQL9|#`=f;%|wl3MKj@`C!rO2}?hsi|0oCueykBGUSkDYl6J5u|N{pp`i6&4IkD zHJm$;kyusKaBt#5ww%G|Vs1d-%U>iV^Kbv_Q?tZ;_n{-oCj~FX`ngE3-r>|J~J7aINvJc^=C>>@4!cdh-8nqTVL$|g64;dvWFibJC^rW&q9rC0kJclTbUn54AP)5>5)-uoRE8RpV$s<;iN@*Uyb3)wB2%j#7=K? zcYXK8Q1^?Ljsd~kljHXJ%K~W9$;lZ*{L+%|q+Xad;CdiGSWoYDQSgKP$Kif*nG&U` zDw=Pn)}Yqq{4^TIaoV+mAqvNwrvU{vB=i7k;AA#4f#e?-0{q(}bggzuHEXpRFnk;1 z%2zf1V1K(OeT?vP$9!}A;j*|>*Yp8&%lJv_NsZJba4zd+v2@fV1kY@X25HXdCpk{q zMcl0p3dH)P+X3Wlw9YW3jxYg`j_YiNYwfYL87-Gebhp$%=ii;eypIfk!Rh(=HN+zn zzn*Mu;P1-6Au%d>wPtBIDsyM+6YX20=0bRFbebO&_uMp229HHk{D zmz0gr>_9Oq2)7vS{dN)lEc!u!m-?zuiefBfm6jWnR)n}E>zEz@2TADf{d4zKw>Tb3 z|Ri0rpu*??*nNM?LKPmGRtRatATRpohVCBWB)(kHjI;2dfLd;~(oyOERvSNFh+N8s0C)_O4FPk9s-#(T(0Bazw6`3Pk_SqeM3^m4k zAfI%dYrwq|^ zw$0u*3~Fv^kto$5xD7;W*h^R{Wrpj`0mtU$F+(P5#VkjH1k=-Gr*O$6wP^DYu=5-> zCm=yX^2F3=pNA}9t1)2rV2gjh=?qdWOwGl z@wJRnzamhjeFHP7YWoAvt>Tvc=@Vw(*Q$f_nK?AHb>JF?a+e23Uda}uU*4PLc-t720^CF5~r0X ze&G~H@O<8!emBRCyU2Wc*bZ_6o^TgKogkDCGn>36HlgE8qFc3+nH%wSoAq)*`nXzU z{4LD!1f{cN?#1KHiPI>9yh*cuQv)vw4N2Y+_l>uo?y`NNvZ4?vm6rMMX4kY(u zS@d=0(@B+!J%U?IC<)OYhCPusAXCFYFf(ATIkeMO02swWuvn<&h`x+5Gc&6Y_VOi! zuy47|lE|;+ep35F-EF5gpBzOc%hH^Eh|`w>@n1(5JnhmeTmQ^_k_&FeF5BqiWWfi~ z`Pe~4EQ{o?+v{HAl#q-Gh0-*a^2A@(-!^w+TA*-HXcpHQxBiGKQC95TM+OG136vFt zZtW<*aNHgJ+^Ph^NsoD6eL{J^5$LY@`tID8`Sy#Yva0j7ep}BiiG5qwgKtvqe6H=- zMUu_!&Z_hHKuE@>k(B4shltvPx~Gg|f~2Iee?$+!)#FB?#NlHw25sr@wf5?5bSkDr z4T1Az3-8D-xjqCRD+?+Q$RGPbnl8r5(Kzu8M!U>5{3*nSE4yrX>^-LYCx76-3xyNb z(6ZzKI&q~-HNHS52c4KcQYsBg%;AO3GBhm0uCoNMLc>f6TYBdL~JCWw@%fL=Hp}2M;Coj9H zG`4;xOn+NkX5=p;e-@RTN%`N~)aoYBb#_VO_us$yeip6<{hOb|msdF88k<5brcRf_ zHA^~SB`8lTsC2004W8$|YUXTA_abyrlkNVmmh{-s(L<{?9K2nvg+eomhxqx#`Y(3} z9G}s}Z_cHNrqe}iL%({ja5 zmkCuL^)~<81k)SPA~}*X3vD8-y}u6&%>vBZqyUXApc&i#V|QT1=R8cwQDe(L;Quo# zE!VJM0&qMulT+hToHw|98u-+Blp9tlYQDN)x8mm;=|8vl%oCO+eY`6}nTd6^f6>rd zG{z+BZ|r7~XH-|g#+ZmzhcHv39$Mu%3Cj3_8+w{5eiOyF#GW$G{^{{1xySK5WpB$5 zCMW6kr7sU+0QL4=kS~kR=lpKeDbg-q>LG>}3fFFpViN`Gw7&ZpmDh_Kl(08o!}lN} zYBc%(_tH_it24{5IJciBFWb(v#k z=ZWgvD?rPbQ-!nWmML6euPF0PVLgZ2Ey;OrS9AORVZw5mc7ALOFW^y65{yRXbHgKV z9qH#%YuDG^+6ZEl$%cDA+OlHHEG5)tgw%@a^3!6T%7o8wZK;}!s4RO$o)>W!CR znc#TvmSCbA6{vjKxJOGOiQj+E7&xE4TwD`5roOCxvdf;#?3>CU=OW_{^udeh0|=8}6hRY5s@MAeXvujU z^kZ>aJNtwX2>}gXA1Q;$MQY$;fga(zo5P^&Zpe|2?XRMrkE9BDE$9`p ze%<0Htnn|hbnVRW{r;RKjnTfO6%~$Q+j%&|McqhzzV&w-%^~%7#sX0O;msf9Y+-!^ zF-)lobp$4#v8G807w!&xov70w)JLI=dpAPr%Mfv2k>bDH!&>zn;;wN&i^C+bCzSWI zG7fi3e|<6kT~G;g`7L>R3hh6o&<7s>%%-H=s(zI*bnk}#91gXeSsXXDW86iJ8m?_E zF3lYP#au#%F4G4TxpEVC?vLLYzeO|N>%t$5KL1$hzaDM0>3cm@GyT&N;n{xKfFE62 z5gi8w^fBE8&80A}ikTJ2;ENvcOJtk!s$0fEPM#&B_i7f1709ae9XnnGFx zsA*?{YfGIO@Y4~BSpdDlC=Bgdx{zqccck#!^AJKpEVmnkuXE5qF>z(H&+Ebt_Kq2H zyRZRSB)8T{^EmLsT8?%$qsP=x;+>+|cF^XyD=$6*cJ396Mxi9dzPyp;79(+g&-TZl z(%5nZ-18oKon^IY-d;Ba#*aY97UMPFzmam-8Nb{O_a8%K7H=)?ZNs>% zz6w9R9nq0YuANeud}G@Mo^*6Qfw0l+SlwIjAHM&PwjFw=41J47bgzzUZ69`I(aDkJ zGMb4<^o58Tkd-9{1#yqH`g^_&Tw>w}-(&Z&u;}D@^GG(=+nZzrSC(|AfFoeVFB=)l z(0qkh=|+SVl1?DY)*MWBF$NSK_84V_)>uZ)t9SdX@Fn+kX)GsK3F{x^qgCkml)9U$}C#Y+|-u8CWi(ipK%8} zq6Si(#f*5jq|zP2(Y(Z}vi(q21;<}kHNsx5Tun_1lnREAB~OIZ;*84NDrn4)CWrIu z>SlP}@IJapQal{I6HOOs0|(_PlXfAI0ctkB+Qp{h8cQ*0+wLxefQPq@O?yd{Ci-fu zVz0+$H#T6emCgev#u}r&QJCkwde}NCi;7fRUs?IW@kRl|P^hFMjjTU}p za|f~qxOs}E#t9J$Q!qhX^#Pe$d$9#i@9Hw&y&D_R#oXYP?%{u(p6K!>j6qM^Jog;4 zjJ17W73?+F_I=){%`!ccPkE6d{^ie!qq=cs(cH^lcV-pz1#TVL_*nw{^9}Da)eGO4 z2HvY!E*0wFJch5V0L6^Ig6MUto!Dv}|3%(C^i)Pwc3ghgOlB~%Fy4JSdW@IRG!4o1 zU*3G6yk&LHtBh-Ui?Dmrwh<-z^y$;F__>=5|NV{1D&&D=!wQB*SACNB=qiK%_Z8>)IkWN0@jYCwV;rv^ z6+LlEH1Nl&ZS7seUYaJ2=iJ%<1l(Y>HFwP)aq}-Oyh$Z>8TPGJ2|b{xUoj%Lf5A$Y00}H;3Rr2$G4)mULwQra>`&Q?X&`2Sdr^ zwsk?>)FA(<$|GTy_D$t<_hATuo9DWhi|c7ZF#;@hBEI~fr~UM@r;(xH4OxM)8jc+} zD8Tzax%{KhHhi*V$%}v@MPp`r$DEJB&e&_>E=HT6u7JY%5R-JUZ{=;=`-|I^(}#2N zu1&e0+50~31M=BJFJJkXzJE_oX7!ro?4&f-?ChX?CIkD_c(^W#AfvLF+IBVK)v(v! z!}opb&y6@y`zOX2bb+)%CzO|dmHy|j@u0uQmA4C{*`p639RUzg%aA3B$AQ71b(dw& zdtCI8Pxq;%YWZ#D!+zI&d;rM5qM8h17!crerb^O#_FtW2pVsMDrz)<@*1U@xHf$NU zl#S$A7kq4$EFm_g0sfLAg)%`0vV4XO9zGDiaE&TwS+8nb;)fJLZ^s*7eSIQ85SZ-g ztz>h#hB)7qIV0REcCOqdGQv)vp>e=Y@~4i@yL?zy8JR&q>R9~SQn0ZsVI39I9q>xe zwXZ8{B*J|ow88n;UYwCgR6Vluk8oT1WcER?U;CXNiag&Smhf>qu3sj1{W|?uvbh?Y zVXZ$=h>NSca|MFTu3BxVZaMf$)vUJiWoEFo*#jm6L8rr_NPDx?*@KVxINJ;nIF$CL z!>0`}(^yF2mM>Ar%mL{B`{ zJC`r>xHq$xJ0#s58hJC0f*%g>=Z38=?0#J+zyso}uJ4+HyXuAPrcZ422x+`#-|CvK zq;dA}a6!j3M(Jk5yc0iNNuQaclLkkq)LrW$KJX0e7CnTO&+UijoU`XMEu1aMZ&Gr4 z+v_P3&eqj8ug3LLtKkU4m z;W%Azkd?PwR9m18b@s<;c%U%Fc4Kfl!o*L%hdM&ydsq9l#0h1y3zUe%bthu$-8}Yn zT)7^P*Qm+>x`OoC9^EXF$_oY@i+FSE+peoExqdw)S94zvoI9^&)xZ#&zMiYB4|uu& zHb$qrNAt*gi%%YOpTKX@0j*9aA~vst^$kGF0jluF9z>GT=5_O5b+gM=GEUkpM-(qd z^g(UQ@WrtVHh;Q-k-nj8LtfY3qVRKt1>@09jp!}B@T?R}(;AYX9j9ft!`DBa_8N2r z-&eJifzwBI*fbEAJ^03-;{z4e>F+L$_4QxP8w)Ue?dh}~-aX~zbLqI?sPja*pDwxt zPgDKwS~il2%!y#r*ABi8o90Xxbz}HYmtWdZF`|Rn`@AUT=$U>&Zxs3&y4~#~i=Ue5 zYJ?tpw4vsA1vdAI+Kvg!0G&IwG^TUm%&Fs~RBs2JmtX5uG=IJ>$8VozJ>nE@`#4e9 zQC>dO5fOdRJptTvt#uv=CAI&c2fua>}M!3EE!_R_@8yYrMnx>inJ{VlKDw{ zuLya%zP5Jpe4dR^m?6(2f%M}Rr}rU$dt2U+^&i%umO{+T=ht0FB-KC>-bd9SEtiDY9Ay63_l)q0ezYs>19yMbor zh-31Bo&9}`w_TNl^^K>um%?5&&6?5R>fGK|vrJ@4W__OORLjmG5(!Pd$|DE~Od|dq za-1j0vvS=P9GvN3Y~a)?mPipOeY(m0P0`B=F3IqrcY>5WaU-lCSE@>cFf-pH1J&R& z3)$@WyLFUNxsFx7wE)S#z*2%6wG7@(N(IeM36QqF3z7xJMpB-@sT9USeb@>~V)y@K?g1A{Pu(%fQWbRgIe}N0O7T?5Jsxn5 z|Ej(EgDE^)9Wz(UBo$iN2KidFG(l|psino!p^4#JB$RZfFO3`>{;LK`L%#mAmO6@p zcJ|57tUoR@EU@YB^G2~H6DCYWr@w2ZV#9~IeO|6vSx&=V#+#W=RoPZ7;`Q_eP986H z_F;T`o{yW}8_zHF21#C=M)tkBJX*GUSUu{pcJ8`=L;Tfm01|DGKN`4MHS+bnUFMGR z0(Pkmbj+W)YKn^HvBj~+m4Xk98}!1AZDxg9{4%q_i9=sXHFLrG9p?>BB5Q3V1?|pk zpLZ9Rp^4uP(z(4H9~do39#Vi!fK^OwE1`J*->;;?6jii~ zMaP3$-%{(vXG_)f;?X&x&Mr1b28Fzm&aG-EMcU}K&zi0FA09%x9i}TEjSIL0`&Z=J zaz4S?-%4w)gsG%w{=G8;*)F*+o^RS+F4*LSJhb$9J9%|H+=RGPG^A$%J#|cM-Yy~e z8?<$1iEmzsFRfuEQ&-@X!{v$J{T@#P!?)27%ME4`qtA-0vm!%|Pqe1Sp3>Q!<7v;m zA9g3%8$5h__PTTgn-2tvm)xoVy=$#|+=pK=&Fb#af^W~A|2<44nu1`>w1CB=zXof8_dQvK3z5{ zF;gC2y=_&+(8J$fEsMJCBGV=4@|$@2&Zbmg;`I_i@G5k4z8#4aUlfThSq{21KB)1@ zl84L<7s?`rk2<$1HfcL=Brg^EfUiF?Lu@$g@{+spEKzt?0f33Le$2;?HEpr)O3_@v zV=d4CFgd|_b|$eqp#<@VC($&(On`(J3k;t7=DGH*s>*{GS>K#7kbrfTu*vHiOhi02 zD5zV0aYA`zBi`kuW|(gw7PyI~``=GBS!YqSeuW8E&#LRJHka3T@YlPFnEklo=%M%S zBxxc-Euekq6+WR14rGj%7PX6&O8m~dJ^SzRqS{+8W8q9Acw$Bg@ zOugZWnk-nRpW6wE2*MxFFt0f_s~%S6r=CYfwz^I4N$>NmUi6R(N=Ues9a=?%KmsrP zLAMlOj8(h-b1z5xyNH*p0)hI`d9O#z%4iJB_8q#kuY_|_SA||agp)G;-E(TncINy< zX1sc0>=Gio=H_$1uX%blbFg_g!kB_f#{Iz~M76ohuaQ^+fV#YOx%JUE!XFQATKX6< zdYwIMyf@ZykpWA3q{SkfxwHS`7uC={!%XS+bH*7uI0(py_%d9naka9W%n%7zi6em0 zmMw>`;Bkm%Q;U1syZ-ZJlN9RkR>D0mt{Q8wgsp1)reOo}YIx3Moa$Ky(r2HY1)!qP z1A`>8hNi5LYRtWqzZ;iDXc_Wse!l@grqAU6qphPv$>GmoEaM^y7D=$OD|76s2JD4D z{i4s!1&&8QxwDl+Kt8DX^^M_u^IFMNB4@VX_~U4Dxu@FJsF?SJPbozkU4x4u_zYxh zRO*>8Qvu^AHO1NPiO1FnLqF^OENKG$c_7@C{e?j?%BJG&zxm+OZVG-@YHLk3P2WqB~xVxvI#RZ@Yia1#R?D-X}xGmZ8y zw>5h70FVIqSo9N^Ig?>IKL8rB=asR*{02PBGm7G?lAeW z`D0lIS{tqf${l}Mj3`%e{ChPxGTx%qOK5Mrity+j0Z_HOE4sk4$C3$~3z43p$l?5r zEsa5_ITJ@447Py(GZ>9C&d}iAc*)cG5=!YE|B*ZUg2zHeTecAQqkEcpD?*PZ9{o>) zX_U%nG_hd1&TjsV4F*Vckzlz*YmbP-${)MLq_3HNh*BwpWmi0k{hZi)t@}E3*P07OqhL_7lq$3s3=-}x5CRPiN-^4_s9!uDG zT!8AvQTCEd$d9QUA42ev^EZu)PV6?`@KZxsi1ucS;cJ`{R)Yq++auu;d9ls4^kLsvCd z^I(2W#HoRc$>w}Ogl0qmA+#n!B{eGHT3jH@bBwa7xIToB{VZnWzgPOg1W|A19_rCP zfb+mW=thosV(!AZK~2%`n2CaKa}U5&a%1gR>O5Ppm)yy46NetoC5aepspOip$@5q@*_%YJihS3imGB&HKj7AH{aC;4t9zU&x>;7io2hT+oo8IC z$@$#04kTa&P}-`tJ$e(>+k8+wvy{A|$BGJxYw@vdn3>#lnMMY&dSPs!DG?ep*uEuXMYvXXE&1x&S zDWo)aGDAnicl(?86X_pL&bf=1sifLDr;SZuAr`c3if(Hkqd{KdI@w{Lv)R<{VYuyt zpy7rXguEd$FVA$=oHVqX?~`K7kk6OWe2VA#Fsq;@lhRuWMC{nA5jI*|ZX$#g^zt}=ravJ&1I{ViE+ZiWj$1BUZfe7q?Q10+b0(I9k2EX zk)v)0t^-Q>xO%qCBjAz%&mmdRJ7$NdiJa5od~<9XTdVmsp`^S_5vexZa zXMrCEuc^p#{co$Pphf0Gd8A)7rpTy|{}u((q|LvKn0JB@$Xdf~R;%g&S;pB^_VLfq zeTLw2HZ}>4ddoxxyangjRlHmHS(wnXl@b(anvgb%7x=FKOXWG(Y0(kO6d7bo6yT)K z>w);VtV<{m7r!mN?`q~=A-L^7>YG#d8^3MV%%pmgs`e5gSw|4s*Fkz%$Zy?uMPg_t zlXfWoCEMOOG=shqCtb&WRr>DBxcsV2Z5zNZfl<5qBq4b!c`&nUu3XuHD?spnd-h%8 zi={A)v7DWKZqw~{^P{H=Z@NgDprp@7ma;OTch=u5V`9w={Vu zR#kgBcOXCCIz6Py%!Vq_S@V}`2^dcyUIn;T6W??IWuLTzM7he#ROKk|i=^p@WBO*p z8!!6c>PJfLNqWwD{|==RUFZMOC$(nRd1{pPog0rRUN~r7uFnkY9tX1!(F@G{F3eQ3 z6$?oAR!ZfxVaJ(8Nr@~0d1(hhx9JV->VHvu+pMLbD2-L)m zm>&OOZbUrI%{iJ)JAJRri0jyRSdkSO`J_q8)k{986)b$ehl|ra-e+}vm1lEd0Bz6uFVJrh1Ft+KhI$~{vc!?T*DdT-q zkS|qp!;Lv4T-GLl0-=Ra^2PGw=)7vpGORZql>F^*7^~SwC-uKS3RIK=_t9I!OF`5& zkUYTzue|E@17UTE^CD8sTtF^h!;(Dnoq@fhBh{(=We;F@u(8#uwUKWI{9W+d8%axy zo}P7R&6%Qk9Xc$gz&0Ro$;ZyeE3o>)03+2-?Ky4D1zqqqQZ0@ESnh>J*d0CmG%fJV zRMsZc!N!nVUz4wq%(cSQpZMrz98ewelAb6^xP)>!n? znFp*r`eE^qOJe#7#@U@pz{pS=fxSk35t~3PQ+En!FO?K;u(MF14feTHhJE~NvVMQl zmp<3%jPw_{?gwU|JI_tGzsC*pkbVxGEKt0T3UuQtI5bN#Qw97M@Q2i zc1&FZQ=uBI^=n`MwyC4;T}Eim*a@+<4=Jh4nR8`(B0ASc9m3@-4d`8`f<+pF4e)`*D ztui8}=`i+2@?Zk0Y=Sge?;#tNuuO(1QO4F7wDdeWn!w+(2tR!S!{%$4R9o^wM)0V`?%t&TeLy z`FlGR;$UV+2L;XMqx1z3JAbr6kuO~z7R>vLP9oVATy=G;R!VaAMflMfaN)*$mp)w_ zyb#e+)LfZEAJw)Y$fdz`Q@V9uVNcR>uhp*dO1$2^1SSIE9N8_^eC&EO`3IzZ*X9FB zE4FwaE^0tA>K`+Z3FY|PV>B~pBD57p@6tWBo!l9&K}cSZb9y$Q#Tb^8p$8HP6kkZY z73#2m!~a~qH&!Sca~pG;+W2340TEd=WvFuh#LKXBUfzZFc~g7$T@LZdLCe)ZAD$Nl zV?CCJQ-_#DbnEFw^XHR}9=h>OM{`Fk{p1!eo)Y?mR7S`bmw!?%9zC_XRS!w%>0l^X za*WU{WH|p&XZyY|i$5*%r4Hl*16*8O5@Kw^pcTo;hK2@< zh8n48dFNY?QAxW58p&PEoFz48O&_VwAHjD26^;`u5;$kkwL zYis#1h4k6}qR*xi=5s?IeY-8Z$vH8^@Zh(X-+_8HSCRO2p*U9sl0f?-c8D22*>mO0 zp4c%1F+z;0t{eVY9t_St)GPfKa>1|-Qu2@d=U!es0ZkgBEqspkei!}I$ceW@Q8U*b zBZe34vedbsKQ?caHg^1h-*ZB_-M6M@N8HzsV=`G+;c^!eHRi6={Xa0>tJlA^sdboa zr=55_ z^U3+T8(OxuOaoCSDqzd-aM^J$h?G8Zq{iLEecbR6Bx+Z)H=;|Ccpq5R<-aECvsiL# zMb#CWyd*}Jt_gisBAr_aarJsqjo@x z%PjPyklYrZ2lXq6R+g0Y`^pno8wF+{DnJj?+#J^1>BT>6KgTwARbP!&hVdMfG~mp7 z_5ko?M?%RNA!t)2U+JR~M*%Y088@$Zn9`1cr6ZyEP&zkN3I#oPa6P7d%z0m>QA_W? zHj#Bg4uYH2+#JIj@>2Q20?rXbufE93ie!jQWv<>Q#(pO z4><#(I#PGkGAhzi&vh%fc;xX0zAte3?Z>Ev)Uwd?85eqrG!$k?u!D1%%^ZRa8f>0^OTWx%Q568by(5v2 zjSVTkYU~m4^_O#78lv8usWeH?aEqNJ8(#k$2&CrQf@+GEp4{D+Yi2Uq^?98!7=SnS z@rY<7wdG1R&LJF^J@bLBf3P?5SIbi+NiojW@N}>eK_AQ<$aX#QzceMQu)JWy-9-3vF^|(PC z!i--CB>^V{BrP(I2hg@XI-#C}`PYTNgAq1Xyglm$n=GiH)PqHNcr5+jRG0zo8E8H; zWDeZ7%|+te%;*<<2h1;gd>+(;@4Z57 z#oA$IddwmcPAE@Kb$a*rG|0$!B{6edN41Ww(baJfm5A>YRgINXRduHP)zPP|r%lec zVb||8S;9)--1Gw*q|5K(X@Ty3+FY+6bXh|5leoDdL!JSbZ#_K6M2}oXfxo5EW!dy6 zSPKPgpUu%RIHCbLDseAC6OEXAXPXo+doz-U_%NdV0g@m_unL^i0;Mu$bbb*(F@W6d$si&J+o>vF=(!Y*1< zZHO-5b6`*KN$kidjHR&=T)E&N^{fJ3fJ6Y6fHy1DsZd51^OUZ#9jM+71^~b=m`HZH zeCTrr5Z7hFb66P}8QI=&(d_YSGdYfj-89-s1r8#dIc7(!dF)PUl*ta~KzFU*r^#wt z$quxFf@}=n69x3S|6YkM_$*WNyR;GMCj7tsRe!Bk;NcAGZE|g&3H$VKh5@BvcmBms zmVZTId7Y$3x^^gY_siAV6Jv&kNUFI-(GGCf7Y|x-A-d)lbTW5`tzbGDI|c$(zi=mK z$OrAa7bDE|QkAv8>ywuVvA>7>1%eyC*aowexa2mTD3Wf0Gd7qHIIL;DR^3`git@Lz zfq%U@+I0u`D+WPu7~&wso794uHiVZM77{`^BJJepG?H0T_W3wj=V81En=q}0fzm8i z%VDqrk)reK^%9SgCWrj@a{ND`5HGvYi22?%Vj$da0tFLgM@y=PHURc?jSiGi0N7jr z_QENxV2 zw-t>sU=gmR*LMPdbNE0!Z7Q`bSY;cG(T|G6+g$Wl8;vpkAX7|D;uID}4YE>^ObJG- z#cb<^rA=LyLM0^FwYyip-0yzZSi7?i+OuC9BaOVlV@hA9Ne`?<&F((GQ?DfzC*k$H zl{?cw0LnZv2D!xED_je6;EpLs$o+i@V40z`O@Oxm>rMLlq2srF z-Ce?2f<0ADHCjF~$A4!&&YBEs8JmR`B(`}>6aR~r62Dbok7WcJhY!BIZfv0Vv@Hfl z7}AFjX7RF$)T8BZ`(v;Go+wpG;1=#LZES3`Zo>h~+HrU_tt;veSZ$7odSH_QqgD=i zBTX5_*n@%H4P6JwsEJfSZ{S{=PwP&pRKOIQ00cPWJ6d}-OVE~)2L4PHxo|+&ZQ%Ru zH`iKJ3#lScuE~N0KvzQ7d2FfqPyy7BM;pvYz;~Q@#0Ln@ZHDf@@^i7+F?f*+>x+h! zyQN~2DeS$}P8F=TYSVH69GFTN!{`JIhxNF0eG~6I2WLmmNnkdgS=!|66CnZ1VC{+C z05&l#&BJD(lQ?6&uQ+9$^nVLBV9x3qWuml|E~r{;bZRZQwS<0Q&lW~OWect&gTI6>qcO z*OAKN7V4X{qPQGgyl$7#7s6Rb_wyZUB9xAJ~5msByDHAFV;F*OFm*VDDt;4 zH5(>$PXm6jN`3Mf8em&HhF90ZRNd0iA*t`2lPKQ#3}f8suV#Du?WeLg&CP=jteP$+ zdIshFJ$HUq;_8~8zkYoJT+gJ;g|k+U$L?h;uwpkaliOC6;kAg5fJ6{2f7*5wT;8z4 zr6N&@ddid#Fm1E-zwH`Mg^k_8t_`yFbTVJc(fg32xq~L~k3*fkMY&;_{~!e zKtviK5e~!#4VH~sg={7MPEXyi`Ahc(0Djvt&^6+${=5UYLB=`TnqSFbCtW5ZG|N2< zN4mZwmyBV+u{Wetgt**es1~)W00E`%bhhljSfbDsz_U_)qgYirF5^9$ERR3D#2=Pn zwq)Y!r)zW<_wLO6z_9bf{zuVwr+%c&|BdDY_gj&YN)Fy7KCvMJ2Y>viQgaXK`8Bn# zw3;yE70fu`&YaWRG0aBzGA-M~!pHkF0fQ2-0=Rhwpi55KE!oz9p_#cL8SRg!CsU2o zwgtnE^+V|m*&8aHlf=(zPJ18$gO%Uf9U~gVxM}i_Umk4fl1NubDhnz4hxGQTfscEe zhH|71cXy{q%XK2BDDvsbIZ4F)XtxA z*@t&$Ur_QxPiJD#e8rzb*ox5uFQQIAdy&VN_7mvQzv^){$>+Tnvniwq;>7!ZH!Wua?~X;PNdD_@C_5if`iN~j>@No4fmJ+q*XV{o z_qE+xy}Y6;8aJ<`3Fioncv1yg%J@EoA8EQA9g_cLTC@`y4Ly7F?wCr#vv=u8vkA=e zufOu!f|uKC?5r}HXmUT5A1Lw)hM<~*VL%^{WIc`-gSaa3a>)NhL2Gl9Lb}cvufWLk z!*GVVF={P>Dw||oHbo1IfQ)u$TAmt^+H+Z5^K}Z7WBz)Y`|GIW#5JH3zbnF6IKL7_JQPYi|pj@g`+v7O>o)HZ7q9UtLUo~|#EVq5rk&!jIg z8Cx$;mouG0zKjel26DKKXx?B2=>uJaJsM9Nv5uBZUfsY>B<8L!C+r#K`GOe2^= zJm8Ez<(p`ffrFMtHd^bVF6)7)&}@VyI&^^IZ*8E%Z8K(;N@)4V`O`MRO$ByL&h-;! ze9kpf-Bc|mT^pW;oIfwMA&FW_wOOeN-4#7Or zx6DS;LO?|z>k1SfFMdmyR!`;BIW#~@GoR@gpFM(>K#=n*nEI?kTYz@Tz|Xbf`!-Oy z*JtEjxdRwBb~+>A#JUvgtJk1kii z%&EEge9KAyi4YSo(tFFy>o7@&!W|)bgr?rO1@U|O%{EV`*##coG!r4}=2`>H+{O*n zu$+=)Y>L=OQ>m)vi=~GL<&thn`Z$?Y0uw_-AQwvKKz+P=@wyJb`tJ3X!CK&t*7lq$ znY*!X0OYTGM5~z);JY}vdc3f9eeXH)3eJ-azhWzm*_9iwa;|nzI)EFI=YM6h%W59t zsE`fNqnBmbqa5?}hFPYPfRsWV7=pv{Of++Vadh#_0 zdcOfnNL)~1jvydxZx5R7#ES`~mwm>W*ck^*(v_^Sg-T&9Q9~yWW!`b4=)>U2MFI00 zoXBCF!bTRmt`wuXf`);qk-^|5&wMc(F`NTC3Elfkla71Ch8dxZnVW$<6UE`(jYmXZcCkOm+P4ovVbr}Ym$*zjjmeOk`{=G2NhrHX+1G8(!W&Z_jUZeFYarpj7JkHJfjq_ zjA+9d4!e%mdnCj{({yBAwX;oYyoolTsGbr3Jk$LEs;l&?B$i2u*CNYTTs5BtmV6k< zrSr5?X(Yd%rXB!fG=?N%iK*P@gL{JHJisQa9XfMs#HCu8(eTU`?h)i*aToxl%v3HT zxhSbW7sS^P0z74Ti?@l!M?cTD-I}W_2I`TV|H0k{Qc0lfkR;K0@!THoKRUa|~vJJm<)2-$J0qg)4zziwDF~ygOUa_IkLzi;M7d6);^MBF!vKpDN;BrEbdz7%3HK&A%+2Qdj!l5dlUUVM*cmPkx2eO0FK@= z@HY#T7QiV{n)u;;Pzy6Pp#pf4we3B0TQQ$xB2*zklUkn%r!#P#Jb6bmFET$RJXJmh zY|>(qY|$~LmFdcc>YSfMO#RjK_$OmlSKD{Q_y`Sy)($IH@2W*7f_kQYAJsLq^dVU{ zJEr+u-5qiBec$OvlS&A^n6<+6A$8bi(2&vBAKpy?x)ab3)W_vIJN3L7fNUAAMY|V1 zMeuOK zce<IA9>=0W6xjsTC!r&n;ab2bB`Y;{ z8)(c|0=zO~-2Vv!*wPpW(sYq%rJ*!i`{t`)LJ8Gsk4ySG|7iw5-Sj2x^Y(STS%}JN zb8)U(w5!5U#iN@I-VhroubHK3pj)X!T~SnwRmRa`09_WNHKCx?Ra4u-N$iHo2tMy& z^k*GoNVw@d=8#ZjFrGKbp*!*nR{_VF( zMD~s|r@b{LI3fh7J*UJx?sc#{AC9dEQVfh%Jf^0W2{Z&lYkuh5c*t zysr9f=9t&XPIOQM9)kp<#fAhb(anpRc=Uw*2Z?*-d}IBiHK(Fcu(6X~ef(J-oq^+b zic&Gkz3YEmkIYy1vO9Y42!)qvgSC)zJgwgjy7WNxH37?g2eZKVS)l1q{~qg0!(H># z<>BGs_pfz8|HhWatg;6qPv+KfN7mL5ug^(jZI=D6jCQymju`j1R8n3h5Ssq6x4HN>p;jzb4y$kNU7~9 z80nURMPh5Uqn zgY=#GJ^fN?SIF+v1${(&p#_u}1hX%9aV37H!{_~UTsu4auIgYIURj-&SPx$*JOfUN z4cnS4Ci>^f2NLS^KYf+Oi78Ez`wtgQEcEoJZ@k{K5^{=rAJ+Z#g%~>nb7W^IFZa>y z-hsG-=~B>GJWPj%BfQF}MGoJTLm<+ZOn;$A`+G%1oaV=B=<5c-_kX3-Sc>-?ir9hB z*DsxiG|E`&8ojS$(HKg0iP&!xW&ul(pB6PO5SCj5g3tRsm-&DI;7vds!jB6rZ8}t3 z?6TG@4_LJgHTSpO^VWwMmC=-pV3I#y@W*NX{|7)SO{X{y-I~|GSUqt^HC^My|F)m z9BuRO05XR}GSXRi8enwXcxtv7a)Gp3P zfVFOF5RCg_Q-pTm15*m*vt0@&Kit+V0o%LGm48+i?OMhAN~UW%$Wj1?N;A6cPU zN?B`VY|{k`Q)l&!q^sBM;?+0bt~C2&r%Ce_hj!f^_WIj;Cct0=e?owdf)RrNl&nM& z!yV;A2I7g{AAv(^`cd!@p zKS5`fH6t zql%M$YNlKkjP;r4f8m+yLrS4YgOJ{PAx)u>RCer<3GX?(b+ZG3liPsH!$ouX(Z6zC zsj7CYUlBeI!nHIs6yrA6~C`D?lrfS=thw6r9RpMl=$!MMpuABtIi3S za*UZ#RcUj46D?`FEb$Az%_Fj6OT$37SllanadQYHXz}oX^(|)fjDC1D=gpij3^kCA zGO373m8ZSC%P3h-i4}mh+Rpt(^U(wzcYB1a3kR=n=$7j}KU(fuW_~bds(ymyBNG%9 zwB$UR?|~G(Mk0|}i)}vHfZ&5*DB#WT*QI8+V!zc0axg14ilwHxFo|J-mNmG&Lk|?V zj6hHv_@cc+_pWX}v9M)o>xfW3eumnpJ(P7{a4$MWoB8wDI+D!lFCG2tNF#P4$VL6+ zYqo!@R@)j_tYwEzqJ0Ox19T?X&luCwpXN_|NeEt(OC)~n`6^G!MGeLGmOh%TUxVYZ z6r{f&#a!VW%TG-4@ne3yM_uo`tRc8g&ba;ecTuxEUTS4~gM6@lzumIZH(f_TNAvP* zF}?jpR-wlI9<{b3SwtI_gd)6MckVRfd;p$*wX(7@%c-iv`>ONyaA@F7f^79luC(ee z{w8L=R{G?!_(w?|nM18dLzJ=qT|U@x5pm@($|xc6`+c}8WkMMd1Fdu&(tVRma{vCQQaVswjlWgy%$GD`E*2|}jo05RhU2ts|1jrub0#e`S7nrE z>yQCv@_;!uIIRBOkdaMFl)Lb&9NTQ>BKMb-)?cYhv7Q93vv?8X9rmTEqgI zC?NR*A)nOc%3Y*p#eN3RtbKX^0W~rwO7J3GLjnW9D=TC}%=Mj$vBb&yRV5cy6J%iHlT8^A2@bdE?5?7DnEe2uH?KmC zdBE@&12xmrtl2;Mm0-$iVnE!0lOL_aq-sEB0c^KLML;Ap@F=}kQqJ{URvz5_7TF{s z7o{R@#Ljh35-kRvrQgb}{A%$I0iAIXS%&{c&$AbC^`$QcJA6mF8;J#+!G=P+AMcPg zen4rZoQ0-pX3Hxd7bx?i!~azH{91b>^CK= z$lZhetG_YE=6APdaK4`p!0)l;YaH(^u{5JICy`7oklHg!(#Fg9grer8T2=AC8JZa4 z9&_!Ki*Uso`d{Agx6Lg8B8gn-djlghe@r2R^tarGEw6v|ja;S)Fq`&kxcRrzmYU#V zl>z@KF@z*9>C(G7soDD}84)1Zx8`bTj7rwl5!@OMmaGpZzdmQH$RW||h$uqIbQ5nw z>%N*E+iwjGY3adBv3b81KfkR$viCs)BswV{56&l!?XSDD!Zh5WFK#~aZ&764Zmkk1siMo2@4V3Pe?S?a8aqZyuAVp8L;wwdU=esoQExbLKhfAO z+45s@rD1|DD@bkTJo%7sFN4WR?=Hs`W#o?Z8!}$1g{W3QcsoV@9{(hmU734Y2kZG{ zQPUH+@|gBl$ir#A&tr@8ZcHZVDjF-@b1L`8f_(Q`GG=M0wmYSY9jC~ z8>0o`%4PT;0&md>sb!h;CFiyHVvI(L48<4H5MP;dlH2N}9i=7U3t=Q!#+%F}~h45RvMHe@S z+C&gbP+p0LO!ZYx$iwx|cBz^b^K0od!r^wmQ(K>BXu)2DBKWzC0-OoA0GNF}kAhP5 z%6|FwXEU0Bh>k(=552cSNF6R0?s-FQV2FLkM|-`3_Q!xk!1O=PzRO zS8kkTwm-Z5vd|+hVyn>CPinv^Q6QH}eQlt4{2=O+cl8uzN4}+$qpq*7uLcL*jqMYR z;j#24^4f0q5;z06u%cHJ6tuu8=ODjul3cG2r5^xrOsx#oHWqzV+4WL;SAu(*cc4iw zNpyhfjas03T_B(e&NDFN42-NG0_asvc>Ew075=gu-Kf(HFL{k!56*14w)uX<9(+f-kOiq3D5<0l*8r>f@nZhXQY*1@zY!}oufSy;!&$KAL4 zlhH)Nf%>LY@_YjpmQZ*AeX7cHXSLgt`$sJ8z6H+~3j%j=>D~O4JcZYyZX~}=1F?Uh zBPDg~hed>wD=X$m#MQ6(Ey4cM8x5Rqb*Ih1NLI!PgFCUHB#Gr+t(?-Fe-Gcy#cC<3&pZ=Bj0H>)KHv+vKg4^r#xU6LB8-hs4jZ*7^e?g~o#}!VIx8 zJ$&nzoK~NgZMNquv&of^(a9Vv6bdBC3+5tU^daNVv4W+9FXNRx82o#vLbyOLmc!E& z7$(bP>1sePwixeQz4w0?Vm~WabXit9-S=qAxdJ|sta?k`sL{b@Z1TJ0M-;pc&@4yI zN|-&DmEqn9YSt{|Uad!-5Hz)JAwKgD7Po=>p+Ak8QhkDF7i#QGRFkq?Yon4R-v>z| zvc#EK`peK0M`ZQhvn$DlEeA4|#3$oM8eI{{cG2^3)ysy8DCJ>k(GIsw)jB9Wv*DCd zm4}Cg6F_%GVJh*aoEvOx;$mr0EHf5GdC4WZXzbRu{f^uoKir?*iXgxIwypX79U$98 z)p+D(heCRSEwd?{&BlJhf|dr3~N zt@yj7zOoVtYo`AMUm1uJc252NLnTH@^ShF|OP6M&;xU)*@N10)G3TNXyBf7NSt!@69lI?oZ7=+K@S(( zGFVRJSK}CW`q=|P1qVae+ExT2i0jvlt!O^s5c6*jrasAqJOT)F6k5wo?l<gDjY{QD`CD9F45xQzcBYvtdE(J! zG;@O5nYfO2J04%-;=wXA+R2abV%oBk8`E1&Co&56m7s%?Sg_XmXNbJW5xf}tW&-u(d_q41^$Gu0AB+7*m2&~XwCqkKYd9#VS ze|DeZ1(RJ*u1Y$~lANq>C9TT!1DykZ_ggd${1@%^ba4q>9xcfb%mh>0eL6-L?^W9Z zB$11OlE*jX607F+k&_^k>8O=SV5v16DytA*qyu5bpN}ij$pS};a}8;wj&2D&PW>6A zwAJ6V;f@jtRbC94xH#_o#KptML@4{JF>@s-t}n_)>FLNN$y&dUSxjmF zds=Az^ib)u@|+zK@r=x`@~A-@s7{S zwO%R65m+K9sjC}%O%&V;lTRK9&g6@}z3#hVqb1j3l{iH)3?&kc$p85~gtxAMKuBp9O)lo^ki_Tm z%7JHnoAgO2X`s2#ea@XCjzv_5C=V-Os-;=C>PygZd9$|3u!OMlq6Dr_dCj=J&kr+A zozoq>=r32*DLK(JP}wR~y5Ly&+wo7K_PDcFw?BO|!C`5vFB$de-1doRH|wJ*-45Q~ zDXOZo`S*CdzoRmQo)%@--%bG2Qe}6XD8IXFnGzpQcohzzNyZfo7z`Q{b$riDxB!?* zfsKEl&qWch@n?;IfB?1snSlmX0bi^ZF~K_KyU2N$u0<~ig){TOV`%VTQ#_EHsi_9+ET~fox7cVTjmJ~45?(B-9D2h^y^u}b4 zAd)K0fY8Asvt`Q`+EnCy9Uu3emKIbpPH3Ge@&1ppzvTFEF2~4n-1-t&j1t{jY1fNy z58YcSF-M8_BkPGVJ~4k;k%dQslBD7ySCv{mfbtCbbcHt~-D7j!;v;>!@?n5;9L?%d z6h%=KV+IBWips+f~zDFP2?2Z*nPng*y>FnSuI1}9L z>iKBPit2OxM6c;M(rJ{*hxEIeJEzGgiZV0VzJ2>9Yr)wmUX(<--0N5HDaD6fZRKT0S6IUGZLBQpYFe>XMcf5xBef`B;JU5fwhw zWAFYBvv;2ckxzCgJ3!4PMNwuRB79gTqy(=D79U((gqJX9AF1p~;V40SifGK47(bg@ zO4Vm?74_HfF0M#TdP|QBZWhrKxA^!U&WQ05Js|BWd~?#g^ymUa4|8ICj8auPI@HoL z71+D4!zhX}GpVnyceQ(%5IwwK*)F0-+EKvAe7Eyhc9&#m&*+%hRN)h(jYXxQ!#*3{ zcZg5X1yEt}@pXJ$Y>%(y6Ip)nOA^@Ztvzm=RBd)QQd5%;Yx&4Ff}1s~tGZ~fuBn5y zh}wTc`m<7$nFujHyjvx+*&jfS1X!U! zCC0~Bnsh~%0P&L&<3kOfNQ`0eLHyWX_BFe9?fR3Zrly;XQWYM3{5h@IJO!w4s52Lx zx4|fiGV^%$*=Nss=R4oIBdpjAylchq5L$hNAh6l32d=Ie8Rs)AYiGW=taLsUKv;R? z3l54HqCnlU)bsJCIK)q$eq9_J*71?*%~r|Q_`qg2Cb2#udeTzel*+Mu*&_4u?gK`d z+^krtS&oV_Q=yIzb1XjLcHb!>inHb2%)6Gxq-=<1W(O3dwa*k1g-?EDMMk&T1eTys z^stOI$9}m*t~2pNhTG)4ppFj$C}#ywUSfJ^S;3qN&asl!N|ji$sMTc*9q#CK?i6Km zp;=vuq5#Q45;u!sA&jf_Nk;Bnjp*63WlIh3+$euNU)0RgepsBIJtX{d;D2k#e zlM@j=661sQXWhDW&a&f0kDL1%d-2nY!dT1W*S+Qo8#L~pJb6bJ`zO5cFE)l zix*;!V~(p!&iBW=b?@@ZNt05k+>8LCVzUSzt=z0AilWS9g38VL@I4#2x}5llXLg0+ zClo>cJ+H1Vh_!)_M}eE5>ucms7DePE&q!6(*EHyKbnv8u# zg^hxi712idV&jqH@70l6E5n{C5V+r-sz@b+)#^89N*1VGP)um z#6AgV7HCx=%LHEe;)T(E++AYr@z8PIP_>!r&A7R+_K5hglZ)>mxLKQ&DoJ3~IoY3X z*+KXW44yKIqA1e|LP$c<1LAoJ8i+?0pRiqpbZIw*1fv;gRLE*s;Vdm)=t!`X7=EM} zVN}gI?9UaXw7VY=R-e$?Lz7a8?~!^w#Q0dTGc1iteE~z}uBMO1_*9qvfq_Za{Bl-p ztrg7mxIGj1mt)aE7}eK}e9VW3^QPCHKb)W0{mGgd8|@z-HAYdC$qy|nFepgIG}}Y~ ziPuG}KDXR*OHsFlSd-FxeA`SSOZY@Ct}qP_e6)C!(PeyabBGr|`LL%*Lqmh>(7s{A z20FCoeRfwasNO81Cnpvksot#auIhmZ=;-J&laV=dn#|le%_Fw+JcjPAGzJHfEk0ex zyPfcv>0mWmw6N7EiZb~b7#N6(7K(vEYWN7!r9G@k6Iy-(M%JyFA!Pok7ti^JFy2Sf zP)+fIkw!BP}Zc(PPC=PAoqmwY9Z*EIvVSvr(!Gy`Ux|v&MN}wY4>)+O8;y zGT8~=vsik%fqksp;&q$~_R5mso$KfT1A%M3kp=hZPZIEk07c z*>z_Rv$}}wVLNd?tP|fuoDbF>gpWJPUb3U3!zk6GrY2_=w$C%Ch9;NWr^Y^Ob#=*) zXwL5&ED^&~lOk}iA~iJBo0(4CtjQ^O6-7yg_h?waPy~=qXCG0O*^8cNop-r(Tre|~ zR5MrHwG^xNSQIz~h@Y@Wm$*%3sC7qCl&Oz>M{auao8P?4z8gpUl`2`)@vJVf z`WWxpn(0-WA?=MI0;UkjoKWF0tO^+->J!0(DrMb@B9{4EM6!RgiXUkdXhlhQc;=Whshc!HrMWs!UUzFqIwmmaHRe;uasQ24r#NM=G5eYf#8^ zbtP144r}@p9UnF-WiARd>?soT?8=Fo3yY7n^f&=#MNgh|c3fRPn=6mIi+kC%YnQsZ z6h%>#k|1=B964r=pXf8aeTD2zEW`qCauPcDBWdoOCeyxPzL^S0lTwx?ojnBO16H3P z@hD0_O0w|GRCiZqaC4Z-?t`0)dUZis+lnyo%KjiC@q??2wD$h~eiz&0yR+wgN_!5A z4{k0IJ!?gtdEc zbK#Se2Bp51j|*-NA+VXt5$6+6XK!zB*E&9mq9{r#2M!%I&pyA)mB2XnoONdP%Eiu_ zGb<&tlkKhZ=a_YCmYI#~R}lJZw(s0$I*xRj$&6%4hoT2CKJad3iudZ0!ownXSohwX z85!u4YVir9zop7!VEu91`BDj_@}@O9vFAj2vOjcl8Lu`GK)AU`VV4Td-rD0TG<%Cr z4)HTMIH-}SilQh=Dp+n_d}+67Y@F=`&#W=j=)%ggd|A6`ADCxe-hIFoH(j-IiJ6RG z@$o${qnKLXoqdGo1;YNC;Nx^%mX{A{QCF8xPQUIHArv`2X}W1?(VJ!J?NWz@ALhNZ&mG@ znmtuBUERI+^j^K9loh3szv6#|fPg@j0Z6DqKztE`fPh?phy6Fh2SdL2?}Fe2&~}A@ zK*apdfP~1-!TmQ0>8dI%22nFbc>3=K+EP?O6au0y0rAZk1_BZ=BqJfJ<^_4tg^)um z?MBAjEsu`5PL+dbxu~`GUS4m=d$1qPITlR zp*C)-CxaWohleypqO3FDGjd&l-ENL$1SeL%c9*pDyeu#D9G@QlISbh4-O5Tj6YlYB z?`XfOsd;(26Hqso7yJJ=`tL>3kK6iCkAq3xza2jx1LdC5bpJn>Jnf9hJ2L?VWe+4j zprf37&VfDMyPGk!SY~umIT)<6fd3*BEfhYM73+Xn$|IG8<`VtMEGImf1dmH35##?L zg{s8krCnZ1nMGz@K!PqH6_39aC_acsE-y8xPcE1(5Op6XBSA~{UlizD)tlN}IHnRQ z%H#h?B)**wEjw@^$0$nVk4ywKq&lVml>g5_q)W88AYW0dPL5JiR>y$5U%p?S#3ahk zGGO{)@4k_nP|F1Wp9MjQ+2!NmBed4Y@KMzKrD-SknQTeB?g>xx6#L4zocCRkl~rbN z#S1ROlyv|1_MZrexnHAXqE<3%Da*)81S~QSz6l31``CX&sC*p)C|?}E^V7~{B>opp z^wfEXL zYGpW(!T&ta&mk2ok=oo}_C4H!&Fv6mC7SD#FUS8Xc7{z#b zH#33gieN+xlC&1=#SD$(sG=RwG>V82N6mE;V>p-)E1g200e6%jZEwdIP>^LLTA{Rt zCGRXe*Q1TO1~BhCZvK!kwq2G`MASm0r+&kG#-+uaTOmZ$B?E3 zjr$za1&*gXu$J@QZkB z0PSFSQI_7F6mY=wOZWsUl(~XPc_|x(D&doEn~S+H(0Q;)@Oor6A9W_~N`TwgWq~F> z?1i4x`FpXL4vfW@`7H&JOY?81v+Ui;yRQFDn*IZF9yu8Ysse6I&cg>spEhiz(C0xl zZK|K-fH-Qlof?jNP=%EQ9~P8)dtyTU&q*6pMR@h5LeW*8N``O74wFDJ8sB-zlc`& z#npvvcC=Dwx?S(RmX<)=fd;p?*6x0;d)2xhk2ndB_qB2;Hu;Op9(8PC(foafz-_t(#fuBR>EZagx*utAxM0onP}xJ(%8Q3GHmj0UbZJ>6;}x0@mo`nTXNZRXY8$ZiIM(@pkNZGApHs*h#O&A-9T|5Yl(j-ak8rseLr;y=(im z%h);lPvALfaq@KAC~AIJsGV<0l!V`Z)<`MOcX;tDxIh^WCt`BS)+7H@4ZdXuA!3UD zR6qnd<(eaUS$rA!Us}4mL;E_qxGmA>tbfY-|sjtnOKwcg*3RMc7TSrpRoX6ZXTb0O6_r&FDpJui!zs3<4~%T^py z96un^Aq1_}O}N+UKS~(qrDOs`e7NjZj4qg4jwNvuf5VaZ%`_n=lD4k$IZM++et%=s zt}iZkOC&`j72;1+RN^9f&h#Rp7_h)_`i_}j==$&fhl{h&_2b8B?k}$Ez)%PY8QAHF zkY)@F1vLpiMrbNAKUrgPYw3N2qypU2%5k_cWD%2nD)jnp7y4c66;V8l7*#YmhhE@0G{mhmu@7O>c+ zNW($Il47o69qv}n2u4U#T9PB)${!9rBLlJ+vG4LBhDVjq-4R#WVntZDe0=e?MIe!( zXce!ET5weFI=QK9{PIxzG-5sGI=C5Wi+SUy=3V$eG8%%gn6EM|)tyrCcoyhgE`W7O zr4<-#X>2AqcongSqOa8Sv@K}_7l87S%@XPJeG~K4_KS+kvVWOZI-4GKq!c!&Y%EIg zM+ZAfxcJ*7`_#KpHeC-B2?VSmQlm%hROk1JB*L71&%-oLvCG9EvwoDfT5adbKbl=< z>Qf%(%QgB;4gen!AB{gcX+%PPt;SS@VSI?v0I%|^`sCE%<(`j+4DBbNZraR9DcdQe z$RtzO{oF6l>#cW>T|I$7K)sV*Pk`WP9>27|7|1*-o11+CifFe#c z5kD{G4M1+z9&0V`_Y6b97ql0~V(y$hCgIqNBj*Q+)ly7SWMJ_*clh0Z6P)Fu>RtZB z}pAR^7AO!XHauI=*-1va540oCjSWkHSew0%a)Kn(pSi%D2$G5^Sgmfrz~Y z^&gZhge`V(jMmY9qkUy+pN0P6YZa6;ZR6tBEM_@;L@H8Q*U5p4*ekr)(-Y-SV#CLG zKk(nY1KWE($rGpo+mXN4&wRln>F;HZLW+WZ-x+z$GinW1L~&qq4DK|OABb(--xI`e zCRklYo5U&Rxu5oWH~Qu4OOzpuG(1WGqO()Z(SfyL^u4;!a6)PNJ0kqyjP{%Q!b`-9 z^uhZ!#&Dx|XM279mu{32EZuas|LevxJOOdl+PKht0V?B4tMfj|sxPL85{&+0BJm_a4cLX90th(eTkql z(cVTCzUw9sW-IWK-@7z2QHB=ng^z6y_wEPZG=<^BSpZ9oNGEGx9k6Lm@DzKjA?lz+ zva;Xb(#<>Z;dyPQFjIEgE?2Ih3Ma-67~88eEW);#i1yNtm1aX3?z2vmN?sdFW= z**y5y2Yw+~E}WKnHO=UlCIr+nkal)gAGO;F1n-GV*pV+4kXWymo!pq(v-d9Xg&R4f_f3m$=KP zSIvauyU@yW{}6k!f!*Kp9_Qf5DvvJ8H6%W{feIbUT;eiO1B)MWxIdtIRnX^a64Nv= zUUd_e7COim$FeaqUaRJfM3sL}NrlxwN1(Xt_oSR-ss|Xjy)e34f6oi7Mp4g|?|3py z&RKDV7fcZl@mk4?U5jPYG#igR9iJpAn~{p6AWDwMtIR z8!2ld9*7iEqy%+VVStvS>)6$l8UMF$^_5w}cvu9bo&wfe+Dqh%)1Y7?$+-~37qdl) z9($Pt3S^#fl7h`ce

tyX2e0v_>z%DpCx{GI`^XeoyJ$og@j-CImumv68BG9ZdI z$>|3hC@p`cE$O1y#Lgxq-U{^d*oJoB(n43(N>h$Y_uqXScT12GqIhVh&U=uy{vLkC_mn027g&sqnE#Gy|}pABp*$@?JRvCCFTQj#pH2 zliY!%Vyy|m1apXb`#H(T$t1~n$3jnSitVQRM7_6|`Oky0a%dmj@e6|h6{4&Pa$a|5 z_iU-bY1pcxj-X`5I_g;ly7%4aY09|aYeVXD9gNk)K>?gf$61^_zEB#?kT|?k!mlke z5O`;y>d$k_YQ>nP!k0Y@S(zfoCORZ0jX#zQqgi{r%~+<^X<(^Ux%(Qw??r`K%amQ= z%9N2Bjyux&7xAA~x~LztH4BkHIGVT@fiS4B-LJ5U*;1fHPXhR@&yK04*OQh`S%Q6R zp!1B!6hf`5^27pcwmz*kf}ez+W6Pdj4e>?F`_aKk5(Ne4x;Yu63Dn=E06LA0R5;Lp zY?q(aws)XqoX3|v+#S#(AxT*d`ybnl#gcaMWL=z)r!w!!> z6hetEiO)L`w&$K`p%5`p@oBLOVFU`D`&l@;!*NfWp4$m3YlmYA6hf5Y&ryq2h>IDN zg?>z`_?h$-!oY zFW?EUs$$k8pZQ(-}G6amT`VsEtITug) zAw84?mB?JFKBDP=B-KmD4OA6I(|pu*hH$#@RwpVchgc~pYm&dp%Rh|Lnq+0l75q(D1Oh%?vp^}sWi1ab)$#l_OPNHpuW*xePNVl34l0ZpE zO?rq9J(NUL6d6rPu{JC&6Qctkk4MJa#V3EFQ$}&&2KX6mPbK02aOu2MWQMb7HNFYI z7nAj&-TWO5CyYiy;PLn`Rs45v_eZ#O*dWAyBZx6=9>xmhpQoW9OJ;8Dj(1O|zDW~9_h9MP zi(ED(Dylf$khd+vzMT1HJk$P2{x=#@6;*(YQ&wLn+P9M3P6DR+pEjXHQ7f$q zeaw|Z;QEwL?W(U{V$_Ib+eI0<4P*-vT{%?d;~2k9c2d66hdD}u7eU@&?Vp_v;SEklzIPX|#Zac$LtH`$%lwAX&LG)F$%gQ5Nr`xq=h~v#&`viW5m`x=^R(soFMn6om2ODm)Enb14WEuREGFT zgc4KIIBM{JN=*?2_fdFk?s2O{xIfrb!gZy$mq*gbj`RKPsH~ErB$<686*xQObDkN5v>kr*b1)CRy(Q(y#_(_H|`EXm|TKsc=a7C9~5LLe; zO@*Y$p45~GoP`*#6-mg;=ZsnjGqUS|I#nIF;Wy~r9n^CRtuF!Ru&;Y$bNU`jbwJOw zjC9ey%%zH)VKvlnY5|^wWV*%OW+WT7zaD%qm7P|qSfDB@Ms=`b1jBG%z6QbxAB)M( zjJvAlha#*55mzd(mvj>pvuIf!TlW?Q`bI8YR|_J7KOnQEaAY)Vj%Nu48IU2^MF9)9 zN+(}|QF4*+U?;)g8`ZNpv=5&W`&dfC?E9N;H4?mnz26IxXe1v#RA4e|av7D%WW0c3 zbtTe2CiqRA-49E;i2Z}FE{{oZ6AIGYTOv46+{*`jf|V_lyO5A;4VKf=)q@ln!f3(m zuBRG@#O>4Pi6s8jax~bM4Ds0lcFz}k$8p{p;qI^^7U(pQnymE%bh|M z#rzUMU19KYi+og!lz^C|IO{s_D0xVZ5G?;}5N?<80*(=b9l=u}R7YauH*hLZ149CB zAjhATAcG%pSv817d;y~HI+4jJ6a!KVo}%jEkwA^6h4>EQUU%(vDyU$->9a@?hA!o; z4zMY4lZ~7($q$Eo#>4pok)^a${Osq~lqMAt?O6B|x6=%;;ObW3?S|V*`ZFNltxj|O zFJM58EQt<2`sPP{)~<-<;3VT&|05lp?yS_z(M>E5JM#_Mkn_bs!#O&P4H?UsWvH;JyxR9~ zm!Yxvk>+It6jJVn<5KQ3)Ei@J?!`mP`2jbRDe*uX5!A$l9Wg`uRJvzL-HZTzBFRWG zjWjq+l4qUROi6w7dZcXlGs`81u7$**aUeaWPVc{}A)>nN5FyW^|cN3m$DuZX-+sRW*EG)BK6 z3hQ){#`l4m<+KigvT0a|B<0t{;7OZ2XQi}pN%Wt2TyGM0S`T)WYx>nfrr_CW#f+9p zhqCim=0G+S6o$%=gN17ot~5t!N`Cg@0@kqzN`ho)88HeC-Xmaw!+n+J8|Tl-OmxsZ z&Ke(it*TQ#(x+|Dqpkm(RhERbVo$h4KN9K`l(M;sDrBK}eS=ZoPwzK+G2Oj&MgsMl!(SASo@{M{% zJHqX*Kv96r>8boxknD5qS!EbFFlK6nmMkm@@ACz^U_tb}KDwPL3_+lbSRaRt@j>24 zEz^ns0a>Cw-aYTI_lG7)0VtD4#@_l@@C%aACwI%5W9nM?R9S_iuppmj zYaDJC%7XrcFFO1>rVVCryQ6nHE8XI@!Re@%#LE2j>a; zx>VUrv^Z`Xym?%#Kw8Un+b-AmZFl>F`$FJ!=az2I{50A;K@H$HZ1|!-hJXpZ-1@ws zyYG~q$E&ji-S2T(fO1a!Do5Aq%Z zG))GFc;Zk-^nL4oF~qOT;%Gkx>?{xNyz>dKH``Ct@-x<;)9me|f7q>E>Ajc|x?8h+ zxSubzIjtb$NbBtrzs(>!PxwW7>5LC{0=N&rK0PVydkfx@yk`EaDB4V=(*mL6qXG@` z_cSV48e1``-1szoe2dPZdx(d+OKtZ>=RZH(^L8Bq6`GTjh1|hSJu~cI7vo~t?axkX zu$`Ah2L2-%X?xb4H;4&vxrA<`l&XF93AViw@L(1c@9iV%B))XycbGP;+C|C57x5wM zJ>B1Dc6SS~>ex(Uaa4d(p3Y={W<`vOQD`Beb3b=O?{x9Ji;KYmRPx^;7R@~vK(ByO~JvcyRk;C zX|{X->g`FNwr~+H-R};{_JVeVMaH{3smpl{*+DDsQ^XI>%mq`
Lt%pZI5&=`a?juPEa7@CMw2MwMu>eU_18N)>R_BH2=_+XDCLl#=0aXcIsTnM z4F_cL@Dl_;fB{(-wnXBCS&Z~aS!aAoK_%gLH;ef*@4vPNDZ%$ar2U(+r1;1AfdR~; z@#@R1kI@vv^z_=I{%v*VO+f!kxLsjXyVO5G*ScGm6B@F2jON&3Sq~9X!duAGJ{Gvb z;O$sE_OD_gBvZQ8b6l7k3=ZdVQS{qiA!3N*R)`2t&r_psvrmz>jHg zIrp#9s=#dn!oDF+8PuuSsTr=>QaY#Jj_^lXFhp{aM44{!NQwG!pPbw1)Ys4K5^wZ$ zUpf`*=e=4(OkTvZ5;h*@Vvf!TxCzKHR5VHRg1u_tV`*83lZ@k!so zU8M9E+i#KnbP;=1FVUtt%X+Q~YG26isP2ZD1)jSe*KAz}`(E1_*qCMe_qySm)I>4} z9;gt?OZou>HfWv7Ea#AL;sYMCh>&763aQ{4xEQ z5k8Inu_Bd817JIv%-Vlj4<2a?xiDhsS2;om9l?m$z<@JVm%`Qaz3TZ4Snd{;vgow;q& zRnW|H!dvnZA)944l0kl1Z3=O>@PZhn5T+uQF~n*9o1jK z(5qZwA1M1QLPEb1C)uugpSQoAx~cEYYsATbihxaxkytcQac;}V;%MY3Rzre$DM+{Y zh9L#aoNlcz=dvD0?d&N}4=NrXE@s+9>3V!tWoM_^VhNOsI>@2pzM-MUOfERMEa7D6)DZJj@M1`OhFRO&3y*6W z&XyxtaIClx)E`+hFQ5focdfRP4k+|B+0AB;>^#f7KZ6ee`+byeSrtP;d6{;x+y=FT zi{|W4m~xkP1(1Uk8C@MOpIXyTs9p#jEY#U1W==CfxgO^C==W)^n^kiyyhWFOs}A&0 z-~ZR+tVV>wfDVG!Mi6_Tgw`{`nT|-mzpb#f^Y~RV#5Jvkv>Y1D+22*j5EWvDrtpoDbA=znqEv zqj0{MwyL4J{}`u0S9iBQjqjAUMh#HoOzK-YCF~ln3IZbGbMRNh3;3|I@o`E=i~M8( z^ee~--f%`2UA(!`OvX!9(I*Nm3Us@&>2xZR?@Y(xnBzFt?H2-aO?=)CM2_H;GQyQf z%)1-t`@KKI97wKcDF$|hdLaCXqrH;ifr!V{>X`Qfe#X6NMD=QQ1$6N0C~#S^?z8}n zC^wbde#_mvo$)B(kJ7kTO2I;80UmjQe=oMPrB=;ImVUZU)9Kg3 z`N4kq<_3$e`3gf%`O`#CqCO&CXvg*~n-&4*EddilF!iQU_2&bk9v81mzoQXC;cERO zDEM24ogdB?6*r;hpej&{8F38#oChjL^FGsCIuF;=?>na{=Tc^1tIaMaz3-0vCm)Zu z>%K3lC*MOfGxED4IKJ#{u+|{*1cE7XdEhXMC8Xs%#8ip@)MK!>Sk379S>r_cEMimf zsu{~HVo?bs(kZD3~X6&ge8J#^J%vekA?Z?_$5BchfQZZxP66+a+hh=2-kMkO#YS9 z68=2u2%w?*Xt%FpNSmt$W?j)Q>>s8jRKy2=TTTnN@NnKkx0JG^S1ruFp&lsK9jMQL z^hp0We1E!9)DHr!c7_;nnhTD-Bs;V=9}9AQP*Vwz3MhM%DXdJ*Xuxr?8$S2rp*YP)GwlEvh#-tQdHA4<0!%CtQt#5$s9PfkXdnmMX zshxwWzJ(5g3P*vW;4h~y-kiiQQJgI%T?mGWjJv3%hx?Lsjb{B@Fo>2rUww9WqLxjG zBh$=lm<;rDs)Mcs83OJHUe9eMMUtVW)GGMg&#Q0;iSrLFC(b3qyz{>jq<9Fr*G+6% z?3qm48P+eTxhnA^E$&AYtxYn{xKeRkeOw1$BGhlIgezhhcA{CXB+nNZ?Vm&_*TFnv zTw$*_B;+)pwsBj$JhTwu63%}-LUF?F*^?;JMakpHBh`AW6?4f`%AuM>RjT2`TdauY zR$!K<5M0o8B9OxM^|Iv$NcSL&z1c_&sXs(pm8ahcx03Of)b+n|fH3N1ZMnO7T-NVj zb2ck9lyzq?3Uk}%$~FJhrjeiiUG#NtC8j(6QLtuH1{7ku0%M($9VJnKxFiwb0Mf%( z@vgd=t7a5zh!K_zV6htKp*%(3ZM09gE%fL8F@l`lUz=1}og7INGJsDeCC%gKUzjon z^_34ZdVjU;`)%XhMDTQeMz<3>3Q^kiee|lzXhiIM1RbB>c9ezc+Rs*At=`;p5P9I# zW=7H>?SklJKCNkz&>d=z5W}7$02Jz4k!&m&bWbWU^nOnlkRgkBb#iwr9JSJv z$ng}~C(DyE=>hNNxRs9Bn#poPzRh}s9-4SsWFIq{ zH2fCmWig2ELp&@-qd-Gh1P*6%8B>~RV1Pl&=vT3$v!4$KaIzU(YiXbTp3^wTHl(h> z?{^kLqM4Lrqi?Q^lxWFnYv-`%l1O_xLndBK`M>bABO6E$Tz;Enc~RDhez16rzXk8m zA%yeS>*3xwDXLZ?P^^V9WT-_w^SRiN8e%gdGmO(AKm`C29{T?I8q9#J*j(f;CN^wY zEi5JR%k?43HfiLi)$`MDQhB7~eDSOEDp45(5KSaC!dR;B=PfEUuByTYRNE~j8kY+% zmB7i<4vOx-^ZFx)LpRvsINr7<4yrc~?#G;{S9SKpP{e~)^}+EaHJNb^3k%E5UGrue z*EFYpIW~1K!NT+QLl}}=RG6KQ7~Y(R;8xE!-9rbO8A_zM4(C?9n>x&U0fHDGP2G9&=X zfdLfyz@9DSIMh2T0K_zTP1KPQmxVUV)(~(YbtxMMR3e!jPl<|G!k7g#l7x!mslcYk zaS>L(y2~!sQk(DxKbN>gNGKA@RkG$f&c<8a=;GHd!(+wJtqV&olDGfydPT`VHKxE& z9ZH2Y2A0yU&1cs)#)tlfKi>QrsG3_vNhHZvs3+MqAD8?E%oLhOwFyHdi_69fY2pbZ=MQCV&Br_ui>UsANH^84DUJ$BdQ%bC~r5hVRAlb z^G$-;9UF@Mlf%rb(A)qj41=jl`p%e43=t~*x6_zr^+1!PCM>i-8P7{b)cOW${k}3z zOko)4=_d9u@zLfZAY_VJ{Jadjt<(dR+SvJwDbe`j` zA{cJB)0#rM)sCXVQS+ivFd|W^QPr9n(kyVksg^;*9)DBtK>*x7Oud-<|2CC=NlpE& zm{mhEs_$=PWb`(Pn2!vr(CXZru2g0fIEtE=tLE;tFHfgVXVJ4Wkv;vda>%S9sKa4Z z#TxJ!ctNu##Z-mRWHQzb7Dru284u@6^u9gl3mXv$wVZxD$JNtyr`e*r{LKKsdw?FsCMdS*{2Ef>B&wU!U1z4$D7*?l;?) z6|N1DU!4XFxaVQGJod@=&n)0#89y;Zcx!IwzpsplSZoH@G#SHF%0x3pc)E(c!x50= zy0RyJ*bSmiWN{E+7>QA!R=}a?^~sRqTl<1)P~nM`U)uO{v6)x^m7#{Q)%Q6THaoyL zI@?-tD$Jg>AEZgGsCTj2AhYl^a77OlB>;^pnXndR7bhm$exo%hM3?#kcOwbBui4X+ z;Np7vnsd4byD^wyFXsaiI?p(Mp(pX)%V>)gue3AkHp{!6loXnQMJiFZFxz1zzf5in zwJogmQ!Q+DkWkOAdZbuSTsIIEm=_bPcktr8r7lr7&`Fd#jk9aO{iZF|sg&70X^9e8 z`_y}vIEPE8K5HgwV8~^{;1Fx|3Ti92bQG`-F@wCmMyR9{yydE7#GZ>!Y*!HPin*Z4 zwma{Pdb@cid+tuGUF`g%Ldf#*NO}2y%2^GgSkO(kJN)1^Prv7|IWs>|NgiQ%XEh5o#d7!pNzDR zqNnFGJyJj+jvV1y$T0**3$(H|#A`(e8F0g^zW2ow3MV55u(C5EJ@Wh<9J3szddM;y zKAbo_%aiJ)q|j@iWciJAO4h@nEEV7#_d;D!{B@Vfc>m`sAjN!u6FEy8WueKDW~kv% z1JO#;>Isn2%uES0EawrMk22jziaqRaD0G~0`dW{B8^E!%oe}b3b+EQ$Bws-q85x+a zMuue{;vEh>AWWY@a{x6R_7Kx<`s=XaXBWa8wj2{~!JeymFhOL7pGES0FPEecT&oIb zf4z1c4*oFljOi^?Sa$AtYl7=kT~~YOecEreY~wYdHPJJXh)`wv1T(3|o}DSs%aXb6 zAelIk_`*3%J#&`&Pjm+km!PtJ@QG6bW6t$PTf;K2exrAmk}{Uvu7hC>YmIl9s8K&b zYz1^6tA&HHZj5o$=YRL5zmPNMG1 zpYv2EOm>@Me;B?mNTp~=6U>21C>uVaZ!Z~-kS;-%v#WJUxuEl;kK$DEfUaGk%~|IO z$RPDoO=b5dUM%%zPiIro*0ahg5jVXu>-np`R5fAATFNL0EdyhmK2gSuT*e8q63%M4 zMQRsWPc13!wF?W-sUml=GYTuRnO*$#1<$yaR=7|q~eI)Q5 z17Fvbykz2Z6*3MD*zH;k0Rf$EKr7~YhH}d`Zj^b==?pFou6byIhm3=eOrra|5?G zi|EeJ=G=oko(@iLzjd4Xc+%P3SVjk?mvPN*jQPkIJiU)xSO24blrrd_6!(6H=)L3J z=xr4S5t}kfslT-!;}Wt_rrZF$1wU4Gx4%7RVdNWIlbDY|H%9SlUf$$?klHAaaG`px zY3^~XcYN$qXGOMeTt<1`BbCH2h$<{}(3I zjM(si0A{=vI3&_1|6R^U=iU1Cm%QyA#=Bj_Tb`$KW6m+Rohd<`V*dG~ZZ`P@@V)gs zNtA@jpGnmHP=ff$4hEWuzE^qiD{G2z9YCZOv4Pwt+w^2iHR@7_6Z4|y#YSUcxuKAf zc7{+~tohP%JxXSLC(Xp9%PLPcsIB6~^Vnqh)M%3hSeO@A?b=Lrq`6>SA;>yit)y-F zQa-`3p6jq)Ut`Wc2c88lO{+BXqUR${bC~-sKTI`~Z_!cY0Ub;kABxuePKi=aPan(I z#OQi{+`LDK|2RPKw)5$czx`h%Z zpPmnS2$J6D@sF&(q+jrN1SjiMO*JHcQ3081!LtiIbdqUUO?Jj5AP(RegZ5 zGuhM#RarQ;v2diroYqNLjRw@cmHktH=d%~-S?IGXrZZ~(8X9}OY-i6?*5eb*%Fw&C zPOpgD=Zm5^qTYOHie*Q%6IWGNYGnv1=hjDSo8S$eL2Q}8Xv3?+2XT|#cG zW5}_fYUx-8Y!c_Z4n=rf7V{*I&ye(1TyW@UtgFtH^QOdvIVTA=^=36Z?AR3o1K-a- zt#9{+wi}qQfvaxEF#^(XjHkrsZe}zT{P{CG!2Lw$T^i5%H0RJr$webf*O;^)M^qXf zt8HhtrwQhX-j?0xb5j(dRNth_6~)XG7Ryp^Mqle*7Qt~P^!fI&JA<$-%3Z^Grgo%p zX%uS0?GA24=a7wOtDzwB`KTleeQEU(zPW;RJzDwl4Yr=)(EJvme2446Na!)I8=+BO zv1sc~=!sa^o(H*eqEqd5yYRMC#B~KRF?doz2x=i{YBruR)5Yq;C@Dwc)da-RFYD@& z@gqq@#w!d1*uJ=sq*6xS@FU32Ry5G$#hzKsFIVB}`gm=j-NJmn5LHzl`t)n>`uSG1 z>>A$o&WP?zoy-X@oo90LFUbCFV?1KcBO78+f8v?bw{hKgEBEGaH8v;e&zPOPYtCuX zuV)}=@1iL9TzDbE-p>4%(i*4#icY+tWS&c6Oa(j8RH#q$SM3kqXuhN68dD zy|hQ{*xKM0>3LH)j&nw`T0qNH1)}>HKNN;-wDpB4TekVSWpo&t9${NpJBzYU zef?PsCoJQ`QOOs%@|!^Xq)K@kh>ujoX9Z5l^%)JMt!915wWRGS@RNDc%B$L|8m9od z^)amztxK2n0juG}7Z?pG*%;Xot z7~Jl?AeL5L(r#J-YW)Q5(I4rsx%F;B?Se12NnE}+Jgraw7Q#nRcvaoHsFX=LNW~%I zd!DQ4CpoXREMV<(P_@Kh=0ct^N*UOCY`j=wuyHcLm-HtxihiKWHJP9CI5X#%?y>Za zoDS*5BthQ08n`BpCYUx=$n1|iuhcBrFT_n{gO5f9$?rH}u)2IdOrfR;{ zp;T2i%w5$%*36@XB}Xe~tA_yeNA$Ua7X2$UhlJM@+0~XjU<(xQe#8Bqfk3Z%nk}t( zZxn~9L`ybm($%c1YB$0+b5YaCLrRl(Yr4lXq6zJ*OWZchVJc>HTAOE52K& zuTs6GaAq*AkAG14PmxFENAsZpF+wCaBdZ^gUQS9cY0mn0@$-|0pF`9%?y_@vbjSd| zT7MV|Qf!B7?`dC`t1cPtZJ~$~s?FwGXI}W~LyFBL?((KL#I&&{-;1_}xnaS!;e_N9 zqfK<0+uL7|76JTWCU@dBRpdCgd)Z_>Q~AQe0H1qso&T%+-N|gfe)ltD=*|sxN-mEU zh~VX=AN;Z3@iz%GLYEH%uSJnL9Zt+szQ9%RgFe3*3!#L0x{0H(=B>`F>a3GUmC$yC zj9bCSRb}hXOpTZ1jgUrlueeZxLytIn`ilCzYCOcbE?v6dRR1pfGmHc|mw$+) zD=;MGQSWsrqI%qRCJ6&z_xtK6hXsSz<}Yks0`g+?FKTwr>9CNpne7m#eGKB&8?lE( zHx2uqs1|0gx7*--MU=g+v*<=RD;Zj5YOA!ej6;*fBfr5EQ;Cg^36furD?3wX43wI7 zOSbpDLE3-ZbPi%{GiG?sI=0`hW|hkZ&jmH=8}F_<6z)dRkC$4klC&SsG=FWaHpGp- z-rSxt>++r*$MOak@%#ki5V&pu3D|9|)Q_cC`kYHkdQrxSrkg%mNDC%D5gp7o?+TeVAoqkj<4&n+B{!F?%R^ zPV!BYVUSH=v@cnQUJcE`gAq&mSlq!Gk;oc#Zk6N8()eOfhO|m`&rQDkmj&PRJmz8# zfw|3Ui``Y@8c%QMbI2w7%Ut5OYUo8;opsOD`|la<<9h;K#rn8NivrHWZj@y3t@~R%}uXwtZwFMscWT6xDByW9~wtD)C4w|(WOz|w> z4Zk&Ea$(n04oup@)~sY0pOjr|mZ+!Yw>K91rk%-*P)w+5da9dSbg3K)gjM(+sUHIWl?>de(s-o~(VpsBMAYdCE8gn7YrF|V^OsMCWq@tkRz9kDG8VSj^|D!2l@9*u71ZkmLV@@nL&+mS(rSoLf)-C0W3TMqN} zi2vf|?;yTZ6anqunN4<&Ab0M#t;F4eLDy|Pk*53I5%S89hu7CUoxINv%tLe8zsFvy zua{2F^8;1#M0kii85DAlXWAqH#s4;MFf!O$x)V4s^wyM8w&GnbuU;9M*n@U|CPV_I&bTOe@KXmY7>#H+ECd9OgRFf8tl@5^m^>CLts2yil) zj(!rq9m-P?>-RD`98X)JV0yA)vaI>C3#+DgUj$HEUN-?WN zS~@FW4_~)mWfBF}UZUH+emlPlx_2U>Y9k|m+UwimU`e*ouq$82)b>}?I0RU#wLS}S zkAI9mPFIyyr(j=_?vWG4)%6S16p?+CYS!gK?#)cs(3?!3G9k~2Ij;=DQt!@iKX>lu3+^xmk9f}kRRD!!xT!IrEid%6>p*R#T z?rtGyaVP``Zo&2Cd%v?**2=vnbM~G+GyBP()s>o>l5&r8$JkO$%GS+!Jq4?ad*z>2 z4@eu*GPzQF%ldZ&bMK7W_OhMqhdAG-bfsA*@dhQUdqKwO zWOsOYyZ|4p+n6FUH=H>eP9qx4IjPL*ayd~}hVr|mgrcPP%&*1i08ri5$lif-_P&9A zsjqbW?Sk!b9aObVGp~4`^S5A2hCR0#J5k8v)bZNH=nR6&h5m+ zNp8-KL1`l=&6eO`R0nN`tu)etqqabExxT6-@P^TK|JSR^$|e438kyQAr3uoEjf6&- zW~d{F>F`U{goj|vfSW$L!A4?kYG#R?I&QCARtjlDHz2N3(Mv#3|C*e#4;&pe;1` z9yD^kCwTFwm3f|pDx+i?iw~^|sw#1PD5B&l8N)0zBt? zl=lvaYaf-YHi}c6$ULg@6OS6~QjC?I%uG$NneG3ou7(O#l6CC9|A@)sGVvbQ?AQ%y zbrf#aWb0;esT*0V;Fxqx$rNJ#Ld;_!2f|eG(~@mg7xIC zc^>iA<@ZOr6Rh)6UrL=^)7+Y2pqeh-jp;o23HS}Z@LHYEuo{^J?8MnTf2#?dx-F-C zE3X{4Fb}llosEdg_f|j);5=IB(8AzB6wlaXF--ZF*hOjyWrU+Ax7xg+6x{MT1ij5A z=bLsNWF`W_eS?0Fs=dukG@(&ysx61JVr-Y6M^p(#eZQ@UsUPj&Hf{w-5$yd%!e~XQ zW#G@{999vRlmTvGc79l-KFEOPM zeARe}&3QFfs=qs0{npP-&1Yl{cm85SU%FWnR*YmW1V#N~I0OS9V04kTxZ*vmEHcf; z9ln2huCCTj?rk7hz}#^KtDd@MQRB97bO(xrE2?OdR=@y&G>~oYFU`TUBO3wW2`F|j zoql%UB95P{aoGMLgosMZ!tU#KEpebYWK!yg$3tdTdl`>PO?e2}UCz0u(jCeCI9WiH zk9{dAwKqWu=UqRBUz;4abD!+dEbi%mya}5J#igaBwgJh+M1#=?s7x1KFE(;Kcca@( zLqUI96NZ7eETpLEO?{1d8i(~h>gIcT>^zISqSD|i9D!%g$WiGmNi^&W&44{AA!3PfSJJ;dA8cmOIleto-dSs`aZd&Ec z8_wcMfj+;n1+pGuy;d6$)Dm1UTZHa=WAP|wL90qPXOYLV;lQRPMG3RPd$D+y2k*}3 zJ)u8!!W{IIaI?xJ+#r3AbWwebX1W{t=(;b5SO@4vwgDVEj`{^cqf+G~G| zIvcoyz#Uc(f}UP+AyIor6A}g8uRWzeGTc5MN)USKSqvhO{XOmstny;F_37#9Kt9~G zq#H4HGs52acj@-?9H?SKVQ)mx!ldvBR|9xEZ}*3)>i9r=kg8%#>+XC^C{2V$n8arH zmYg`vrQyIN(jH#X9$y)zrL*P~We=o|O9aYr)&u&0g$0YMpAa8Y^?$gDL;sGRet%S!1*$(U25;w9To-U`l{Bcn zqd~Mcpa`r+yK5L=jBJRZB_6u8VRM^f?%{(Ls&Vc*#oTvf1GXEBx;eR3ym?Q+LGN!S zCle*$S66r23AvF~Eb_UkY_0&{O`>af>tCojzW`G3Ec;o zMgx!4e=M*cj(^yHm$#Zr)_PQ|luu7yn;+%pqKXT9drn^|xDKN07-<;I1I{8gqmp7; zg}D(+6Za;-DzK|j#^j}C$1^h*L*0y1=Jx#?zcef?i&ug|o(^!vhP+CY;umVOv6m*%c?V&(4#~ky~O4#?%@PWXHzeFC3*?%0yI^v!9_!nHSb>n zpq)FZjDc6vXe0PlJtRHkNAoHOq&5++bxec)ui3Dl|AxhN#U^HWQf^!)8wNv5$TwTS z{A79Wop{>37ERcBDS|47$CKwrQ=*wYvR21+u0Y4vb4D7A^!Slgyt6pOb4hb98Uk=E zWY0~Ua^N(gFiz*Wss!FAY*(;9PTkFew*FK>7ub%Zf*Nb6Q^_*2}(W!cAA0wth^^By=peeF_uGf^9OZj;gY=BvEc_HS z`11|miGYZJgCdlxS2&wWzsFG!zmob=;qe7X7;aW?;_&_8mRt^$(?PJAx(x8P*f}&h z@lEfE(m$-s`tF-a6~<0lrj9 z<_er#RKt=s0(^{e8mU>H#n(MT0|y}<YgHRWS;d2y5@O!DzJFOlgN?WGRc%!^TxYmkCijTZ>07Z<=Q38J@)75a_Mqi}B zM3|iD4nSJ787Ht(6CQb#=%ctIy&iGNJvMqi1|NIyVf*X%30`;feQ&JDO--cVY&9l< zY)ygoHQNFBd&7Wfw)N_*uP{H}qoK#zSE{t>x=k8$kmZwQo^Ev~_XnTp6Ce7c5&!2* z3^;f@(Xksu_{K38N_!IW`lM^0$7l_8#iZT)jP&aIO-dqfd<~>xJ(-rR|8!ZSWiC5^ zGbF#BSu*D>ORs16BT0R$M?nj}??iZTJ)6Su`(Ep!X4(8^lXMYdc1*(S?c~96qNN&& zJGn&KL@?3@PR;z*pbPZ)NBi=!!Wws&34P_9ydqNk`A+fzMZq=vVOR2Ns$66RHr1%Y zTc}U>KPer1&RL;J$r@IL%43CV%T9A-jXrq(+OIhOK#SJWxWVr&ZLZT{>uTN5v=iG2 zaGM}JuBhVi?>0+Xwdc8B`nv9YY=;H#$cMe-nPC^c@hIU%D6q#0eYMA~8qCR%JwHD$ zGLIbXFlkpCOO2h39$UN1?V3fjytvvQ|I?sc;ABX_V@aBu_V9w!m|D$+5}ESh)fJzx z!5$m(AnHf=As9s^pdn28a1C4G?f9)1Gq$jlcT(*OW)|s2#_IZhcmOLhab<(#z45P! zF@$94=xBI!g0M`+-1~Q*k!=Pl5goa(!V($;|EuB3pWyn=GBJLoWALQZ1-go+>J~)( z(3A!T5H!P$?^9M<1H#fO)2^sK{tL+eP+(L}~DoD17f>KP`p$71C5x44p z`B-n6Vis;K^{k1ASP83oMm`S3nKVo~e;w^;$?5Ve9Dv2G1&J=eSvdBl9y?Z2!007) z2JaHGTaQc1#VQ;fq)^dpIQ4B4L?g4skg+EMRZH!_18p*qtIZc1Jz|&%si)+ey2F`< ziuLARSO4Yle0iPRFPJ@-ooXG7g^j$4@Txmpd-1CG-SinUI=rdqyvH7S4utnmDaCsn zEZN%>jv1l|;TwH4g^~7`-BD@FAU4mYsCR{mTJFDfYZ-?)xpNe7mD*@!LdRH6xO!Z)ye~DScUQPY7baa>JzxVW2 zq4=-Z^zgv!RpHu)-8ap)45nCGp;uCwy6sx-u}%sVb%oJt6Chio(Z##7K}b#uU;*>C zsLWl%j^GFayjrx8Af?#3>kIO^J>OoW@meu+2@4?jYD6!I9NK3goclU>C`6D^SUID^ zO!hC%GRpo0$b)YBW^XeEY_gD5Nf>x;E-Wj?$@m^twY^iQbQ$N(cd!{+h`}MKP-mg4MJ&*g!9y7br<3s9z^N9{dB;_#Ft>woEC`bn$Y~*GX(kT*tT-Z)f6B@dEAK+XnFkb^vHFbpi_9uBi!kE z{)4wONDIxggZw!kwO}!ZaWzxi6Yo`7B1g3Eo+d-`dCBzTVsLiK9m4!C8FnnLFoA)~fV9aKyT=X8f>2ReW2crQZiLPA{qvhG3@3x8ql>*2t5t#) z-cthk{+jE1?|%{Au5s&;Pw;h;Ev#cMGkjV>tW`8tE~EdnZ}WG}M$||jLrOmOvgsXJ z`9js}pXgEe#BcVw?cfTz=^LMt8Bp4O-w)skuUca&+MVhK*Pkz1%$kEqC}(K%wiue+C#N&&I&o34)YjcxnL8!ZNR~d4~L0f{N zTzEf+N9l6uWOAio9>RGr%pHt#1yk`h9po5)IyO7=1--oPpIc>_yD4C3Xy}R1f7yH` zZ_n|s_LHv_W)_v}b;wjy)R)}5bTm=v#l1k!s3L{I8HVPnn8(50hE`D{;KviHah}Is zP2?fqNDU^z^4`PYX$-SzNm(=WpIY2W#>sHQcSqA)Fp{hPB1>+2$vSH=wqAu_m-E=H_w5~qH6b==P3UD>SY(cu%7=VyVFGLY z@X50kYH>c$rF5SlWZWXM^0>xsFk0yZcV;)+s!plyKm(a*CHH+d?C-^f&v#6|*dv3I zE!HA>9j%JTJ024YqGCGEslCV3NST!&$LayB558Mfhbg~%_DH`~O$K}n(U0B!SVVaGd^oCzImpzm!En`Pd6WUp|F^z;e{&Tt(kvo~o9e zEOZ;^&D}adZi1s=n&7LPzad8AH(E3P?j1hTf zM6;MXbOvUApK>|Di)NFP>JUgor19x1g@{gB`+0Y#(L#ki`_c`wx^s6kMRt;bW1_)A zMhuyG=x67XnXk!1RTXU?4z4FZ8IcKe`S0lWDp$p9m!noLjqe@OODzIK?RQwAKY(GdOe%I_({5Q%`?a_~M?|xh4Xc&6& zGITBaRWh5xHVd8>zu5z8kyMK=Y$%EDcw&N!W9u&|sD3)OoIB{vn)W8FxSB`5VYk1# z8>LWYC49B4Mb@|Gv%puQg|!!-k?~>z*s@Ck02P9oYUg-8Z`*VV0&1C>E{jd~-AOY< z-#s1nn%~+0S;o_o9@nuK=p=6%Ikv_b--qM=cTh(Z#o6RSHX($@ria0yQd#s;@-ey_*kz^1>Ik{y76D~_&P6PhW#X4}q1 zyT0l`htS(Skt!jbjt4hn4K*3i&=#L?!4~!XYiL|kzs*dV(a!0{FKfn0TTAQR*dkW5 z{m^pmgcTeG(YTJphvD#)HtAfW^7>}w`C)ESrDSk2DBcMq?#HcbREDApjlB2#huLSX z5+Lhqx^7Wz)}Pa}hM}zDV>sBqQ@GsZeBwP8{WzrbJa={pQ6BdTY@U~_b5<0(T&G*G z0j)pb$9|6XXrHprTD&1=fj1)dJtF1{!VwT>!+W|F=SwjgvFW}8!aWA#yG=C+wCi$R zBCtB}D0Rnvz4=cw#`(*#$vZknm-%?=TlW|9rx=popn=}ceG}kLsNc&6$;a_}mOy^P zw^hi>9u+Y2_cDz%fA7FY`VR7;Vsc%?HDbE{F8PB~tL(y62-#NN1qmT>5v!v<*;k{1 z57EiNnXg}bNk;eS3ZGSe6a9@l6J^fm4Frd$y(UAN;2iCjwZj)V*iK#^Eg9LY0k3UD zDek>TuBdB)#UO9m`^>J6%b$zhlNkn0)O>Kro{*;A$drO#r0V2g=ny2_UHLi$jGW|4 zT6>{3eqQ0Mc6#yo*OOi!J6|cJ^QI^pB32POepkJ|k2&3S#zXcu<6uW?NR}PT4UET*jyl7NsFHwSRYxlN2=A(rG^+0fVdqVPNcQvH`t!m=`hm19$a`I32hMeiLVxQMm5$zV z(o_YUPo$O;QW}A9HLaQ%-xa_mmXWd)V5HA=;EkS|H|THK=rQ4yLJi*wpO$^n^z0H5 zDT7S9#CJD7^E~)NmxfeVZ>YTFd&@B2O^q;SCk$4xg*;psYzo+TJ~u!A-2ae48&m^B zGBRXgj12d*-CUoT0LH`&!IknO&5M|7_An7^5hlywjJo{_%Y$DGoUJ%ibmW}Nh+dbh z>=bkD`O#F4+^r$pY8Y&Dkh6wH!gGsRmN;C-q-)*LqPFlP(Pu1k&g6~uKZ_@)J~{V# zKqRV+$gH4!dcu)CvxNWfR(34$BQ;dJo|#W#48qaI75oV@d)&*Q56N&Ad?7KS)`doJAc1f3C=NR#Pp`Yj!FyHe`GEmOfda*iD07I#XUtoA*T%ywq{dx1z`jltg*?Q%XOMn!W z|Ao0*QR<*91enrvrq^uRnQ|(-*+=&veUgl`@+ecwUOflLZK*lMF?{M(!|6IASD_OZ zRd*|-filv~QyIGhUX#r_An!Enug}!sdF+*rAS7|{V)ooc2cv49X`(1-KrEa4*M>kM zR);}t6FQ5`W{ANUnLumUAJ3=@`kaNfu_QW8@1Og<#zx`&%G+2KLT$w4;xh|AjK4br zuQ$bZR^hl=0|9+uaNWOOsCEi`I(GV?;Dh+w7tA*QQUXn_LHF6L5^lB&M^7i2W%#89 z$Cu~>&b~1HlDm)&NaWLy^7x59kR0V4?K9d4B$el$4~`!4vd?zmxm)I|jTh)TZ?u7) zFXRKxlgw41^cR&ZQwfijRQAU`<)?%;3{pcQLo6&d92H~v&)JC|)GA)rVP%cxCCbYF zV}KaPrtc?;d=JN2EjlCaP$$dRapHEA>hT*46KN)9M%tA3E`F}{{^9+`J$=s3<2aRz zd|mt8feyR(H}iX=kRwjxn>d?<3}dQ$H8KcqXa^qKSx;7GqQ6bmL6CieGmbllkdQSsF~3c`J^u<%d(%wk#+QWn)4pm-MMB|4Sp zm$ARVt{ZSNyFR*^?#EmSFVYR~xO*sOASW6oIqZXqC~4w+b{*Y<3fv zHNgAeXSt#U1h%@l)0{2(-h|!8`CYZT6Q+TU#m3tG?Z9p5Q!VyzK13ah03@D?ZTv&U z#NF}4FZ9Qu?U=|C0ON7jy33@${c=q(0w8?IBUO_1-SFnQV|jd~`j<74G3v+K z`HcUn@LE~&M5U9+cq}Ypl$LJSslW7+nb=28ipR5~jo3`gnO};s1NfVUPqBJa31MP2@N`(dD?uh}dqw%vQIT94p-AHQN^n(12)oN}l?(RF${oI>0A zXmgPbG)Tzk$s70I^>K#;-u4(HVi%4cFFC9;Me?M_%18IXV9j%X-^!Y7m;~Fqj zh`|3nH#Z{e>fp*ysaEmS8;8}_IcDZC6u@4{E1VBb|78nQH=QNzSDM>VZlu{YIpwE^ zvDq7mi_e=7pwbxC+K^d$+)@~`OvA1ZN2NC2)uNG$^i?UN3A9#@jz#r0o6IXx$l&2! zh+-u$OD`vEIfYxhWV+j2XV4vvjvrg}T~3muOq${b_y+#snuhgG7?_@i`i(iMY5)JR zc@kz%&>9qKLI2XknIWc&>?*#8U2O~no})GS)=!h9zWJDkE7-po#L`B+VZ%Js%fIYB ztuWcFr{|s>aDO#&<3>|r*qta*?>7xYDT6AOwxTTTw$GQ>HnSrf)J&gX?&y%Q^l?G^ z-{lAEzYj&|{B66Kq`Lg3sNrxg?hM#58oe4_6FWb-w8DGs$WDV!2kW&nNb+)Q+w$w9oozK6aXTo17mYhQw99ABLD@H6#Jzr_+m(&xE&&OfGMye(dP^d8Pf!`jRcRl) z37Xe=Oq%|Ek zJv=~@;q3KUBe|GCst%KfGg98-WaC-6cIagr8Aot+F#5=sjY$RW^4a{k2yC!h6Bg7` ze?LTW%_9jMPNE~EBP{3`%G%L6S+b8`ZVhh0aJnF%%5U?e6`Idkbk@GoseX3vdA~+U`VDRuS)c|&!VNdM4hJM1k~sIawS;MAvJ zN{a&K!@9D?9^>z$uY&Iw_!Akfr=9-QlEE85hMR*ULobzvV%AmlbJU*kG!QIl33~2% z+}Q{iT0Re)E$`^OxiCY3PZsUxOpZx0JJ@aL#*_3W#2tC#|xQu$JK1RNraICSP8 z6I>)$%hfg;nvk--&U>W!Ll-6{{wnGK*`^p8?CsfQ>_sfZE+>9dV^j?wVx=U1e#P@P zt$jrK3KdegZ_i&`-xPcN0jgcvj8qe?@q+Y0+D{H|nWcm3t~Zh@gwv;KDfK$UaKc>w z1JQd{a~isT5)uhnY*%1jooaI91h`jc*PNt}Lqt71m;vzxLo?(0!Nr?W{O?VR%Q3Fa zGGZ$jwo{@#;TFTt7z$_u3kqZ%siyApQc{62=oW49X~d^DD$W1PIu=R ziG9kwUAO?vzW85_V)nu>D2e+7Olb58~QPaz` z!$E6#1xX2BGI~4 zyXhGKP)C7WIGx@&TAf^SOyqr?aMh+^f9U(z_HvHvZ&i#d$nYqfN4htB{gvFt_lrhO z!s$(CwdJvnJ|(d=Hn?*)=QHqD*n31oSHup~ws`>i%wT%HMZaSBi!`V_RVmWsn*?L% z8`R72{-;Nbk4>mV?pwC^M0~mM)_dTSrjqR3tZJ_Iz?bmQ=pb4tap`1;6H4~B2AfK@ z8lxpt!Rtt7@tRp(us;iAc&-H9LC5}`z8At(t=Lz+w8EcS*{(KF;}Ok6h}|;)ip{9A zX6H}1^j85j>yX7|b1mBWp7+^{q#8NsE}dmB$)Kx_AEY6pmRkISmkUXPj-b!JJ(AZI z%fV6+Ys5obW7Kq;DMFO22S>3tru*M6AIlh8_K)c4W{T8P;?`8lBKMd+ib~@-cAO6j zle~=58kSP^C29?m??!b@)UCF;+4+9;7OWDdfjQve8MsU^$X2cOW? zYxp>=))O4rQD0nN1Cau;%s1oNMyde-Cku2nhtn4ulM%nk8}?(3 zhKP~&czET6bt51Z3+-u|WgCLfDi+WHFHnSh!781ppD?r@M57-ZV>&|J3tNItiWW&F z$=;VS0l#47U-}|vZy!A5%d2$_VZjnz*Fi0a`|GCd>*}W+r4*Oxf(+{S7z~0V4W4Do z(|Mlu=SY)KhMy%^&)3vHR^1Q7I=j=VBthR&%3;fYmO8T5A|CbkI;XK94?B^J z?_u*|9dQ2iVmpzw(Gu*e|FCePaq#0$jv6lhMP_(f|Ak=hrq4LOz9r(johkD|}I^A^3n|@{I@1w?y5L}{&c%b&x(92 z6Nbe+=pmNR(T2nUr=XJS+5%`Z3S}nI!FiBsNP2~UH$2h;UPo!H4`n=)MURK&iiF9VsuFSh}>MF6Yf$J5r|G9o8K52wqh-iTKN$|IwLy=QVBSWiBv@3?gOT zE3UOV&?xcE7OQXK5XP=`dvR1q!=Y0GHEpmuVy^DyQzQ2}T3Tjgs8H+5efYC%J&0)e zE7j!*R^yzQN^4uKIikNZRx=OvYfmfu4$pPaL#;#3xo)GG_`na#R5YKaj@}c2?`i#4 z{D(dSfp$Jt*A?y|UbP^PCP7BM4p&p*E*LUEmgmS_+ZDs(5Ll5HR+ix4J%&wBwyZxA zq3>Unbh>4fH_h#3!y^|}pua%=(O>ti)7R0mLlj|-zH#ou8LNJgKP#0aMBufxwUrNk z{M3B8G=&&jHiNMg-U?4{?fH*$nmZJt9bIN`CUU3^LS0#;t_~dj){(y2!+QPF z2&uQwiLUMo&HSYP$8;kKvWy_vxF}1R4 z{ulQXPoI-%Z`;MozMvez(m@++II0P(QvHu%{Y^pTLm6#x2!n#yx4z}f9x1qktpl)gwe!p zv6FCDB&`f=+6@IXJSl~WZ&IX?#M$^-1REFfDmDu76)bx@f5t9w?_%TovH9>za=UnZA(9S@*8BoK~LbkCs2fNaPQ< ze#}yTrK5%c&k-GUUmdby;(ir}(p5Vm-AIkKG4`^|P98+z$rv0Et|dXvLH{ab6(1xD z7h;yCm-2eU3>SMf#1zkZll zi!-CLlgb{=vy{}g{pN_LmC{EFGEdIOc%XT{`|m_5a9$3VcOjRj$c7>9_JKwwChI^} z?L_>}!kUp?295_Ye@3r}q#|#{*X9u|xS{r~P~MLMh+PIX|M;{AwJvc7PCP6Ti^2JJ(Ko}<+f!;tKRXA<7>&G2KZr2@WPN=`S+NkOh=wWcoU z(=M)MZ&Icp-Ay;f2St@!wK=jvS;KC^Eev$XLh0-P0nB;@<2KllppQn(?PURl4a_1+ z3oQbgcGEyYKPxt8X3Zo(h=kRoy*eGnEw6rvl4u3u5-1)nY&?4#J*T|O@hNidY56x9 z4fhYWqc_-*@w$O_O#+yMd1|EPOX|i9b4W;+pRbS{4Q`mZ*k}F-a01TfpVkUjVa%6b zF#vdhM{l*g?j+MnY7_HX;o9lxt-?m6jqIAh>jul{#7kqHw5h*(^72%ZFOpWeI7qS* zbut|4-tcm|vEjz`k0qyf;KCD>Aa~nm5g*?(w7b@CfQ%OMY0x2wH!j=Oerx^v5PA`0 z`$bhK`F)5=98OLWRo;^d_eH@2zR2hG?M^bQi{`1-%BsD%(W|LCfCf~&fFHs6?@D)?VnSYuWK1a#%>>MPEDFz0DNYEpD!N{M)iR-(F(10 z@8n8B78?~02Vm`NZ}_{%B!o!#p19O`Hek4&3tPt!_kl zl|XihxF2q+pe-qSMc=EVWsFPh;)SUr<}KF8hktE_2!PHLp+=7AR9%~&{0lw> z$BZLKzt!f6qu;XB4g)EG2pg(N$A(;Y8XXBBX$+4d=C+SSs7)#IMjX>M{MA9mw8YQ_a7uZ68v35BuJ9Ka zyVayNtZ-IrayJ4-9|{o6Mea@G(BInGCo-pP+U zA6vs$fD{%R%UmXaWxEo{nc+@;Zflu+J@7<<^SJ!R!;fIQiMv|8s~ow>qIJ?L@feA! z9p!a%X6@^N$W-D)Ws0UB58XypD-JA2#rRhif~x*R1(O130@eE=VaiAW!g6K%gkp2m zYq}2VDkjkai?^@y4BJ1R!hoK~OFW--6KSJc>5s(j59?PCIW@g=0vB%n*;Z%sdVo%Q zFsP6)o*>Z#89*e5evY91!}Y4aonuuEr5?1TLJZ_@+<`|21>Md_3u1;pgSi4mbv! zg!;$py{esu1FViN7B%Lt!R-8vJY~m>4cM_h&hQpi=GfQqA}Xqyt`=KH-(J^jt$qda zeYBQOrX`{Y9KWrbf}T9CosfpZ*yE6hFz`x>P6e7H=I+LAYl%fzB(1TDjO?c^1e-~L z894T8_p2h-tH%a}1UWBV4YdEk5!0VBSNXK%ycAmn&@PBh_yU)>(wel`Xx-=c!a=n+ z7JCh$cmcwI4t%PN~W)h6AC`~%Muz_wRw$8*+j*rp}Lg}Q1~YbQU0nPB$~baB=2 zANjN5JuXpBWegD=5t#FI`i(?MYm8Zh_D$bJg}-Zq=vaBj%I%zvNkGW5&dosV0-Cr8 z2jf_a>O0)rBcB;f8?V=XesoRI^)!lglWG$O3p#pg!5vN#f~G&P>2|+oC>1MC)lPIx zbd(=W4>lvuTxiC+=Zb|@ zbnei(;pO<$dMz}jZ2CV{;^dVq$|k_b)>7;?giU0CoG0Gs{ku<>pGhj!+XatV0fy%o z-4l_GQg0=Hw>?fd>d7tBmNo^2Vi34_rZ11sC_Cv#KbTpQ{k`znN>Z+>zuP^K=cyQP z@^F7OzUhgT@4^r$o|~cDmRked^NiMFTZ!Wl`>l~x>R|yDPSZ|()-~7_?DYIf4cHj! zJMsRxdPT=r{R?7aw17OXf)nL6GiQ$p157gL9&|&$JnJ7F>dsWj1zreCXY34+$#%W< z*=p|>W2MAPwRKgnq$f5Jk1w0DRVC*ULt;A_vm~ zN(CdmBCscdxV%CuvMK@Ky6=Z{QrUUw+q6Q$r>opz@+X{;-dmboUo5$Dzl)#qSV=P* z`@Px$rD2=1in0+=9VSOVo-KWzElwcbe~QAwP~46uxmP)}HYK43Hi=%pKM7kF_?XVr z7`8J!%|3<(B>4M0u*ba@PY7 z^Dw@wJD8~5m&QQxsbY{=(aQ4}q8>9SMN1eUJ6J#rryG}!`#`!2=AS1)3O-nKdT&9w z8l2TY=sOSGfnR-J=Wqy_Jn{uAir;wYJ0pRbnP0p5sPGG*iD6F<@Hn)jBXMo+8gX8< zdf@};tf&0*3qM6O|5_%^D;Fb|dqqL`D{|%&G|poWqofLFlCMi1N%-kcB+TKsST0HT zS|s&qmX~qeC9)Og6n%oCq@LK+Kks1+Ic%Gz?E-1W2eh8{iA#U=F=M|+3jV1Yu6xG0 zUrgt5a(rZu@%$kP2d;Kkyq|9kO(w$N3Mi#~TY;{=ngoywtljhQXwSc|t?n)tpS`VM zM}y7US-(zAiH&*MAAN@?fK8T`v*m)>uaO$34~#{K&*{dy#QY|TaPOq zzDmKh+Tv3UN&x9j5=X^Vj^C1ho*Z2Jig!Ne`^Ih?4Y53Wq6J>qfZrL(oK6xC+~S$E z4#d7f3RZ=e8&CL&U-irUmCQ5$>6l>e61-^izK_`7!i`zl$7gOw-CCokCG_-P&X|@0 zgeBqBy$?^Mc@ZfGm~YGJkKE@{^UQg~S4C=~k)6G8dg_Cy$l>G!Q*1V(j-=d;Hvx!! zp)+q6xl%6PvVcMnN{efO_VF!P{23Y1@YdK^XPe`z_7R(sgZ$q7ZP}6>2{EHJ|x;razcV*L8pci4!o z!R%k78EnTihhDWA>b)(r*>!Z)2Kl(&{0p=CG5y@jRFtbQ&V2`Cp5ghpGwwMH+&aqK zK!B+*jJ7q-P4K}x+KBO-+xhbn2Qlek3~p@ZScWEgAr*E8!ND=V-AU-L)cx*K3Y<5T zT_6I7!0}`jwXC$4LX`$cUu}9`>P0iq-}GhNZw{p9wIz`u9*$f)NT>Va0z(DfFr(2I zgRAh(0*XLN90fCM{_}6tO4KGZRtg>!_rg}-66-DQY%v=uYsleWol%=7MYa+mNtxCT zx$ARs!0K`PN)q<m+K}D;ykckKDrJVUz8}Cu@(b6Ne!S8sXt zLcnsysLnDt_2E!>z3tu%u>)@9&yh1wQyEL*QnMxW9NE0JF2j@Ho&2^NcY}_xkJn8+ z8ZqXVK{dCK=tFn#|A+4;xTWJ9k%ipCjbz7z7@BHMsnX2T(hmfGi&xq~v(qkuFk7G$ z5-1e-=cl1PA-S~T6nZ^}hKg({7D|-pex9-4ap2$8JXIuz0{TBb?_Nwsxnav3V1ds1 z#Fd&O0`beA@(yBOlSzoLSPU-I)-fqb-AptKo-&zSE#dLME}X(6{@q8U!R*IqJ2Glgq?6{joYw5 zk5h9S$E$6tE!8Q8l6~f7_DE*6IySLTNFTT1Vc%h)|k92`Ro% zx{Xu*{9&hK)#Iw0$C0^2b@369$?zA2@H%gf7o_G%Ge##HEQO*%pCW5}Bqi zzX^dqn7WLmcJf39tM_Z*^DU>>;X+cdY;yNmfZVOB?+zpzzCY7RMK9XIB0Ui>ks~&s zOgM3UvJTekg-KH%4V5y-z~afe{=GoVgF8oX_1x+`TrqVe!SvJckK<@xK)Vv|5U(?YDb8P-1~0G)5wfDRIT)Yh8A_>foltD4j{cgCHb;+ zJH=KsX2c@HLPgh3?<0$!^=j*cUQ9PVa%OR3sBPQQk6Ehbq-P(Jw$&U@JWY2v?KG0u zb~Q798GbAi+X{qQ7Y4UpZF!iUTFlP1sVL?>D*yE@O086IFv?l|6CeH02~o@u|7aUMLQu!~ zOQZbC&v!k-!PUO4d_2<5$0dbm%=(#hTm0{^VD46ndYYZ9&ZV4 zFFuo-3^A9!H>|@u_2I)$7Nkq4&`bq~ten#>lddF|Aork)_E$w8Uw0*2MsUTkj`DwPw&(jz1qufbeqZhS`8+{$>+rS68SZ{efZOY z5Fl$Vj5Jow<#}Up zaL|1ByWf4Q4fAKF29v#x4+zs_+Sq_}4O-H*=4F?fgjU4%>hjxI6{a(^`w_Nkd6spZ z^EmKJSblA^|b>$D;Trd$-m- z%Ns}!UUEu$Jjo#nNspKC=1_wH>_kMT=luGPyy$e;v2tMaYN8@hAu2C* zdK6!y(b>x40=U69EiNysu<)j`1-h;LPgM|j7Rsg`d`MyVa8U6^ctMIHLG6n>|}`_K1{X)`Xo zpCCPaPeG&y1_pZVwNrB;PD^r{RBu_Sk;@n3uznp8KXBkc>0@9j zI2xU@Jc2+-S9v14IIO)x<0p2C-6#!?L1`j7!vIF4|&Ie@s^!ZL@t#DV>&NQ0hb} zotNxTBv8CRN_tteMG-5Y89k5 zIc?hbo+2#N`wusL zuvsbD9*7P)8^1eyLoQ^;<+`HF|hk;x9Rox{JCrY0t{gJQ-a&?M!;`5g4x<}l2=aDD+ z^L`s_*NhkbY0JWEmRUTLoPJ0+<@-;=aON5BE1m<14!j(8l!{%ur@kyATkgGT6!{B@64#wLz zdM{mVi@n;e$CwdGPcGKtfNIN(5S~KKg>x_UDWdUyogNxeA533|qz7uRwQJWp>5!l0 zm2*}XjGDc=E5|vA4`+4(_Uq?b`=#vGl>S)Whj#o;*Y`hH zYZ=K!US;?^vbGi^I5TF2>%THr>Up!vT)d0yMVB>=sT>uFW?|HIgxLGOOys8V~-iJ*}O+rFK!YPV853gda z&-2Kli1LWV54D;igrw?9s6vIRt7w?q+cOZntAvLL5+5ag(nN?X&b@%N)b_{BTXGi? zAc*-fn+w}kfY5Me7x(J2l}4$Lo6+Tu&d%q+fdf6vmnA~i4Gavt&m<%yB%Gqi)k@rX zp0`bW^mQUW3FCnn@lh)`muxQck=4&j!jx{IWv?!w=AuncE{?@HZG8OJ6>i-gkR4ib zgYaN#JJef#SN05hb;*SbL3(K81Jbi{<;n&ZF(mxLg$W4>38y?Vv7Ja>u_Z^Qwi^NR zyzApj$KbwLo$QS@*1fv0me1$EbgNqjXgq-F@5sO065~|d>&pv6c&S&U&ot?oXf?h> zBjlKkOszE}ILIW<<>sV#k!bS+!UIWLE z3^Ao0GfQCD?54BpRcR9v5)w`|{vU63ZrEeX?faD(M=r46)Lx>Xu4RN-lr}Dpb0(Xs{!XT+B?0=1)6>dh% zX9@{Ww@D9BY}`kOEHJ$+U`$sMS>ec;T!Q>$eidgY@xh{nRvJ{qft}g0q`F^yH@o11 z3!v&M*%6WOBz0FpLPA2T#3!WE3Q2OR{C68z5D~`?0R71 zLwg_K=9_QE#-sU>BS#umz2yj*B|Jr((`A*>goK2Ii9$PMC1(3GSwu(UyLUY}Og8+U zTNhcP0d)|5R1X0lM+5yMduX})o~VEbaj!4ASCFL2*KBf+7R##^LNy6^*AY{N_e&Qd@>tAzuaecC(YE~B|K7kEN zv(Or6-6BA4bx$>ut0eVNFI6UOddT2VFht=JxhM3AKq|yqxN%w^D1q_+F zb3E`X@BH_r>ysT^%7&UE51!M8*j?n2#X=_t&AeB>(w!%qS_lyqIZ#y%`qHnCFV;bX$c!Wkq5H9LvGdS% zEYCW>Rr!y_*{*wxkcHAa83<6QZY8tk2mTL-Gx+V1~O=HjE z`_dGdqn!_~L5jF-1%PuQ>ER47Y)I;Cd@R{%ST&fN-9#-L1xvzSane*d6Zn9akOv1^JJfM!z$E;$@0gbF; zp}KaYA60R;gHSqE(xq6~i9j8(i5c`xX95(G^vHYiyh<8`A8dRQ@ktmf>WGiD@lhX0 zx%#l`KJk%>7dQP22FVXH&T}kMhy@FAyy1o$n#ir6zFz3}aL)K4^&zl&GZrp{eGeBe z3-&wrY+C2+bH)qj-+SB)J$52^4^p3nZ+erH0wtWhs3JXe16Alc zt5FF__bPQ#EY1+JHdJQq_|xSHxpj&V@zU6Q*R`oG>2H z#K#Iidj0qhknrGGsJyyKazZGiEV-E7m5o?iyL()vndjV=0!K)BJn6}~h7aHgP)^bV zbr&};1weX+hIeB(+@_+hudm5XOJUTERhx0NKQms~cVxHu(t{rzSJDHVwkh5*oY=L? zZ2QC~%-(7QK;_k>Vi}5ZF_Q%A2@iMp5~SeA6SJGDnY(uZ8`b-A&Vf zu)Y)7`_KPuj&0gx7TQLUR9`0-@_4CjUE^JpQ;b_OsXu3m7ykhrxF5M>rSiSgrcR%FTxxv~wyvD+SOP4Hg12d*} z%OhqzVX;E#pW$(T!;{Yz;W4;leH|`JIHxgS9y$7?x!U}=y9V(9Uf6%WSutaA3z6aV zm4h!Zn~rTan~raH1OnF}86Xn-PT1?&!`HJL{VpB={%+lWx0%y3;QWISAw1&)9zFV$ z`N_r=rX3(We#|TKgEVmstUGW|LAv0Ed+?vZtS%L-;jo>R)>e;GwSILd;6$lx4yJs>@> z@u9lwvdb=W@%uG=TG9mBA>kqX{#@__(o+~WkChEh6zKsLZg2m29kOaZNPqIfr_S`o zQw>@|0!Rr3R0mSB(&>+?2OURUeEL4~(atX=+PAPnz0l4R4sHSU9u=)8n*S^s&(IpPDsLOf1-V~@E0%$|R8`8c=ljNv@> zE-w84-l>ve&x7OIUeNW3)8{>qG8=fm_azx;By zh#^SN(@#H*Rh-AXLFssro;iKPD~-ZN=;(=&xK7)2qN8|@TB*VLu0_n)Dmh7zPUF3+w-V+B((vi)_cQ|5!bKHkC2c(A* z9k}DiYXE*Ecn_pX`iJC4k|qM_z>m%=<6;~jPcHvfqIB-eGh8^_-+XLGk>=0Ne|4cM zd-uYB*AgG#w%Vv5GFW*U#AtAEc*_slLk(~3F*qSwh(chZq>Z^aH{C{w_~c>*MP>+O z2jm9sILf^@s{|Mx8Hs{Y=az0qHPb8$Mqd|IQH+PJviaa zuF;CYVAJzYkA1FV(lfJXmRU95zA$r#%(6K{1?eF`es&&u){&~y=G<$JZ0}4@dgSrP z&F-&$)y#jx8%)BxiaRk+-OTq*P=&)_;h&E~|CGoSf%K`G!+oTIyaOalP|YDdke<5^JyH;*7*N01 zShWg}3ehj>H@27A7*VX-qb4R~Lby^K^U0jmWlVV;AI|2=%2V2%_=>Z;;4aqmaSIjB znKQ@j%nlL+@*{U;$5eJMUMP*6v4#((zHhP;5fGj&TecLEr_s*p`rN&LYW5##f1|kB zGv}L&m%Y@iUi5s^*V{e=KE7+0v*9^-_uZxwEU;?9!56&1Of>T533F`UUUR|@1}6?3 zFjj|X62=c4P)@+M#DK6{l}p}q$A!;1*Yx)FnS}1*uvaJ31_(cZ$LUHuLxe^W9_g-a z&E&o5x{eQbTwim=rOxJOECA`CL};P_qDARUC-+7JkWP>ps0KAa79nxteSJMzgl2W& z=-2cs#&A-KE_ z#3vjPY6p96ywMC?c;SQ@J2#IVHHWuvGspMuwd5yFNu3DDfu~7Hk3SzavNZ{l01WIH zv?2YW?GMjS7SbaTQhkL8(R2qSKG+rl1NK-k(WD1}q$UPRnZ^r{f?ae$gv4a!`v(sl zuJ%7lc*Ev=Bvn&fi#D1Gx325_8zDzeGMs9Q(6a$RWI&8Ot8>%WgoMZa0Kafy=4W!b zNNCq%2~X}-U2g3jY)T5!LxAw$hw1E{Rt{8LcR#bPEx(ZRy#D-mm=~;AGb+*}0Rpt_ zw}0EredR0L^53*yrMs-kYrFvo&*81lnthM2H^=twPNZi7QL?-6Zh7&~m}JQ_Igp0s zK;7)8>4=y_Cp42wO6|L$@|tKYSv=p26%$Q*fGV}rc&ASlRtO-B@;>UX%KsQ4Crg!< z$MIu%$*tRH#6&)+|#+d3DXyT)AG{#Js!olFWPKZXLkh` zFyt}tWkh?P!hz?U!VXE#)mLBb{5|^Uqxofi!%nEUVAq3lxoR^2sn1hSJ>||heVvOJ zE?KgqA@kJ^AnjSd{Rg%FNP4v4@_3;W@-tqG8)|`4baHdI*r8 zcDAa3#OK)N&1Nhh>1iH2W)iv!-9U>MJdvJBg`QzYx07Yimr@?>ra^;dd7Y3j-thLn z#kgHe70_F!fgrm@q;2Q#{$?(&-0GQ3?r?6vIm)X93dvo#P#T|*kHtu@3*3at2&o@I0&Ntn_ zf;YXXE#u`w4?JMT0vItT(leP*O%(6;DBkB|_8A^Lc+l+Mzu)ZLyVw2p?c3+3XP<0_ zsuPWq&CW$jYC@{4DydLxs(V7hI3m|ak{=_aJi^Gi>u6+w9oaR>;ERSS>!BA0GrO+6 z_F8wt@5x2-}yD_j_`(Y=EK27Wu{UZ{oEZ5Q>fRY~^Q7Ep>!_7(kYRx%kc6l2g=ZAiw=F;0#aEl6@ za|%0FYqnCP9MT@s0XDQD?O8r|sBIP3vs-tX|N9TO6h!BdN1rse{o4;*W`B6!W2Rlq zzT%2nsSh?YZFd^H#*rOcO~NEXs!@cqA9ibl<4YWp9RNf}4@~GJFcrJ6ug|Sa41$y} zI!tFZm(F{P+^KcfN|LXMmUdyL(2Fm-Fp-~A7g{TnE6J;z%5L&}3T#;slCA_giqfoI z9AUwYHGFVSPHQK;steQFbH8PUgO~KUd$jBEHa;#5sJcuC;;_y8Hq`p*MA|cd?rih? zOU@~-L4clTuMZzN>W=NAVM))Rm4NO4x9VMl58Jw#{o|dj?Qs0yfu!P^L==Mz)Cf5D zbJ<5Ex%JH#YAq0+b_Qiif3QH`thCu}B|_4X>1^YpiHw9+fyM5{!jC5~Rhla^C!Bil z!jMet&R$Gk*Y94IO_6ncJTje0@v`K4YCBhO=9G4l9=}(Yix2+<>1i1eEa7q2+@^xC zDzCOG2P&?*1q&ArEbGXidDg7*s+1r+d-m*i8UN1r9yFbx|H2Dv{Tv;Ifiu)xqs8tY z-fMQ;@tv06K}$$?|KMJ;@6q*UB7yWz9PCVWgFJN_rX3vFvBQLwFfh{OTi=%e1$M29O zI-760xwrh%Mr8j07%}Ur$*x1w|?|QZ&=Nh z+cG_7%CC2I(*=P6k&)Q=pMDG2}0XnEwOo?=9jzym@2GeYhdqj@!OkTu&sbrfg5w!9P?ct?JSrtFbzm z&MLozDUU+kC9kFMMEmdRboPpS#UewhlPCxgS8WC?!D&Ro7i;<8oLg1^VgbYAzEPuu zB|&nEeI*X>ua|r*n>%DW0mu!0kF0;vJiBG5dBqjacQ!t^-TnjfikCgVqqv~%>YZOL zZR&LW9jwqi7B$z>tFAQ5-|#wjoVjSBJI8O%xvL95ARHmd4+zUaK^owXKS^fbCq{0T zT(!mwoUyV9U*-4E)6bYiFMWxby?V8&g3Rx6WeX$ojC+tcfkeT*Z_y&N{Pk;#GUzyU z{5a~b=ot$an$y<4)~(-jyf8wJH{FoL=IKAb&TRks=iK@HE7scI>)iP!OMte1{$qtd zZpjeBj@xcD5C8VhI5JWyza<+_{K@Y)0)lf#R_?MyW`Fxq!c{-AxkbRCOJlfVLMu;Y={MZBP^Iy8vT=}x`-3lRLT7ep!($sY& zfo#tXU;@xvb<u_ly=PU!Zjp4n9dG0 zSGy9Q1C4sp(AU$MO$2tXQ9hsMnly-?wfJHKlsBF|W< z(jg}iI`=>SC+GgCmAVX^e!6>dJMR3BIpfVgZT9@|KC|_fTP=ZFS%U@FROER|E+#Jm>R8?DV zUFYIKev$9zy!)42o{BtMY`bNhO9NyIIB1WjS@H*o(sTafJIzCX@Q3Dt_x+CP8R&QS zAOF}VZ9MOEWrdUqVG;kTpMRT~v0y>*G9TT!(`@*=&zR@@>MxsTzVsDy?7#uD^n)KV zy?xO!E4?NUS^5L-WK;~KKBuG z#yj6@W-VV;xb0Z?RkQ7@pLWk+-eoUyVK;y7FWq&Y^|$C1Zz!&NXU{9b!ELVfKe+n2 z?z+d`Ls)tLm?c;<=FD^V`b3(*fk*FkVL_M%&%MMEu`}Lzo!R=;zqit$Cv6_jbYyAy zTYtrj1|fnVB_=0>#6ue`ON7eT4YG$lpkDXd%UgC`PdK&U)m(B`cIG!T=hvQWj5Mw3 z%GXF0@p03;S{#QLG?ZT#jsQ{Z<#l^4!HkzKVI3diiH&-$-u#iU(E}*Bs-<J`;_~2irzV4$>PyX z8%v~zSh(g5j)Xk^$M12{Ae3V8+~+zH0u0!5lA0$iQQ2$>6vzrm4*-(#q?h_EvhnS= z#Afw>{n)Vj%Swry;dgds;UC23#4C0?DbUh4yxlzhC+~6UulpSHD3B-iBHp$A5C)dZoZ3E z#HW;tz{}z+wkZ(;;o-IwdZSV!mZye|k2i2G#V@5j-p0q>hqUKp!j|O?^VrN@GhF}@ zP@K;i-fR!#2NIIaU-gm+`?RRm0pTEyZr|yM#eT1L8nDCxwnLPnNX&o!bDh5*ee@H< zq~)qLPW3bzFb~K($O`j3``r3jc#Yl}rhnP#=A3u`a^W^>@lrGQ+~LJ8&-;J>Uvq5l zUbE@O8_mIIH@UKu1{Y&Wt2Sq|`YO}I`%^{k_x+$e3$SPQ{Fd;Y00ag^gd_&`KEv@2 zOLBUxbOr>bXAUGZXIcr&`^?_EZ#N;GG>B0-Z}ZHV*1v|75`{oX5dEFlKYC;RQXXrh z8z+;*M?#XDn4c>5goLqx%iBN;Bs4*^@yQ||BUv|Xd@3bAx!S@)-oro7+zL`dX%DBh z(=%**?z-zP&gv@st+}0H!KUky0Pi4=&eNT(sPYg5L*jc@9al^4-_n!S`{=C_h ze)b<7-$e!+GiI7@1B3k>OCl%LU`J6;^HZ%| zI`UHrckXbw*@{?o3O$8epHD}LPnBvbT-|@F2O*NroPCQW+ej z ziZGCjNCR9*G+^&SKoZn!o-l`ZY*$9k!<&{4*tUGu>)v3cNb%PU?-*uuLFz*iwBjAV z?T#Qn2Oqn?@Vn~0e_ZJ1d*_ba}PGO;>i#zl+lF&7(^ITAyukJ_0%3*;r^0*2n! z2U}N|C!ToXeS?F8?==;`boMVj@X;zS=Hk;{>NY2>!Fw}WMSMVd{`T%atW0Xd_cncUd>gnu0$VIcqhSj1$BL{t;fz zDwA=^8DEG?=9geRNDoKGAIl5MG5?}V%)E;(F5-Z+Wc`2tk1cU8ef`@?1AK1D$L_}- zHYbi9b2GVM%zSA34l_8k$}CxZf$8n5R&@a&JzT)BR1Tr$0_hq!XC!}kC$Xp3Tq}J# z*+82h7)39!^&PWUssmQH206rv*C09a2TY`$%S53awUtzz#@6tmos({JAzGUjQ|hC*#H6wC=%=W(uoj{pDe)m|bA@_9 zY_N$9OU+23oL}zO1>L~~EP<)(AUptw&H3k^ zSriP?9}pdoq|;7YSQ}S6>kJJx>jz4Wp8mlx6CV&9O(L|^MUox>ee0qz6(Gu3NM9rcb*g$xnuDB<3wJhjwgpIxf`XoU_g{ zi&md!)m-g8OAJ6*ff;M^$WCO=^TR86ll07g&fJ8`2p+<`kshRFGy0mmB4=gep zlKjY{KJPZLm)C}k&zcL*A2LgqEO5l+!H1u4*Mo!omKfY!tdXY44@nV-iwuuAF^dfT zL2M3Me;1yAmh*qx?RUE){L8RFhwHKB_Hh&5DBQY@Pl-RfFy|Ot~$FO;?eCtsP(hF z!*zy$zJ;SR0!|o4LrgWtlxi*Mftp9oUIO`fQmk3iJ22ZxdpMZPTX|;Nq=x|Ef!Yd- z8CDHSkYJGXOjlLSWI(EXq@B!kgC;TU*y~KS1q^Tc#vP6ntg})W5QUq*`R(Ew!y)$i z!V)4&mbCWeAT$qHqT!@9w+~BBKJ}TenX^_cGefJEJJJIZgmbtrwQ)d_bCD%2a9d)D z5@wwtju0S?S!(aYFZLRPza{=U1P~$Q{r-m`McU=ca^VF-X4R@?LI2;o`$6-f7hGUY zW_W2&NkAg+3rVX~eRZ`l)=n=HUr5Ccrhc>1jPvv{WsT}C)2cqpAnoz0uEG(E7|QAF zR?6ez!roojs1(!Lu}P^Jsl2u-X4&i^v+qbX!PvO#VY6spnP~^B&RAx?@h{&sv-)N_ zd!Igggu3gsuX?e0_|ake5ANSpg?&eMyL9WYcwy(_-K^1LX3U+Rc4r?YdS;H;^&Ac? zss}X_q&^ZLNCc!l#W|!m^G2td0rA0V%-DS$(ongL1yl{_Kj3FzVBn;fku+JD&UH6d z!{kl~(^vpPqopGRhy)}eOD!QeXxlm@FyFIMmZe_R}D_`~kD=~V|<>Sq7yee7; zCpo0XM{eE#bWbG%TS~G&PyfQ&_<6d2y)nx*9MMFbFMIw;= zfb`HW?0Y)Fxl8I-Q@(%8owY{BugQ4?Ve)=jW<-rkz=NEqVC){5k2Kn(7FZBXq7+dXn&Tqy7BlC;!1oUP8c!A|xwM*!#oMm?5i*I^=#J3Lq-q zyZhnj?(X0B_ehN7LWz*R-0-Da-Tkx99G21y1Qt?!@Zk-Gt<8{?&Op6|>uY}MDo2Wt z7mQ#~jS))&5~x9MC&)A)ajF8OS+ROKiIDTBwL=1h4RE)mJxn}$FJ{yXq9bQ?N#kbN z_lONji@myN-0Wv|!N?h3wVc%zloZjp*&g#B{pd&6)fqPfAQzwe;h)x$jCPHj#{-b| zeCffD)-JF();t~AwZk0V+L;qkE8@_al zBN**~hR6g+5r_#SE7KJy|C%c@+&;=gOZua?1+t*VPVS=X37j=R2uR6>xvmg7e)hs)@$8 zp7_VMR+YZU5}&c4Z^0tdJa*I^-L=ysj1HmoX3qSj_qnxiB>B+`2?81Zti*Qb#Q|3P^pZ3DT=G zOS@e&S*YBdJtRgI3lO!C9vC%Y@j|TMBUE0QHf{!B+-y}`?tJCSl~UpZ2~Tl8nw6U` znESl8{QcAV&pBz#MB?sE>&)ghW|FP8WNl90=xjqI`j;*@GZ)m)d`cL>v3>iBb1a^B zhFX1?yO z%JYe~@u}LUYb;w@Al{Q<53r##r#E@40yIC~{x`@I0N?&Pvfmv~RPN3m`-(6^f}<_A zL3(iX>-fZgDeaK>VC80PRZ+x?SzWecDk`m08y{fa{N?63v(Kyb2X)v@k9=yv$PY-* z-OsFR%kK+TtTE#Ske|Lqo!8r`3hZ!yT#ytV6fb?LNmt0%ssNW3K8^HNE zn~D5PGPF7?X74kdQAJ=R+0*3jWU7ONsR!=I6@VMtz2+;rg~dc8N__HRxbwh*L&cG~ z=E}{0B`Y*+cGsBAC1-RYO23Gq3%gRRAA^lb{gxFiYXNnF7oNSgtqn=Bk~7Fpr&HYP zaOaa>Z=3Xtm$XL$#?EujKG*d0r5%#n!QpMsIs!z1{H*@3A9c1o{H=J)wJoFv+g9Yq zjtyTxn&1Z*9P?&wwm$lh$xj?N3EhVt__h0VM%5ZcA<#xAk)DKZfg6{LJv{+Vkiubj zL7z!zkI^?J#Kgp+h9pdqn8apnWr1^d#6DfZxY^nGEMLBy#D}}HyYqHdZr12T;sQYG z@`BUXn6Y5<-VNrCr@rpi3u*_L$%U1g$D8;_K$>xM4|c)c>m(5gVMrtAc7QrPxOrGw ziFbo*`M7g$ObTP?MdzGn`sO5K=WYYW%|Ohk`D8I}rn@$3<~7E&*-QeW#?4uIW71sDyZ1A@wDFSj>-Xp?t{IP<(IvL6D3ZiK zuG$RJLyUHZb`Tj5k-mftFF)&z=B9@~)%LFFP<>$*)}2;rvughEtgl7=%gmy|^7=YQjvsbX z9gr&iI>l%R)>tt-sMind3Dv-wNgzFd{IcdTSJEFyAFvr|uGWvyU+3DcP9Lkn$T;a< ztSbfJsw+0U@AXmpS}JyKjzjY!umj%g5Pd=raQcS zFE=SI_VZLGGX)tkBkHfNG4V>`gG_zk10N{FQGx(v(o$PZYj=ghtS-N2m$vV5yR!S; z*`sNb^k5AixphUSfHdXRtKaI#Pu5Y`&%Hak^_1+N_&nN$*LkaI$T}6qs@~pAEAkj+>A8jcREWu zLZn6$>KHkS`R^vUzZHlErmM@ny0Eql2m;o+A+U%b7CqE^dtpJ@fr0*FFD|T&qo=xK z1!)Yc4?grnbnh%|p@F#YLp%o#9C8aBGCdF%?!Co*yijxSA6knCaY&Mb#Sl*w(&LeH zZ!SR3>T-l4FRkR<^@dnGt4mHx6ED+<<+A{1bV1T17cvB_nyawyvFEuoZZ^ZSxv*-p zvla6Db~$vihEEJw4Hz{W~?7Fr3?%ZJ`*5>75~;N^fzNds&txQL!U zYB%XO^o#lB5mKd4Iy%m(bacG!T*u<9E4waNJ88t?(2EqxXTo#&tfnJmc|!LHNo=&V zXR=A(*S`5}Ct2ZMTyn22M^pqrBCvZp{D1x*Z*^{v2ssjy8Nbr=@LpU?A|wP77{raO zCo~X#nDh+qvS1MXq9D6hC<{jM1ZuKi_c*?Bcv+LBkT5rRGG2sZUQSh zgHU|x(>J?oy;qk6hzRb3bX??jTjyFnAS*aRax@TG-v@hv4f_eJ!G`1NcSMHa-7fJy zKBVV}6(m!F#KhJT+Nc&L+`jG9+&>ZfcluK&%z)H~LX!~baUQ#<>Z)A8(ES^E$)xCt zYxU$jg5~)s?KyOOUeD(kpI^W*gN=_TKW>v!OL(07h7B7s+j%+C1kxiHFvP#n8aHc9 zHu-@~4{VP9|MuPk&aSh(^MB4%O;a6dR4rL@ljK6Mjj=Jsb_@Yy0*UQ}4Z&nL354wa zH-SLbPCjVOCuGwgVG{x&WJAd=2_}JHS|BbEW8;dOTx46tlGSEZr)fqrbN=t|z3+48 zIq!YXxp&IFQ||Nqe9oL+@0~fnd8&$@`H6hF${|Z2_yQobpPVjbF$oC+|M*g5WSu8V z#7sss`NBr7m-p-orL5=WKa0HW=8M?LBXpmE7fZ^$HPg=nO+2mGaD@#0UR?~tQEN-V zA_MGtCsZJWxR;mh%5Gd=+Z&w+s6VA`u zhs4houhO(XUkf0xSz36Ut1EWjL8}XdV2`31$n47f4G=LsxkU`U`NZ8NTwRer=FF}Q zg_L%?9PTdE@gb4kP6Z3nq|({*K=ATY2Y*b}07C2a8{b})_87{{6ME73FC9)X=I!G^ z5HRB}=Xw7>_^j>yWBJ>c+AnXPa>)4`k}NN^U0g=|lP^5;wwbHseFlEIx}25)`20C* zI(ub+NP5*PHz)B%pZeQ>q3hmw`J8XHkwqw--DP&kFDt*#?H5V>y<_3Sy+NxYf^uD5 zF2*Oy`JVf8VBXId>jl2@<$8Ty3_e~T7J zm9zdJrU(Jd@7Vf26+JaX)(copvsHqZhb$$&rAE$&+*x^XGFcXr&~fuDIC<`>ENfQC z@)fdVN$XF@Vpht#De7xm0+so_F9?9d$tS;`*P65jg++A$($=C+e_4W1LGMyG5$<70h z(K9C=n^a1A2ne3@SH7CIu6VV&wUkVYMkppD&Im{?Md*bmEkk)Oj;xnv-ZAr*Wm#+T zt}TnaYszxzgpM&^$oj1OSRbqpS%Bnr5Q~-S&aTuBA&U=Zb#2?WjYWk2OP++%%*5|! z-Wj8eKbpAUEK5=JhC=be*iBo}!*zW4e8$Zc`B`1~AzfW4VCYiXqt4DwScWPld>CYG zp=UR?&f}kBqW4hh zT=UVYBMDh^q$@SFZB_{db#*DzL008vc6T|$NMhN^ad}5km0VpV|4Im}4_|P^MTsQB z>chuPVszyBT=iz&25Ia`;X>E0!oN1VT_r6(A#8!d!GqL&beAn^*dGn0>Hr~b2X%cq zTUXLjGaztUQ52<;Z(Lmm!eFXBVRU3{2`imEO|T8!q<^9dkMJ(ebM7D^J9JHgYYR77Ch6YtM`r#&^>)W zq3ajECU>8|Ir^{k-lacV^4NFxKT21%y)v2qf5*R17q^^C=Qf`~W)qeWUkBw5FlcJ}+ErM}2GRekV#m_`r;q6h5Z3zzK<+ zs*9kKy}E=MDi+HUVI!?SyqY{Wmx%Epv5g$3miwZIgQOK@Rxlr}J)mhdDRT=MD-{wa zp%|d4##NO9)r%i`^qHj11J82{2(+L;YPTFC>j}jQ<@UV4-?9SrdQe}7ixNU$JU9Fj zT~7t!gO@pgSXkZIX@!i0=kcKz@h;sx{VbuIQd0NdT& z_h=#(AWE=59}nyp-b-(*yOPQQ`?IjYIdDBVCkvf>`hQ~A^ZiJ0E!aM^Kc=w^)7TEv zxTiae=;<@_s~VSO9(Ni@eYrS9hQ@V)zirhmKbh(5;y(uta<;(wgXJ94!}DzIK{gk; zz-Aw~y106?e>yo9%}GOwQBh_VxVq*eO>0P*TZGd0^R@+XMpoQ(@s+RNMO!v+p!Mrl zCi4(5-}>JDR@9&c1>iB311|RW`TzHAyFK;~wX8q|Wxo>h8`{R3i{eF&0WWz7hrZE4 zI=ATzD*&v8!w4nJ!;-_o0Q0cKz^cQdfsfTUHfTiyz&^ariX?Lm78MX3{aGZ?EH==5>W@YUnIFjDJWQs=N7lZOzf^E}2?YnN&~e3zB9X%G7crFG*^>f> zJnxMYWmt(CK8iAFPQ6I;5?FicT1c7QWQm+m=>p&FRmwUvUA*;8SJ>*zXf<);+pkXA zKcJ}J?w5+&5plCpInVpx7_wzW7A!{a62IHL@K>7`JcPl?(f-650>N`&7op` zm9+T4&E?uwB!oJ%uU)$~mJLc3W!5oYtv&TiNtyj*#rR-A%uYz;g#L-Gq`PbTGrN*$ zw4gu_E^ZVGfy47$)X#7IfO)LP01+fxR`7bi87PAnIqnu#ns)PYXX^>vSGe2jJ6>Zg zI~dq918WK9u@1m0@)J%f>;AG`56e;LIx&xDi6O>!WCseS|Uq4ra zo4Moq8?O8*8}l<2AUfD0R4MX;nI{tH4B;gO6kCHz1$9@Ntd5S_aoHOXq9-{PL=RkD{_%zAk<0z8FD1aBaU*vB z?cdVATW+z#(XW1$MvomUYu`iNN9e=<^AlG504zb%zWCVXPZyR$_#Es$N)ut{?!9#X zLyysH;6-yS_dWPHO$jfsdFRVxbpKN$!AQrAmt)@Shf6o4JmyOyz)UecQq-lL2n6H?;p0%{FA8E}8RUd7IMiZpvDMIL1eA6X@54;AVRuXYq;AvH}&G<*C@+g^aGQuCCaOHMlv4 ze&b3!@})1)(9WI7vH`Q~un|J>sq%ZoK-l>65Jk)f-+wc0+A#iE!GZ#7js(ltN(5jb zde@C_P1f1#HT%eI{NwQuTJN8ikL7PK&%xu1jMeFV@3}E}PPre?hmPyJx~{mOp5FDQ z@gi{jr{c1N3bygLE48m&hix&)<#Kjq zBgYLkmGiSJY?k5|Ip*weJCcF+)pT(&x#{e#LNhBjS-3DIa-71m<}5L7EkC`zy-6zj z#TQ>3n`tYyTyQg6daU?Cj1R0n2M!#Nfz4|1DZ$A*?+h-7#i#e)duicM|1_1NYxy#3 z4SC;t1`=1+KYZnzDHj#2KOg+fo9(uk#ySY2jT_ckLGcf_f74ox+FDzUIeLKpAQbcct-al*1DuWiu4<~dH5l_#rkp`uh04tw$I{R#b$ zN^viQ0NL#Es#R{z%j%Lox!7g1C#!kFrM25>DS7}bJ_u^&IzDD7%H|{mE?~u*byp{@>g0 zpbr?U6t3e7fBms!xfM9AEvajQ2z$G^-*;ShjlGte-~an`;g)mkJ`h5G{inZY?*rFz z|AUVegindUW{95m{cIy$etv@$77!Xf-+ySt3Y1TMpdd6b4GRo}&lhfOvGG6!Y6^PLB1PpBSY3wvQUYQ*Y1pt~WIqK^58SxZfu(6C3N}lwRuMjo693tggt5@3Fw@ z!!9lqFr@6t&WfO}CI3J5`UT}TON9`C2!TMrm|avpY)Sde5B-4^2L1hoYzu4;kpb%s zEGK#UwhQYHM2-|#d4bTp4*Ot$$iX$>dSPkWxqF{I79TI`SP)#BHmpmoWz(5!6R`zR z$JVEm@H?F^d>~*TdhC8ibhHN^{S!OKl5GH6Wi}e?4hKPZEiqP~{S%7~y6~(*@N;{M zInI(OITl-$7>J(2_0?G`kwx27%qe^~mFG4HcCF8wSCmS@qDR(07SpNB@oKuf@`O)O zpfN5E?k^~ythNXttY@Sn;3TBr0>FlAgqD&_u(J6AGM>ZzS`m)ZxIzD#B z{dURBF26%NS8L?rg#cMy9OGjwJvg7;`RS*hju6=F|Y9q zFIbq~|DliBb3&|P;N>_Du94%4d?4`t=I_5^_r-Q_Zy^vnmlCe7U%s}1KKl2kY^)Ak zRl)OKQco|wJZ9Dp+3gUY12F_k43@zKb=%($(Y2Qrd!dom{_%erw)=hkho@=U(7@(7 zLn&)c-hS1J^+AgEwnKk6p?yAc^y|s=gO5Bz-~PAzGq?T9*Y2_f4FOWy<-9WUxDF1O zjD?HK8coQ(+g)v+8B^X-x$8X>YSOksj){UzNab{tw~cXiA+VX`GGpyQHE8o6dH*e~ zt*y6`5(JrDy?5S8BQD;j`OGtEiTUzD{FDNs<;kaaB=v3Xd+;&3;#Gxdq_VT?2+KE} zxh|QuZY5(m;o3g1v~ark6|cI;ZpSVu#LviM0lzMgjVnT2j@b_D{NQGJeg^w+Jd!*X z&IvICi`38D_&)l=r#@!ZwWRKouNMN1_w}CzTdj8PH0OEvpHtU^xEY8R)KcR*K8Ue7 zbhI#!pyy4i>x=66z_o=q9t0e}@zVOl9mV@@S`#I8d!Be<)Vif`4D5##d02?Be&_yC z>o$|yz^##7BJ+MGNX&r*^R3!@&cP9nbn2jg)YV?5k4-?2YtKjSPK}I$JnV;)(SM2fIA9e z2La9qLWY$F78wkafrF)StndHee)|61_uF9SDRJ_LM902P^NyDDiB|k)Qf8K+;nOrc zEUkDMf12^9H)v>UtqdKSSE@%=0Yk3kBNCtzAtVbRb3wyd zk=5lcLe3{q*-=^vaZqzg@Q4~dMVVTt|-Nz zY7pWhqU9@_=*&$mw0u=*Ys}ObHABHAJ32aMgx9bcMySPzpH(ZDQFBuxO$Ti8vHhn; z2k1X7x;AML`iWKl&2A$_53p@12J!>ho_=lnJ9n29PnfPlMi*OsIy*bn;-kzgjcW?Sd_x19 ztIEjeD80CAKYjg6Cyh073H^^h=&}OG=dCv^w*u%ZpFd;-Q6t@Wb0NdE8ktxf?X7k= z+z2um@2|=bN=o(c`gY33 zr0mb)Fq&A@_Ngj)79WbTf6I~^68^i0Ej=>62O!1=V;ASc`&;qj3K;S-yBzg=kkJL< z!@DSntEM^8|9&j}-bR#LU- zHkO{eb`%JF-mrdE;ttzwtV2AnR3G2sbJbN>H9F%*663?o%_X5`rPU&@NvR!Y6~BTd z2tmyo?VKfD079--_0XOPtw=Fe3Ukx*;yod zo;^82FAt8=fm5UO;L&s`gqz;eMg4ta#{Jb7Ja#2nwset=!Rs;CUM))X;6kPsiyVyQ zyfX9nI$-SKBR+Lf*R~NBP$ok&SR1wWwR~Od1VCFHNw?AA93-1`AKxKA&SvMBq zdjN_TQriC)+*htsSC^tFa}oUQ?Cl$*fBy0w+Hgh}UG+2Lz0v;L!^i2_-VthwN9ir= z+GtHnJ$>r=ep+qDy(44vFISG&tb)sH^M$Q+{!g{r`D(s2_8&Y#O%3(5aN(@&2CmEy zIdy#S-5(Qa>#0~_nt|9JrlhNj1Dna|9oM$P&Qn0B824Av%{XgMjG$(h#%|{W`zPn( zzFk<>)zzie9z{{+4E}$($4~UqlMjy4tFBy>EQ8?r@8`DD+t)6jT_STEF0bFkrD)Bv$FoY?8vAb6PE zUR|Qhc}ld&j_fRgoIonNAy|75fC2Wjx%{A!QQ z2m=sCcYpU7?K0b8=stLww!Wf0wLL6JynPm}Eeq(>sr+pe#Ye^fb2d0Ky1b^PEH0{M z!YtXH-BW2%z!GHFxnO6+Ip5*nO{B5=Y3qM^$zto$y7h^Z^ytyk^!Gal=!?4sXos=#yywTqj0lS8 zOP4QA?bF{klG%lQ*9)ht5PItkopg^8I}kP9M$l}1MVqk{owUMfi*_Mr9vROQb7mLtT2>^dS65eM;m+o0HkwRtC|KX{U%KBdJ+zFM1H9nmv=k)#U%#}f zeDM7@XC6E2I6mmoj(%Nv*T>*E^5xFkhb$d(8Bb#@&mk=tq4l9-`s=X2FE~QaPVQUo zePFw*h(ZncfMACx>oQ-NrMp%{e7dUWxw_4aeB^(oZmUGo!-#Z zOgF4+p=*t$=XEQZ>0=izq~6mp-Egn>yv1Ka-<<}LgF=Y@JQqgvk=vINJ{s7pl#A%j zo>)p;h>RcPENl8y)wt1+vn~-d?&53#A{q08AELlv60po-$7Oe=0+^kvE5ewZ#RuZY zxw&GmQ7QN9nz4NUS_Um2zV_|AsI9e?zHTfAANkO4(ESfRW?!!8g#{}ML=uB;?XVc2 z!RN|1yqZ4xr=PadFYelFo1UV51oz?k_j~TM9of-N0$tT{9E|aTT(P{*;rH1w zt^?{7X%c8CZ=e8!S~4L zxb06qLl>TZ4)yg7*yplw{klRU)~$tB8?a#g^c$|Q*NyYZW~+RCxF0+#9Pcy6`q6Hl zF~03jJpCNK<1N=D%g{8nt7{ou`})i2cg=Bej)RAf+T)_N2gC*%!eW~51INXEaKl!7 zE`8=&u>2j@T}`+B>1XWrxaN&U?BaY9+|vbH&b80y!t>9z>+L<8b00d~W3NZL-!@?u|4*cE-;^g!69mGT|kYE)6>R6nZIyWm+Qu!dT;wZxssVGdvr=x zZWboh;h50$HG0kuq`(y1*qeruCAHkT{phf zj`6|QZ{)^}n=0z=e;;9{} zzFKVAfOx__VCU|=LE*#Th08*tt+mB=kWZb%98(IB&lq>!d*5@DJtnRbFMaOJUe0rY z$ieeqk%SjL1P7jhUH??tjR(Qp+H?AT+Qat`+CPb~rDd zGq&wtU$Cm+8T)XKXrRT~J)ZXY;}adhbKW=<#DCw5*{ZaFEG` zn4dddK1Dz38K!@_V%*gQf%BTH+mp`f;8*_Z>eP8I4unwfWX0r$&m<9B3>3UM>Zg-Vy##TQ>}kL%a- z0nXB6j{{2&*YRG6vy&G2&h>#1noNX;$u$1(B_XuGFiPIL8L_KNoP#*F2Q4eq)ipbS*m-f!Ui+eZ;^`fkFIZT9;D%v$i*+j*mkcZqu>Qa@ z)83XA8nA*O2p9Xo!jxy($>O4|XInQAEKxXz53U=o7hF%lEq9T1=VANr{osD6NqgpM^Ca9#)soCksc*8@?3^MHc;X&oKz?n(BM(^+u54_tRRUPv@r*Oht3_}nmU z-DS>_Cc$&TdbrzgJnQ1>9*;GFRi_ji&1c7-BSc;naF@+`Pdyi0_oD}X5`XER?ntc% z2M_nyLC*inH}8zUclUj9UtfQ+4EtgJwm)Z99P+;Jd+;a8KD?fn`P=gG`26x&`OD;Xe4U|l z$!pE}T;ykgW696O$KI#eAM53P@ILZ-l6%~q{+{qX@b&X&7@h}0M(i>99F^qI z;iK^jJNLzkQZDZOUw`9`H`WWC0O3;?k86rVXHZTd4ou&E z`|Y6^pSEe7`M<5PfyE&Zbo}>!xsTrV&c$@`!O-IJR@!{C2U-?vLdmA0*2s)7hd43 zuGo$3Zx`3_K}tLRSise#%zfbEx#H4SxNNFd1l=r(G9QqCFXgz=h?g_gblhb)Bt=9pkMIescS*TooGwa>V>UF7zXFFDT4Et8_h zkz{#s5kqISlr;fohwuTimXEhA$?76;!LDTmdG&jeRQ3}mPS}htG%7`}F3;LyD>q|o zgimaald1_silQiU6!{M%TYh*srttznaX|4>CgkyqFZFT%^T^Tp!PE+qNOIOMVBVKrzFrxx4bx*^^XtFSaLXVc``qvlZHiZU-?0fPa;!ym(KfH4nJ_>_BmU*O1f za=ManLQ3u8S$wRKgj-f1t=;wNs(M_HBvbhCFNPF9PNF5v#sFt`8IhB?x{QU#T56Et z1#1sme9)a;j?FK*G+U4r*n!n`r$Eq~4r_@xaWGC~T%3>1>M}!&0)}R==R{yL4hnY{#7}2ur~YeF zK_Ec5kQ|EFRpVv;{ttc33XsRmxvSl&8|OpuK(vD3K2qg^YvxuOp-$#IckQ(mnX83= zJt$77%sg4v6hAA=7dFx&lqGsVmT2)sPDm^%N+m%7Ll{CZu(gL>xZL5vS$iTU)*~LM zO60n_$SE@a7lvJ2BJ$a8$4U>Ykjm~lwI}W_27%3-*#%eE$jC_Q1cweCibZ#J>gnmx ze@&_Y1P{N=rTFnXgv)83=gPr;SMIEC;cYgx{UX}r`gh6W`JKzLeW_08{&mUD>GIg} zUViz;>+Ch~dcV6k@Ry+qF)A0!^U1E|XyL)#=J}Yu6-HhQ3#Y6u_Icge%efw9_Tmem z&`|Di7(akHG_L|ksWe-+ZiOKv7Fr=$7qUjuYzl7n6qrnHXVn(F3+LWkN#R0Q$dI^G zJLBA3F|xZbMvybc5I>?xsjcM$YY&2@xpjq7Szvv^3lnVyAWSy7b`xLw)}6MyI_mDg zGIW7$uz3xA@17qgZ6~-R`qy{wO;XnLx`ngI;#LUgGX*OU+FIamHFq_a-PE~5JcDcH zb6~q|A9|m)c%WG*T0(GJ4D<>6=%0MLsDUTeRoO*caEz))RCw=Z2$bzG+28 z_jqGg?#_;9#>E)1?|aQL(BKp2gHVz8k9HCt-u8(?`wu64a2*gaI0kpIzueV4;tuKl z^TD?M_2AqWZaI=%kJd_>{Y+Gd_;Ea;^^EWS%GyU!rV9i&b7ywHNhy210^h}|#{1;D zy1f57qJW<{lC9@)!!n}_(~RrWMR0TRGS2MsMURi{)UH$xh#R~>-)>&AvQHM4<3$Rg zVg6sFC!IS5FJ-*E(SK{(hu>!f0>lM?SiuV%^AI&{qG2eu!+9V`q!8imGn-UAg z{SQ871puxG?kQ=>z{}q@N;T)fdI&0rnU8$k*^)+fDpp<0v*z0_tk@tfY2*< z|E!Mg*cSo}78%^X9}6VAtIN*j0Qb!2#<;D_`{3=B+8`9*%=qc@p>>K2WbW@_6=O2yAvPE^G0@Ff=q&=YpG& z)n#{(aXw0Q;I8X_j0l7UEISYv5C;En`yJM*0NB+8s}QyU?1I6%tZw1uV7t8UDRcw( z19!2G-Eue=&VlRu((T`}Lap46>bS-ypMK6x_YVxD#Fh_)7w*j$GCmxa#ICf71dPL- z<#$M*?DcRLdS!kP%G3&_uV)20tLRZG4Kx6NvSVm;WA4trfCqGCH^RquXeZaD-3T9a?`!Yu?9}e;6-0K; z<}Tc_b2x7!yLF=zxI}j1mfga!EYvN$R9AC({Ji75xa%d`k=);x?*Yg5uN}+!Tvv11 z4IRhfPV#b{54OQE{QWU6?0p6H3#RmowY5mz~@D=Kq+94F;` zBDs#sP$vsT)kf~qWo!Aktght7Vf8_5k1JppA)OtVU*71zfdlf~$PlAlyLM@Jc12NY znmEH^6N`+kIKv7Qd)1uFt=z0A6Cp}>XNRB}3zTD!f+CK`EkH9{xml20pzs$pilQjRLMN2oo@MFbG(0n%7@q z1kfP;_~EC`^u?sq7HsixByl}{nFXmYqAItr$nDNf&Nbz`w^-85SbX-=m%ESJeKFkn!JFwD-}w(aeg2k>W_;XkgRIJ5 zefQhw%U}JTImc4F{#S3lo<8@5Z%|uXi(S{&wt$p5L-PJ-H4`eS;J%kt7d@aHS671j zPr+t$8KE=g7boJTq^^RqMHDbh)$idLAJ?uz3Z}%}g(8M3dK5)b!sNNEr0AKdd+UyG z-FUYPw`Zo2B(TzQq7ljv(Ng)+Gj(Vaa;&OYk+U>ArT zE@a4o&8XPSwS0_)2mg9N{KV+j#V0hD9u91l6xH`pY7Zll>G0vhbn@g$8W=iT&S$^UY6`o7+UyLC6yD!{9ecu_RE+bri@rG-I+uikp zN9d`ichUt$==ebJz;a`U)&*AZeEF;2w*u(;H(zOm73Se4yI{*E`i2o(XBw+gt6F>J z6kOH@ULY3&6AM>YBsxDN?k*bd-&GZ>PgG)EN_2=IBo1oEO^s1H?LBpD*Qs4*b@}H_ z_J#1_g^2TktBb2P_x1H@j8Bbn;=~Esvu6+8dFS`(ymQZ^MT?i%Kkkn|{u3j9TIlt! zznV5~+(?Qt--&aI5r)>qfqq>)#y<}e&FX^1=CVt-1lQT%W@qJT%P8y#(Ieevn8vnP z{=h@qX}MYccVE1NKK7CKCdZR5Hf8oBVtit09N}d&t}RkjQ~2b%x`CkH*9EHUb zXLS`dC}rU4;(a-*3)UVU09;*4ZPVZ1&y8&9$wwciAKvvn+VTAJ^w{H%(eCY!(KFkh zr2YH%)6ShcNl|KnD5C|1A3W->4+TC?lq}H$AYLF+QkETW08+^CwuSv$X!oA|HWo-8 zPnmlNp+jCsOi`ug*_PFnifR_7Rivj`!6aK&z}4kvb&0-RzD20W;^WovNv-ef>_i6G zEo$+p3`3_!Xn1JYPB%3-QFCKMa2pU>7oyjgtD71dsky1J{r-bTX!XivG*O;?_E}W* zw!-J5w|7l@MVL_Ga_~d1E1?T1c0BwtsHt-U$mI#PbXf58FS~V`HQA^qvFMXO=DVG_&< zp;QW7L(j33w6Lv}+S?0p2+Ni%qJ?Jq$!B-cs;*AjW3C7Hea>0yl4pQxe#%T^pWS;8 z*=z625G1(AM;q(yA59 z2)~*TN%$OwhezzS;~H?z4QH&(yjEpy5tZn^E-N_8WufCJN+oa_z<%krZrvKQCx`=_ zId`1`mY-^o8RH{;aH7g$vIqmz9M9usaf2e2ovl6`)VC@;BmtNXS z3(QOY{r}~c=|?~M5&hyj{}cVuAO8^@KHLrWs2$)E%L5_{qUEY9USUl95Fe2h55uQV z+xt6drg34X%rry>EHAjW!hP;F>kAgIi!VIe3b!mEI3S=_t?0D(>K`YIUe_FT!`hX} zvPQEHo;A+R;2Gn2V)@X}X?y+8n&V;~)*;->ju&2{3(h;!F3UToGPm$Wjm+%I3v`zF zF+Yk@0qmE|g~CYDBdt7kIn4m&6fi8p#rYte9kqO-?c29oWrt}aQe^Q$yE(hQfDt>1 zbq5uj`H-mPW87VKUsu48v%0MNY$kI3YYA9f@N$Oufxv-CfThQ$VC6YTJB;h<_I zU`O*!c=owxXv2mxtR;gjKINpfvy}&KJ=`b5aXNNgTOlS00s_|yw^K)33-uUbXXAy8 z2*Gt{F=U@zgWY%a%B6JQ!`oBG$$~9u2dvN>u_A`A#iz-AHf%lW>nn(+=H@1M4I>)A zF;9^e_)t+o-_0fWQIslRg$)cN&;@e2(-lb#A11DH0mDq;!}W!tfK60Za&|F3#EnXw zSxW{sqlOQxJhGOL-4+#_QOwYZ9&**2lbd(0uGokv6+IP1x?bS&f`tHaJ`h2GMGLlr zm8R$DNw>U_I+rdqFYJAG9Yhhn)7Y=Ke}K+y9~U&u#^r+L5Iqn{EP$|o7K;`wqJswy z+CT1{ON~3{^e7!a+Cvv#a1JeQt)~;mj?$7vi<3Y8*xm<1(&v(pe*5Oe=_&%<^( z?zO`VPZ|LRk%4th&Eo;tuy#S@feq`rtXpaPnKh=K-*d+Dwf|6$T@F$A@Z-;0w-c;m zIIlcTR;a%J`- ztv`Mcvu_E?iVISd3J|*nrB)AMEXp${Up`*szMoaC-kg`!Rit_3FOakNNUKnkam+{* zF_Z!*a>321{X`K%SdS3jWBY^k_xJyg*4EZrNtqYG!t%&t&sn!v9?0r?Y4>jWi;rxh z?x&xlg%_Pmm%QqyXz`LxI$&PxBgV?}mS6k@yX`gCTth7_EmVuZO$ITC@&1E7nN~Dq z?vs_(C0kR3TwuN{OtZR_N+9d_;D^ipj%8-J>{CHA5bKi}SsPj$De(X zhRyHCvrlZNM}GVhdgzYt(S!f~1A5_G->2^TAEC3)KD%Z`k0mZx+7N@KqG$dRNHj@^ z%4|)c79XXOh|cU5Hjwz4;eMcgHQ({MC-SX65IVxj6HHO(Qq`1EojH)vWyhjPX)Lz4 z>k-_n-PvoGm-p_a-+I&a)br9yw7Wj0#`=0%HatqJ>Y9v;Y6<<~?|z7`dH;VQMVaq} zM2o*n#{J~g@u@ezN7LDz9iW{Ij@#VaZ0AvZnF|!k>6zqE_c3aB)uJbh-?X(H%DI2P zs&zT%nCNr$k2{%r$-}q!kf_$oZY|!F#^Zge8sn3f)#c5`tlW~|BpMek)~Dx-9$0?d zklGI}E?2o3Rhx;67ZMfq<&x)8YLqqW*3s=xKSuxS)1RUbz4Mpo*Dk(-UVr^{^z(oG zQTq6k531-<6h}GPT$KuVhUh`8Cam_HVYze9ek(KpI`b#81+sREQtz>cW35=+gyj53;(dsRotV$*X_)opw+ZrGOvE%=m7vY6LdpT+5a% zqxSapOm`cakZxSRnhy0qFvMnhjcr)EYfqtm+io*HbNzS`+e1A^?KXv%b7ZDhndzO@ z!tu5@zjjhq8(25cEgoZtk;1i@<(*6H`Oy>=i5f^A=;~Zb{YKEOSiaPXxUSA|YYYSw z`iu=20fA%!%PV?wKk8L50-u9L&?YXeeu7_Fd4BEP2X!!cT!%tgPfNMg-)`9+^ zq&3JlzH_g&PGNsMgI8a&mC)Rkx$M%d_PUrmjIcuMk3-!@?ff;bzl475&WZbr+_mq- zdCWB)s$l1P=l~booTROL!T5=lr`qBtFUBYG>qYP+gwD2Y+fuza*sq5U?S708XlZFl z{l+1_$JOz%y}BA28j#Ws0c2Cz{XSh9pK z;AM>$|En)K-@Js6+D4>Z%NN@h_EoRH*uKay&F+`;U8i?!gMA?~AQ<{wL)6IvL4fl> zxIm0y`ur_t8sW9tu7hx4VTEgGHv$bp0oMS*bN-f1_Sz4-`-6bOy2I9$wxDpI?iyiT z!7;dNJaJgbJ?rYI8j7)8&NSHLts*a4d6uxMr+F zPc(c68->W6E<}t^rpk7;0XdG?ei5trj?ZgS8na5{RCQj+>1n)-)7CJ+%Hu3Oc3)(4 zF)pJkW;YkJd-rbi;zCM$B;tJRz*U>o;-e@^t&=5mc%47avIJENh!l3O!1@472wXg{ zT;RpqYONVd?EtX=p@i*X_s+av^Cr9f1?Gi~mp_072-EGYlS^iRXuuk*ee4S8YaMvoVv(?(E4rt>dF8ibA>ZIx=%B%iSfnt3(|iMKRw? znEhA>Nc`%}as_LRs)(Q5?(A%a@&l4wY)>SDpX~~CW_NC`WLpw`K43HAqn^fN%5wah7OzqNl3C&AC}!Bovxg!Pyopl#V;k z-IZi!88=t*5a8lMW*295A-)IVhZ~bZ0CAr#c_HfRQWQlgCQrOb3lCRwu5=LT$jAsO z3W+{lZaR{gT|m^Dvg>C>j1P%aGk=8(c#dRHGyHk?+;flJ)&(`&n|S1rM-q#VvGUky zXYGNzD~9;t?(A@N@wT##4>G$*Q4~c<@c$w8MGem*dyA)KJBvz)pc5xfBvqT|16(&} zDwU!czmoHG;vesD1q&0F0-hgo#t#t!6T8k2EO;W}_Nr=p4=82v8J`(9rN*v!Valb2 zrll^LJ3<{F&gc@EUH0bL-DLz%QneWtA7|~c$8_#4+n^NjJ;>~ew2qIWC`v*owY{9} zLcz|J0#%k#;TQwAN|-2<-LKT~a80Eu7xT6cJp5Gdc4&VgkL9<4uvMG)yqrvDwTVF8 zpih4KD>PC3omF7+8I{QR&t#n+mNN9@?I2>i_}3Efr9_5FGA4ZVUCH_#Rh$e#(d^6aqw_ zD-;mKJhs6;sJnxgJSq~S-2_@Sp!lD$C)mo+rGU`*hB-e91)_f!7Z^kp=8fxDr?%a- z`+zL~h{A?%d-IjHodm`JRhpSXU#?xYz#_M!z%}8zaerK7@N-|h!=3}h1UZeqP(6CQ zpdy6hb~DKPGCyp)i51Hio9lVnUOPZBLtgH4zAL22K&bi8vTuNkAg;32ta0v*8&=t8 zg3k;0(^Y0OQ-r*@UR|lao-8L4rQ?E}XN|=tk!Ho&0TEYkCQ)J8-6)h^QokqEt4jtr zv#SgH!{TE^PgGxDZ%^ieo5v%n)#9TlN{tf*8jXrriDX$Qi>uN!G&IohbbMi|-aHp2#U zT!54Az3*rl{C$6@wJr3CZTP(WJ`>W92qI4x@|5*ZVZGpaNI3{0PVS1^( z{&En7jAdieL{E@2g2>FSxXR5+OU3y3#2lc++qjC&$pTn?WXlR38%vKJBfy!hJ>09y z#rW7%c5YdL7$1lqEncW7ic-KAGk!`t6KY=(I?i;0D$wiJt+Ri(xgs?8i-KDTO+{g~ zfcpqm3J42W9Uvs|vhK11XK7*)fpr6x9t?Ql+qMdmu)9lJRw#;6>%86`(2n`aRTq0Bw6D+g1$cqyuqFEt)g(%h zL35M&FSEhUPZoR6L#OGq8IOq+;MoXXboJ(58yf1(eK%5b^W{ada6!laI^b` zi9)_w?&HPtT)EkqxGWtV9d^AR-0UCA1vlFoJ_vAjnO**N8rWPVhRll}rqR(k@&d#k za}0QKgH~r&o(TA^_x29ZkP&)w1zhvU=%^h|4GmL$e;qAe++Owz9s48BVsLQ&q#oxn z9p97LM4Z#C6l7I(W~Mu^Sy2MW;AUUwaNCL*$?D4d_Q!FmK4F#^NtTNOhRJrq+T-sR zwD>sjW2cRu*pAH~E9>r3i%;e0GlFM6ya*eN34hU|wz5P3UWUhy_stohSBj2~R%_MB zW4_(b_gH*$Ovxcz$JTAu=ohklT=`)@ys!4 z-9cyeNH!^z>yXwi;(WA8X@xm;YM6Ta22#uJ`sPXc*6k;)`0=^uRV{SmdzM%sG@EHz z&`j;^Et%`#-h!KGzLEDbKYH|JTJ(&L(AepI{F2i@{-4j%U3cF}$9wzDvumZ_zwM*6 z>Fjf;uCaycn_6hTQtEp$o5*u<$ygt`PR=t}hTvuuKFW0Cgb%+kN&6w_7(_ z9&-JE-2VtYvtuVs7YNo4F)+ngrm?Y6BMeX4A&yJC%KqW@J5uww$44G}+FZwA(K>&- zf4KddG*O056^lAw=#35aC(}E3?xsd_ACEo$6z%KorPo}vnGPH}n5-W)+d?Rng6}H} z+-uK~iIvXBG4VaZfM zz2ZV@Z)&C;2ai+Z&>+2j%UP-YM~6yZhrBNe-pj--M>u9{YYY9{TYuU-t5=%iAE2*& z>)&ZQ$r8wOfvh5havt$KqHr83N~K_uWY0Z zXSL8n_nu6a!R?h5>o?y>fB5I0rFUHS)5&tYw4QlxCv|o%q1V3pVk-(>GA}c{ zFxy&MsQc(qJO9S3uCyt~c=16Tz2mxT?YQUYF`JJ3t{bkWQrH(Z;u*Zm9n`ET1q9B% z-NVWK!5Rjz?5xexQ~#vZ-0u(X|95)VjjQeb+ZVd|TX6A3TkJhzJ?{4}{^pBTG`z_? zlZ#)mrL1G)9#Q89oc4rXJd%FVcb>6D?fu|$=Ij3MT|c6i_U^Z0=GWeRBONiHH@*Y7f866Q-SAdgzizenoEA{$ zG8BwkVxQITee8eR?+1kS|Fsse*nXe*J5dnRyXcIytLRNvUs*DBJ?nci7YJE+7(f0; z20WK5fY63QQ9_^&R9jnHa2>2ZTept)Fyowc_dQGnI#-)K;Zr0yLIFuG#)nhd*~FD> zWP}bm!9#j!?_x;(HRFZs>qY_!9_eDmLF zxp}$ag#}9y1QW!-=Ce22m)UZ&J^u5@G#|4RG462^?=!?PLFD}QZ7Zgn$p)(pTr~`= z7PxN!kq7Vs|8*k-@J#;j6Q8Bm8cPs_4_tw_{`Rlim;7IR=1XM@pHwjGsBfhjOUHk- zv{8Ke6fIx1iq@=JNr%n3UVr7A(l67o@~vd}{Qt##Zh!szq*v@5( zjd*#hbz41ZK11B^!^S=KvETduj37E-_xr8){c`T{*PHK0&yl0_Z+HJF5mor^eAj(P zaDE&EmPCAiaK2LD8O=rE!^oJPIHSUI-Yk6uLsVVcE}#g4NOy;HcSuMo-3%~;)DY6$ z-62RfNP~pH(4lm9cXu~KdiL{v=LfJ^d(B?=y5kCO0)YQnz(_40VUCTGIf?9>ho|rR z3x6_xE-Q=NNln|_b$J+Q$rSW3)KTl$xenQw-vVA^bX;PgsVy~KmqF|&+rI>OIuG-Q z5d_+bqy8ht={yhPuz0?CnB@4|5m0q*r1RQ97T^FztKP@qGr428%{IBB{p$$psK-LW z62toZ+vl!F``tSt1&KQW^uANDp04Qdo$Gd0)86C*Ga*R8@Y&ec;`PsHmkmo_dR80+ zgUef@s!%0PF?w8%as0nqF6fIT;+6Rt`Nb9|%@_j@s7vmZF!gm`D((B`F`g+b=5-tf zZKEC|IK>#KVGym*F(&qpq^M{g4mIkA^-7s6<1Lwx?O(6U`KvfpWb?HiCUFzpl_oIn zl|mYo_Hd)(SwZc01&B(-2PWrV#C;hVvB#Tkrn1_sz7aH|aHLcE6J}?NwI(O?EL_p5 z+wT3*_ZwRe&T(ZM3CoPE;-5|WYWoi*t9ZL(^}Ly-A>=Pa1VwYUjIL|mFTdY|E^u)~ zsZ8H?3=t!t%%Z>H;9>2GdS2LbTTH=YB|Z*BMQ-g>zh2;2wf&cmVoo&CHJcmZuZ;{N z|48@|^JB}-DKXnu*Nem$rho0c{IgzGN0H!pxL4gvVAR3G##XI;56Q#fzvqu_yBKx_ z_(LM-$Y+N`Zf6w&NC3Zw=*WHzZK(eb+@~du)V7hGLUbw-Ah*3{6ahFW^lJVU) zzcdG#V}eZdU@%DSCq#zHsDN<_WAz`Zhtb5VR}qw737EBsbgKQ01|(TmWr_oTv=QZ` zmhqdsyCb0R&j{UN@Y$5KRtbbX!AsBLBC`hrCEdvOqnbaO+u0QDH5Bt?|GB)EN_)&N ziXF@EwYhNGE!;xKYpq&!(dg(wY{2Xu|K0XT-`xGw7Xy7ZqmV3_iO9`q+T{aU^bNP5 z^}SWk?3(6L!c63JQ4)mU;bD?=Z67M5SHm zbP#V!Zg8sGHz4(=AA6bktEUUbcV16VEc}TB@zZDQ6_+{DTGI{5=BIe{gYnh;>n^`1 zH_ziH!u^DtM;yolggfvOd|J})sHo9uWKMtIpq`z{-bqx0w5t0m**s0j5}6da~>6RK%93YYPBXP*QL)slP?2cUGEi?vWVJ z3xvJSfejyfdW@l8Xa9R*ao(g0oT0UPCMCz{yU46QefQAk_r%P{Z{8=|u^HG7ZA_Mb z`+ecfqW900J}?!FHiPNH0JAk}ts_kR?ReITho3w?YbKXg_Wj)A{T9wp)SiIypj!UT zM~>u2Cz2LoSW8s5YsqpZvmRrgO9D8Nvb*#F~$ z2wGcKjv`>s6?S{p%~}9>$NPTI7ky4Lm=aY&vyq;eWEYK61R7; zOqd)*(jhEIyo{xtTok|RTdB2k6FzCGW6L|T)w8Y;g=O8=6MVC@90h-knK5S%{I{LL z1|b5^*`4YT*nPA&6!$r`6IXCszTm^*>&4No&{Oiey-=AH+RIe9!e2uc$Y4rJ!|&VJ z4Dmh2RE>&Ne=9wNW{J1aENWt#VfyI)Mz~fES{}vJ#kKG zZ8FGYlpfng4;`Xz6EWmtX=s}1n+J!%$h&$qT$|SQmh?_IgMTA>RJi`?bl+U@^yk-!-t++ zsH0e#4XvjRK%=+%+f>N_dU^7i_K?~CJ7sZe1V-Z79jk+ zCrAnM+!msCrSfzq21Bar%S!~6l)m5gq~R0tEN%F&Ml`aW4~74njk(okA={;CLZhxg`U3!wK z{G@a{(t3HCyi;_xm%*({j%1p)vhpJ4E(2DaG^j2CwpvjtHxezc8vWOCA}n41#aWm9 zF5cbkwe*PC$_^XhBrG@B^>fXriSf#j|H+_LV=S{qA+1Is44f0L{=K?TUV` zN7};7__(3|d&WU$;}?Pr1^g)G_g|vWN@EoCf=tH9bL{PEvgk>J(q>fBS(AM zYz$CORH#djL5PFF-k;c9q%iysa{}ubU%qr-hcnvHL+{^CUsT(&GkKoBBA}d0O$5iX zvt)zGbBNlS0f1W~l83F*-=}c!X;e;Wqns4jb>Mz_cGfryz23TjA!qeViz{W-tHvqR z{4BGFDr~vu=i>c7Oq+(O!B*PKL%1}%+Al^fW@bY+shR7;Z3523p5LIfkb3+T9X3uE zhaI!mf?bOy5>D#e1e7hAuQH4-jNX$8wj-euL-;d>Qq5*4$`~Uz`yJJK435y6ynoqN z*m*y$nv&$njPlsnpqH#@$%pOi#n}Br5ruY-rTl&H5pi~-EN=0IS7A$YOfrZ+t4!B< zyAn!h(WA69TYQwAMzgv9F-Gl?pf*^&-35#ceV@>;%g#pa1eaEP z5=Yl+BG_A;(NC7IYU+*+*BEj8!W@UlW<&L9oImiO# z(%AQxa&{TZ`zPc6PAOM|=!HVQUR!nnyA{ajA?;mVRS%H5Z=5=^b68!d{YO2QPgl)| zY`s@yev}47bpE~=o8;=s|Gk^|Tb;Kzj8PV}R99C&f5P5k%5t&@%kh0WcWpTlv1^=d zZgxFewQG31=u(+LGr3r3^yOo^LI-)flZ3_z#`v(sd0))8X`O5r)=VxKWye>t<{b}? z@EUZawaOMc`W)MF^pHz>q_Y)x*Ur`0=O3N8-C#XQZ;R->crDKF73YS`!+t+T)DGcBepY5lnUZ(>p%92~1CaWOUjN&5)xWTsFv zP(rvR>wP20oYkS!bcS2Uo6%BGcFXvKt0W-6G7DfRC1L77&-X7o;dZ8@Qv^+@bT;X` zG)R7aegd;CFTb#SSFv(j9^;@l-FrNm;a`iaeU<)be0&0cJSKDC(G-4!FC_QQo$gh3 z*%w^`$^dKqj-%?~=Hblw>O7PyR`+w-GJk6PZWZh9SW1+{XzcY?tGUXzy55>_fU80o zq2O$O*FfCm{4)tTSjEaqglW*(aY{)*Hz;!WG-is8<(Q@+k~CL?FU=zBpk0D!{GPW52BaF|=0;8y_5OqFp`R|2G z=)hzH+8i*~H+GDJ*Y;25NmrBYNCl8)*%wD?(!1WLVenKZwa;NEQE_VEsnpR?@5i!d z76H`3?VS)!oo}wMHS2!m1G@mG%9e1DyjZ)9Miye>`}E5{H;V|fmIKXx*UgI$VgMjm zHB9~yDyCNhh%jbgia&DEHFLEwpKU6Hby_)pb5OIiiNY#gsglAPSn#(LSRpQ5;tf8t zV+rHbTXP>T0?H_7l9BF~Hr6Q2P~awFyNo2%lOC(gTlVHCTb@CEhrV z7tjy;H6n{pczA8SP=oB#BAD95c(bAcMR|UHZm6xVUzG<|#TU62%aZtueJ`kE)RwXv z&dbjq4@fBOdPQ*H|1r&+(b8<}i^9zo)rFjj2Qr#?Sgd_6s9mZax{MUek3uIzn=k>e z+G{JwnFx`9!brZT*Xhsl2^&)rna^^Et!{Z${HB#M=X$b_rxqL(mIC>w zPqPsAssT4PKAi~qi0XzCQ4)x!x{b(3P0nqcawGMtKpVCqieNf{xw)N$WdE4>coVg* z0j*c)Vn3Q{%&UXLmw7u~HKwSsgE+25zY;g)x7k=j1Pw%p(>UOUf71vz?=aNsEV+$c zOqOUwvMcO|(EQYCZ77HKLe5t7DjR`F0~D%RO)zz zdD}Cm7c2sgopS7D!96H>=4Sy3yxiQfR{=W;`j!KE$feM*&^PBe&L0OToAl07gfQabjmk5;X5+-Qz;qH87xd!4N1*N9BO1kfV+ z1<(@ymT*w+7xm?McD(nxr6|g#Ww0<@*HpbmKrL7`xKgl(Gm8G(6!W~k*gJ0A>vsSm zOq#9t1nt{rLbe9BnGPReK}T!D)o=>?ZTQdXAK%4~j7-=vzD#t@jq~sxMO``iv1d16 zs)c8|(}Tf^2HHA#-EcxVeimqM*Nrz&pJapFOkwJAgUZ`Y^#I$`G&7d`$6yR1k(J6Z zs~CPlJr;+E+gU^2&d#%FgosrQnaHYDjGI4&-z74>L}Z?#h=VeAmtcMhA#L#%S^*3* zlO*^4we(oTYOd+3n_Y*^ywBxp@&l1HoX<%`=I&-aIuw(rn^14LRY640{PZ+PT2clN zQfkzT!n&u}a(Zf8kIg7OkT3)TgeBndA+SK${4xs$qtDqfo z{=Ns)`3kM7qDJ?ewDO&Vxf&X5E`!+b)53pwMC7$ZuuV6dT~CK&py0LzBTBS#QH#%q z-{9>t_)<^KuCB96cTz%@%Ni{J4SBO*a-B>h`q649@9U(@7il+SxUo z{yl}5JjaQs^vv%1hDk5Rtoh54u3bnjpgd5JKDfO^dZ5l;i5TO2`*EdkUvN4B6iFSAVwu{bF;$ zm{!@+&e=gHoSKg?(KIjTiCRL#_^O|tZaEdH)yvGox1mlsqXB{3mf0oids*`nqERN` zil_=K)!Wb#Ncx8;*RhFtbd0EF{?o^O`!8#mJxz9JHzp=#zOcSn>=U&o>dL~7gHUf+ z?93F8=xIp`=Jjy+zLDXxhihLpQy(@{q~!bw(x99(xWjUv*;58n_FF2 z(Yb9q(UL^1#6Fn=0X0*>I!%`lRTVprEn75ta{dR`7FzZbcqQ9{b?f_d-&?4`t4{z6r{FaEv9&aduX;AmOBuujdOy zI8Kj%k(Y;ZD0h}m^AY@~#YXO_Z#sv>a8{G8!?%Ug00UtU&s4KW;VTNUVxNohI=K76 zaj7f50b9;zO^0kg0!oTL$JK~T>7S@&DyG3>M8BOdL?7+pAM zWyB4}AnfHmGw8Qt9qg1;@L*F}s7BjE=hOjNksqW#-eCtBTB4S1ynUmvHDC2)(GJUF z+`&m%s1GcbN~T1JQjwB^8myQ2Ic?7$D>bpp3?{GWU^8?8Mvi`!8d$wJ&rb=pF_{zQ5@G^Bfy408@bDt`1ZyEPa0fe{yzMGZdV` z;_+D$KA-4nt1V2t@hU2^u3S;#hjRr;-b$MzCz0qt%+a_L33b81yj=eakeTdoV!IKJ zg>|a}4g)xv4iqF5Tz+AD%C0z@-_kRVN?kg-vxm+>JCXc(l2_IhlVqK5a|+}LIquL2 z-gKJ(l4WY5yzg+9c-RqiQ9`4_;|z`(jL0rf(NR!{I+>4wl5I&@@`3gy7fcp%eQIVh z)FPU|6G5}Mdzp2!W*CaeJH5!Eh;?4}r;ZZ!bMZao(-RZ_rI|_XDptRH(NSjOdqe5J z*%f+>k(Cx};DgYCx%I}Re4uw8F`iQ)7$UiqxXdvKN~x_}e}7W>$Q-!N1q*m%@HmX$1hdAW{`0r=0gzFyD_*fW^i( z`+4Eh;Btp|1BF1cbmcoHi2m}Ms61W6wn49{r6^enUvM*l^tm$FAGWgKVxdwlV(&EY zjJ?$bIEW)yjhP<86ZPYW^USwGXQn93gVy&(_R2X}5J;ZeYuiGAjc!LA%thE87feLr zK#l_-X25JaXWQ~_MLsZ1)XnNJi$X`JQ1q@M8ePVZ-LHuk>d__`54q?ntk<94?3&G9 zX>nsXQtj)*=LG!Ux>pt$kff!B*x9EY{r5}psvfh;{X;$`o6?tJ5Z zVxmcx9F*phdbXV%5D-9l-nvG}?%3$UJzM-MV1!DbSMJH2il>E!A=K1WXEWImaYV+J zBb?$gJ;@jOked8c>*5W|!yVfKD`D#~!zOl&eA?D65Eq$#9QC^P|6Mclj zme2btNlt_l(~vK|*;I`1OQ~7Oj);3(DJ6ekxIC=-f{wKefqSWb{i+hljX!*MnBaFo z=vOUL%br-zKFFfEatx}0hCCKVY$G>wPZQL`fjy=6j6APtrt9!&^xu#MD5(P>@N{)5 zC-jilWgWyv&HE+TWQ8?tKk)fiI#I@6<)LfI%gM5&c>Ej|^XxhM6o$P%WP|amr(oF5 zdrtfF>wz0A@&5C4jB3Kc7{+ZO0GCJu{_?e!#r7p$s`v8Zb>b$k~ z3`_K78r``cl@edH4nrC%@R<_GtJpGyF#BnuR~NDc^X))tL_}H|sw7_b6-JT{*5hy6hH5MTt0W`%f;TH=^!*U#o5r>$DoYs$;Leb{O~xJ)7xado6N=w8jgjZq&d}rZ z#v7uwz-{T5MkA3z)z9HQ=uH3IJ2c%In>Ygia@<9AQU(MzDSS46OWmv|hAVIGw_Uv_ zU-aYeCH~&%=eljj!^h0(`XTM;Y*uE^L`Pj%bHZ%-uNQH|!Rv&i(d>pSO3XwpY+*c7 zyv47Z`e~AnP)G{L_-6DiX)aXvaTfxkBux@vcWKMrf)#4~H9gul~dF}g{%?1dyDz*=y2P=&!B3MjJCzwIK0Ld6pU z0&?T?iu2oCdiA7#sv0pV-zLSV3`)wxa${3C`lvt-SU_PJaJ(3b}XB z`n&p-p&`z;zWn*g>2VJ|OT?L7_4H3+^4|C7qZz+%l)7{WjzF$(ig(_6PZFtbrDKTuuteP%Ojd8^y9Hf+_@vIj#GUc2 zzuP!dIweOuGJ~TD#jkHWaf#OI%tZ?o&_01x_k0M^;ZmLV{UxZjtjgWRJ9QeR z%OtKRqrh*mUn`* z|LEq)+mZ^4(XvtKJ`VVp%*oTzS%-HTGdZs^_T_D#6}3xAW61}#fGwg0FBJQ#_x@j#xLAz`5Xa!_?A8W|B4l_xwa`i}SS@B1va)s<>i zg|4anQH1WtuYZP2Hn?g6nT}d`H59wA;~LvI7n@zIX!gilNiffEm+Vx;*e(!gW6p9N zLc)C^bWwyg_oKQ#^qPHrsVDNbL5spXUrH5g2HsHTyQ&E-SVz))0xu}jrY1EhWW(b& zsovvpzxy8GjTpvC;*uv5#Gxq}HmAGVH>V%t{eE7;`j1V{2l({m|x6`_beBz zlI?7v-|FJ*)fPZ$;kt%dTUmAa-OnZ=U%X<$+%)@pm31aP|Hc@_wn8R~;5%`yNrgxl zCS|3Rs3^!6p;Kj8B+csd17#ag_{XDI+>g0;enwk@Gl$fpkHgEiYT%vC5GoBgDo;|@ z%^Wb)B{l6qLC2y%W0Ab0s%Y=%O764Aq@a8!Ca7JZO%9k@ZR|!4DVcNZiO;2-1{=t` zEAhi5(W#gdkcq#RP!|rX$b&EgKaJ~h`1w&Xm276VTN<#lk$U9yH{>N6ArG|nj^BIu z%>H2&lk7t!|GjMUWX0&R@>1XH7?hgL6&$|)+|=PB;V;wIa~Mr+803aI_&#yR}8XNT7-fjP4ZDa`n`kz z^knPzRV1+^Ii<+o-0`%PQ#)2Mm&^&|qu8|iHQ~9emD!7hIxC%lbu(vKzk7*42=mQ5 zir=i#gv!Q7R~{A2vfBvUwH*wM(n4(MNA}JW(`)( zr1o@g{w#6NC1!BZskx3{1mKdTF?Fn!jeY2+>hqHQ&(#u3{kst=Zg|BnDo{L<6C9Vh zpK#u{ZC{l$I_he6;^Qr{p9w#km}d!$_JgB_h9l7-QbI=xx*py~X<+)j2~C1Ch45?z zj<)HS&eH`L!>;1NnC~qNyIdGcT(+^Grtm^p_NgMH-Xwc(i697H6cpab!uA}z6gT5^scaU0w_c84? zdfX+tm5*1$HUlN^IyT3~ItQ2dRZ^i%joY_UQX#_jRo72|!FqL}k)f)82KpI@u-saJ z|4k4V$VN~@Avkw_(K9vlDDqVjaY`fI+P~SwE#t3}XD@lsNei5+{r)6Z2(P}U%UeHk zbc&Lp?Z6AF=mj?8oSZcJVytHsUdJMV=}T(!o9epP$H?wXZe&|Q90_IXWGZy9GCZEF zj>)eyR#tg7ytQY^+z<=bxhDxUe^zu9yA+w^mXNuS*V+_tFN4a*VYdbuHIF)PLE7QJ zy{v9JpIJk|KGbmxXG~{s-64gZ_t2BDM)KFJCS037jigA@nEDRB&vSk&6zv~yKbV10 z$y$Gpc#$0So*VS?Zn~VG?kbl5_TFJJxfC|MKF>%p(XXxK9>RsJL)>Vx^Ndrp!H19P zmhy!#xm8{i{)M0P*!!lGRP}Lu-X9xdROFRtM#IqAzo^y`CsVQ;g5FoKv3+Rxls}q= z#F4D-5}HoW=>32a`WdiLN_$#SElOc6>y*AtF`;MYdDAb4hVhRC_Bx3}(VLp~jsxHD zB!|H&W4tbOS>?s$en@H#k3>$*axUAO{p94N{t=NB`$toVAymf{>q5BJq%Vf(71*Ha zxV_V4mZ6fRH>n7_f#3r>vj@3xi)Q(I0hSa3Qa3wm){UU57{-T5>H6#q*1^L|?9DK% z`+`V9$MlSpnIz%YD)z!FZwTYq%f(7jx{|XUO}x`HNP$r73*wOx^654cUNKNhyqDUC zrZa@+k-2^UTb}9c@O9Z=70^%?s+kR`6nCchCstE`70aQq1Ut>0u7hu>kI~tAd5HGU z2hJNQ#9n01|0`H<>3M_FeY#HG>af+zm1cvHAF@#N{>*;wkFgQ%UWDGoi^}Wh#+QEF z1lC~5vl}`eANtpr2`Qtta*o7niQb&{ZboS^I9|uDfQJQOB2HiUk)Cj?i>QN^(%{Ru zNs(qiXIeCru{d^KkpQHL2lb7IcAPgbK{4VKrw=tweW?)>8D{C_OHGm-k&{{I7(7RG z$7I;uERRe^H+e$gx}%$UlAv%l3hP#`2=kt{3?T`R;qaea<`rL7fBSmJ-MyromV(L^ zX61+lNU;QXPIr+C73sK9zlf!pHgV=SJm?=*n*QM7ZjrXbdd*8KoCOxgX6Xb+j}=(j z=&IKISo`xzwALf?y95K;GI$4%D6^!eIMd%&hAAwfH-*Uw^L}!`aiHG@HhX_0l^m#k zv~vS@;J*X}`oBh&`wZb56b7TNX_T5LG_W2$<$9$M5bUiQ{|;Yi52q42`F2jnC2IEh z9p3zVB-9Hb=~9GUx?Tq5ucn`g@O#dFR1v9B{pqt=*N>JA7}KH#2xEf}s)!Bc>mynF z9kowb@m%YjU$$T46LJQ_xukHOXez{Tp$^2)Rc3jH`lY)C2tM8P50@RcH$31(wKU3- zjJo=B2#X`}xrzSMP5h4}x-mzZuZNd~B#T9Ox^rWFIpHB4gbW_q#XqXvet$kuSSiEv z&AsE`Zov8*luBW0+p*cjbR3OttUNi6U3I*yA4jUc6@)8si(|2z^kMCvqfcCT+}!bN zHEH$)g;fPq0?~_ins)3gW$0JdyLlHVR2-V4OI&%q*s91;u{?bX$Lx5zn>OGeb&$8Z zz=$>_z<;Jk0?S~%D&)?(Jr+^ivlC1Bg}C@7aclSR7_69(Tj3H=U#40Vjh3@mG#w=} zgpA*ZJ9ZiARFnPm{A?rj7+NmjuHpAHTQe6Vs+wCcK(}axipBTRPyVztBn75|NA3KGI;(>jyIi z7`#)PAi=LAw6X-qqwzL9|MB#4iGir3HRvGfu3OP?$ieRAW;)KsCOTbpYz~nYov&HvWLKiK zF14a}gz&bepm^2_FtqwEAEjvW=Rcwjk21yULpW6#dg(DPfHyv_Okx>)lzN$5mBPms z#}XYoo!8K|!2@~C0#>e)bz{<}I%PPje<@h`<0tjDIYg*aJW4>&FEz?85e3K)90{Icf=)DlVL63jsjCZG9*eI zFl4z*q5EE%(O`d3s~eaWb~NTj)K(Z8-Y_)~QFW^bb@c5Z2`t{4yCy@hYKOE$+Wox?CD)vnvm|*Xb_&21{XRvax#+X~`ed;!2hbQ%AZVCc$e0Dju1-N&B5v7E zz)19Dw~^vZG^2aTGndgJA{s_aNHasvT+N;=+2`dx5L~ulGDD9P2gR0R2sBM>Pz;Mz z?dl^Gt9k&$^m&%+je(|eLw#M{iev%;LFzz(7wvpwe*TDAE7eR&P%O2rDk6zb4FLNu zx3c2Sgc=Adj0~AKt6k(L=1qY-MnMgZlZZEwd76}{IZ7G)CyrcVzE7;^L`#C2kvS5F z&0ys^*-|1(_?rZqbAQuFGI7nh#Z`W(D;5DKOl*hYX_Ou5*dA1hmP{GUR7HE(81Qb& z%rD4Dd-E3tTYnC#7j(yt-3Q*Mt~Na)kaH&&3`A!0l?k)Zcp`ua2_g2$y-$oX?St;U zi|j`PS`+z2r%l>pRNJA&3v; zUr8vHGY8>J8hB8%^`B;MTUs0Q1BX}R?QhITRMdV82@oJoF?D8iWUtU!=_FI^qKZ` zET${U@U_`lv!hBGHI=$^#=+%K2qZvoaX#m^J62IN1w#By9lp)av}5YymmbXY{Vg7` z;#;zXPXuX@N=V2`Um*;(4+K(Ix9SWlVVVx3djR!&!yjZNJT*6jsGZ_8s%HhTk6xor z-v-sGY3Zf;O$$ty^{XX%M22T5eWa}v)f~RLZe2HvN+~3t`H#}LS4mrnwdIU5o42+8 zs(oJ9FK^tnWS#Ywz>uDp^-XC|4~o6noHNLquIk}| zEgZq<<^tmtH=WGv!G(#fReZdQ_oh8;^7WpEqNv1NDz6a#BOPQYGI77-V-y8hc7+(Q ze?gN2fZPYNp0uX$n>Q=7Mn-ynD-~n_o@699xiq*U4f%(K4r2UWwfRkiDUS~G*f|#m z6GN}u>}3b<>U#Cpv9Mlrf<@xLF_^KyTzIZ2@m7 z=9ZsS0?nZin3gk61;F6PMUvu`(_RE$WcsTJxJ19aNddSKW+q;#>Sp2dgaW&Wj^=ZD z-!Im)aOx-{aX-bu;Pd3F?c=Cf(%a#}Vgx!(Lii;!3k$jq&`3OckpJkKH+_~7D=!w? z4*!cZ78AD_zb!}x_=4-d@2COaEikcUP*PGNLY6`AyA@Oi43^fm%Gm+35|=M4<`@dA zs5SDY%~rbuZORO^7-H|_XidEJN_EWK+BzCypj8bw4Pa%Qz9OceVAl(i*eTkjcQIiC z-UL6)#CzF#_|wR$SqETjh|#;Yr`N7ZBON>PNSGHU($br!C8Kz-`9&rYFAtcX17O^t zSaQ+{+?LG{R|(ma=!;xp%_6{1=?G&o;Q5obllFf&DCYJL#)sfcl|d}*{@p^RGJ$g(v)$P~g+l-^j&^?_Ujlp=P3 zuw~y}RhfdRbK(BsVI%~xD?LU+2Y3*k;p%VelMPcER(5S(cCsEfp9mYP*`Gh=%2)u= zTfA@tHwUqgS=b~@@hntougA@v7 zfG1&~r48uUf1jQc@qKm@{De6ewT5<8`L%#m!eo1MHyZ8aYF@MnUED6ft-CdHTu<6( z!(laK?2YI_0gk)cK{OyT4E*(=b-RK&gLr#dr!11yMbkB2gbk zK!XoMvIrAr||Ba@Z3U*6XaM^X}g}Rsz@0Oedj9^u^jnz>6w)a+ERFY zzhJ^Nh55@Xr`on~)>j=5s^0JH>PSuB?@1u@j)zT5a=TzrR}~}vju0gFBX8GislhH5 zX=ztX+E$$zaBRp!fh>&p^+tRPDMRm*5*BG`N#w+0{V(aE^K2h_^x7|MLs6hGvDmK_ z4Fw?N6HeQ&0RV%+f4Bvt0pOrZS3TG#sL%1XqQ)Fli_BR_!00&2Son%Ard>raoYM7k za6|(9%esp#mi1lrr_VpJQ3}{Z5#~Wdj)vKv6XVoi0K+kG@9T0(%L_0go?cqYX&D(p zX|}O?Us!WZV&w7U&H)@rMLa_B40px#%}$XNSjqFc84k^t$Q-?5&7nw8H0|HZx_NfB zAH9+Z*4x8&C5zFB#tbfEYJChs^bM5~hA=nLhX`aMA|f zS0?6O?AVDlH+=Qrvf#4a%CP($GQE-z5&^_>wPbFpNZ_b+NHWSzl=Ci(LR~-f3yWci zcXf5S`T@=@r&1w}6sTF-!IOvi1UZ~LW@ywfMs$q&3ica}8(@;fO>D|#2=XHIY1Gbf~!nI)q4>g>~j9i*<3N z$F0@Z^Um%j&k-vhH@&Af6FvdvPj&UYDa*D8o43O*STg$(zcs@?M2P@Ye<=4)D00MI zc}Ye}?T`3y`a&)@6Z_I*KHV;q#F?zDxI?T!6zpxXZAHN7Cy_lk)f#Xbl70PQh{`uw zfgT4CCG@HFH{$Q_mXy^d031;#A6y_!eG+E#JiA2sBmCy$bz?H86~eAj6f<-daocrDf5FaK}*kktt&W9}UC!j2vb zz+kc*mhXghWXqYrb!6%ICR0*T*QGU^>dQ|aLN@7NS&y7pHIa@>_qY9JXuTZJ1oGyTKYR<=a*)$*3Q{rzAA174?EO#LDWpps;S^r#f5UTuAfCs#)eq<@U6IiG ziZdvE3E4OLHo22zb#X!4!IVTy7{V%6s_A-4rm#?1`Ptb_<<<>DUgNY15Zl+E2E6B(=*U{=WYg()ibOdy(!yF{oK6Dd;&#sVPyahN^QHCS0a8`Vl_agZ3XKhumB7g=rX5O7{UI#HMCCblW}2CP!W zS!*-1C^i2#7$1CYj>mkHC#P*!B#{Xe1;8R0FP}+71a!86)w&f5^D=!3TUeO%`b&@Zr||bDk{%3$dl+0O*bD*oJ^;e&oWX~|H_X<}i1M!=8#t+r zofc+3SY4@M#Ow4iU<+-2`asiT<&&e=$IkxbKQt7c4dQ2UnI@(-%}AnY^Z**wx0zU? zdAg3x!`HXTjKC9DI*U-zT>MF^H=(1k_z`^d8NrZQC=6%3sse%J)2HMv@V`2f| zZpBBkb=FuQn}A#efM(W#UVJGa)7v|K4)|!AivyLgiz-7b7zewd-2QAy6A+c_gkAQ} z;@9bQ2x-8>!!)n0$CI+8F>(Omxnh39sIilVwQRDl1#7xTF;O=W*vru!AP-{zfC*g8 zp|Q`-fJ}NRnLCq_YN9+hCnq2QuSlMfIMY(@iyK?drBRrq8nFbRrzI|i%oa^kO zIfUCQlosTe)#MTuE97B&RA4KawwY5XFey)zm05`E4F8ac9X>{bhU4E{OalH>dxdRb z`|BI|1&m3X(RGm88Y?E?#f4gtLNj}yUx$^S@ZJ)q&3s6nabC9mvO!CsN~f{G%`Cux zIQrrU2(T(_7cdLNe*a!Q1E_YDb^mv{&R-<({;C!mXYHFI~|G-v)k+X;VWVI`j_? z_ApFoOaUetL@As+v#wB#I8xl=8d5~lv7^Za02hWQb0M)pq032BwI@TwjAIg9b>2O? zsR1LNg6^b*rYAjAvY)=U3QYuE>TB5tsgIA3w~xrv()u1>@9$fT_!0q2k5oq@!W~cK zbv#?5E3*QmG)h5w?EpU^^vSHKt`|TOKw;`4I0QN2m$4c0(6GY3@GR9X4HuUYDh4l< zlK(K7WevQi<5C=Fb7%{& ztpB_y2JzDgDjDm-J;bxyY*(5^w+qX>GVV)HT!r7)8lld>OOXdHbw3Hse+PT3)KDLy z84im|Z`UIvQ!;t}n2p(yyaq|%m=$(jUzZe`v-|5Q0P%<($Qy3S2ilsKH$R(dF$lUP zN6)aS3-T@gViD*qW~5oh{ZtKL7(L*=ut@;3Hr?fxwUEGY+6-GVAz@L8A3e=#;TO+k z{@K>N`yS{-@$werBmr@n#&*_I%hVN!HQTEdPrd|7Ulw-oSv52iw%<}dA}svd(;YBf zY{rvdx+5%}cW6QL01L!4C1WmwiK3Gj#O&Ao{mA*UJasJ7Gf3)UC&_XLytPM0z85gc z+t65Tb@cn+m3!Se{;o{zs40=aX{L!!`c^08HBeruIYB#7J{mG)>;_HUOx#M_ic3XC4Tsz6=JOM4r5^rAMX6adY($Et z>zi&ZmnEA9e0sFOiS$6My`V%_{+5VEB>t+jo>;4|0fIU6jjie`Yq0D2lTzp1G;WM? zaCI<&YpuH5n8T5>ni*mLp6jAdoo$f~wG zW-S4G#+{Bj)j|X}-jS1IPTbAv$nQ1|9knu;SjrSy{#SO?DgXRpDSxb5f5VVEsSE{0 z?b1T)%6~+au2U!=t9ooQ)FniT6?b%eIvrI8kA+Dydl`Ne!MHGC#ro{?SLXkYdfDsr z#D`4=I#bDmQ?=gabgBkuNYqY5TlhKGs%&Vm6HttDqEy`)`tNTf|Ekj9d9S%$Jp}e2>0bfYsi~e`yB)1NZYm_Oz?M1lR9XBL8TC|qaPHoZ->MrT} ziMBK}>N44N^`PH#Zi}IYQ;hBhmJNwyX1V3{jb>O{+35Z8Zt?{IkC7jlnw}1Jvy3zE ziJ*9Vzk7~L97yx+9#pe>R3|wwIP!7G#AxFSMa}XX=4qB+i6H;r78Yay#*lr0fM;T2 z;+k`6Ym@);zwiZaO0(K_cvpeW_oZiLP!(??f_i#1Z@0hAfp8ss?SFnO2#e~?0K7+C zTfeHc*BnpD931t(bDX6wlqYx3XStk(t3FL_fK}*RK0E zAm-BNIVfTu5y00v15!#4DE~(TK|0eoICQM@W-Td{>j{mV_G%gy%e+co2GoJRM~5Y* zJ1EHu{Q(f(J&dpzOnhd3eXT(@7JmniLNUR{7J$*Y>2x^FnMFb}3dY>v@%9V`5~AH# zWLsUxfzKdHpoqK>16M3?DA1M(&K-L3rFZhzFc)~ImTZzaonbMXI)6%99IpHy@r6XJ zbPIr8k*GlBvZm9Foo zfsSls@8;9?AAFu4sbQuwrao^6 z{w*0tbp6_Xe$(m6of6~SFP3-OF#Mqu05k6Niq_G3%Z}RFR9j~-A1S{ZAPRl%K3uH6 z+P6rUM5uP>te)GVohjFJHgAY(+p-rH>4Vc9L9n~>d?8i0=iVwRD$5HC#HE1t$?)vI z2cw%bG#BDKt+k)pF-Q;pg7bzyczSt7Z5opU%CWkAeKT7M#BB4a*mD*?RhYP_(TQ*b zSfF+Nq!D46INY6hDXcu3VH!&(^_9?&WqRX{t#YyS87<`TRA*CD*CwCCLz-{a!~F6T zA`^1O3fOqnYzS%wltqeNL>Q522_gai-7fjj3_|SrBt<%GbuM;6UF&o zWlo+;;UX3ESvuyt!>ysh<907F&Ur8X~s&>hcY)xeVpAf>Fdq1ff_kC(NFff8%Wer~$)w6v)YHNKLn z_@i?LhD?ex9wACA)7K<1%;c`<`h?%%QvS$tBcRM?z}f7(IRFUoj>^s!a^m_=%u!G^Gp1m&uLAbTABo1L~l1ekDD{^9$Fa$is#b0B)@US7x4gGO*sjW7wdBQG@sk%in(Q>}jQXOSZ0_cDZEZ^QF#L>0-WoX|m9#TUSzI`k(w~@8^E_Y|(nbDh43%boFyt=akR{09Uq;x&QzG From 63cdb81e2cfd3e3f2d5514b4664ea4117505c4e3 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Mon, 30 Dec 2024 10:57:55 -0800 Subject: [PATCH 26/27] feat(data transform): adding dataTransformLogic models (#12198) --- .../mappers/DataTransformLogicMapper.java | 73 +++++++++++ .../common/mappers/QueryPropertiesMapper.java | 61 +++++++++ .../graphql/types/datajob/DataJobType.java | 3 +- .../types/datajob/mappers/DataJobMapper.java | 24 +--- .../graphql/types/query/QueryMapper.java | 43 +------ .../src/main/resources/entity.graphql | 25 ++++ .../mappers/DataTransformLogicMapperTest.java | 103 +++++++++++++++ .../mappers/QueryPropertiesMapperTest.java | 121 ++++++++++++++++++ .../java/com/linkedin/metadata/Constants.java | 1 + .../com/linkedin/common/DataTransform.pdl | 13 ++ .../linkedin/common/DataTransformLogic.pdl | 14 ++ .../src/main/resources/entity-registry.yml | 1 + 12 files changed, 424 insertions(+), 58 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataTransformLogicMapper.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapper.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/DataTransformLogicMapperTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapperTest.java create mode 100644 metadata-models/src/main/pegasus/com/linkedin/common/DataTransform.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/common/DataTransformLogic.pdl diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataTransformLogicMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataTransformLogicMapper.java new file mode 100644 index 0000000000000..04602e7ff6dde --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataTransformLogicMapper.java @@ -0,0 +1,73 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataTransform; +import com.linkedin.datahub.graphql.generated.DataTransformLogic; +import com.linkedin.datahub.graphql.generated.QueryLanguage; +import com.linkedin.datahub.graphql.generated.QueryStatement; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class DataTransformLogicMapper + implements ModelMapper< + com.linkedin.common.DataTransformLogic, + com.linkedin.datahub.graphql.generated.DataTransformLogic> { + + public static final DataTransformLogicMapper INSTANCE = new DataTransformLogicMapper(); + + public static DataTransformLogic map( + @Nullable final QueryContext context, + @Nonnull final com.linkedin.common.DataTransformLogic input) { + return INSTANCE.apply(context, input); + } + + @Override + public DataTransformLogic apply( + @Nullable final QueryContext context, + @Nonnull final com.linkedin.common.DataTransformLogic input) { + + final DataTransformLogic result = new DataTransformLogic(); + + // Map transforms array using DataTransformMapper + result.setTransforms( + input.getTransforms().stream() + .map(transform -> DataTransformMapper.map(context, transform)) + .collect(Collectors.toList())); + + return result; + } +} + +class DataTransformMapper + implements ModelMapper< + com.linkedin.common.DataTransform, com.linkedin.datahub.graphql.generated.DataTransform> { + + public static final DataTransformMapper INSTANCE = new DataTransformMapper(); + + public static DataTransform map( + @Nullable final QueryContext context, + @Nonnull final com.linkedin.common.DataTransform input) { + return INSTANCE.apply(context, input); + } + + @Override + public DataTransform apply( + @Nullable final QueryContext context, + @Nonnull final com.linkedin.common.DataTransform input) { + + final DataTransform result = new DataTransform(); + + // Map query statement if present + if (input.hasQueryStatement()) { + QueryStatement statement = + new QueryStatement( + input.getQueryStatement().getValue(), + QueryLanguage.valueOf(input.getQueryStatement().getLanguage().toString())); + result.setQueryStatement(statement); + } + + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapper.java new file mode 100644 index 0000000000000..e29bea5b3943c --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapper.java @@ -0,0 +1,61 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import com.linkedin.data.template.GetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.*; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.query.QueryProperties; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class QueryPropertiesMapper + implements ModelMapper< + QueryProperties, com.linkedin.datahub.graphql.generated.QueryProperties> { + + public static final QueryPropertiesMapper INSTANCE = new QueryPropertiesMapper(); + + public static com.linkedin.datahub.graphql.generated.QueryProperties map( + @Nullable final QueryContext context, @Nonnull final QueryProperties input) { + return INSTANCE.apply(context, input); + } + + @Override + public com.linkedin.datahub.graphql.generated.QueryProperties apply( + @Nullable final QueryContext context, @Nonnull final QueryProperties input) { + + final com.linkedin.datahub.graphql.generated.QueryProperties result = + new com.linkedin.datahub.graphql.generated.QueryProperties(); + + // Map Query Source + result.setSource(QuerySource.valueOf(input.getSource().toString())); + + // Map Query Statement + result.setStatement( + new QueryStatement( + input.getStatement().getValue(), + QueryLanguage.valueOf(input.getStatement().getLanguage().toString()))); + + // Map optional fields + result.setName(input.getName(GetMode.NULL)); + result.setDescription(input.getDescription(GetMode.NULL)); + + // Map origin if present + if (input.hasOrigin() && input.getOrigin() != null) { + result.setOrigin(UrnToEntityMapper.map(context, input.getOrigin())); + } + + // Map created audit stamp + AuditStamp created = new AuditStamp(); + created.setTime(input.getCreated().getTime()); + created.setActor(input.getCreated().getActor(GetMode.NULL).toString()); + result.setCreated(created); + + // Map last modified audit stamp + AuditStamp lastModified = new AuditStamp(); + lastModified.setTime(input.getLastModified().getTime()); + lastModified.setActor(input.getLastModified().getActor(GetMode.NULL).toString()); + result.setLastModified(lastModified); + + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java index b32832a28d5d5..8d55ca6dbf7ac 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java @@ -79,7 +79,8 @@ public class DataJobType BROWSE_PATHS_V2_ASPECT_NAME, SUB_TYPES_ASPECT_NAME, STRUCTURED_PROPERTIES_ASPECT_NAME, - FORMS_ASPECT_NAME); + FORMS_ASPECT_NAME, + DATA_TRANSFORM_LOGIC_ASPECT_NAME); private static final Set FACET_FIELDS = ImmutableSet.of("flow"); private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java index 772871d77f217..ec57c95ce151e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java @@ -4,16 +4,7 @@ import static com.linkedin.metadata.Constants.*; import com.google.common.collect.ImmutableList; -import com.linkedin.common.BrowsePathsV2; -import com.linkedin.common.DataPlatformInstance; -import com.linkedin.common.Deprecation; -import com.linkedin.common.Forms; -import com.linkedin.common.GlobalTags; -import com.linkedin.common.GlossaryTerms; -import com.linkedin.common.InstitutionalMemory; -import com.linkedin.common.Ownership; -import com.linkedin.common.Status; -import com.linkedin.common.SubTypes; +import com.linkedin.common.*; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.QueryContext; @@ -26,15 +17,7 @@ import com.linkedin.datahub.graphql.generated.DataJobProperties; import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.EntityType; -import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; -import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; -import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; -import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; -import com.linkedin.datahub.graphql.types.common.mappers.FineGrainedLineagesMapper; -import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; -import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; -import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; -import com.linkedin.datahub.graphql.types.common.mappers.SubTypesMapper; +import com.linkedin.datahub.graphql.types.common.mappers.*; import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils; import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.form.FormsMapper; @@ -139,6 +122,9 @@ public DataJob apply( context, new StructuredProperties(data), entityUrn)); } else if (FORMS_ASPECT_NAME.equals(name)) { result.setForms(FormsMapper.map(new Forms(data), entityUrn.toString())); + } else if (DATA_TRANSFORM_LOGIC_ASPECT_NAME.equals(name)) { + result.setDataTransformLogic( + DataTransformLogicMapper.map(context, new DataTransformLogic(data))); } }); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/query/QueryMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/query/QueryMapper.java index e71b569e9ae23..916ebc772f545 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/query/QueryMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/query/QueryMapper.java @@ -5,18 +5,13 @@ import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; -import com.linkedin.data.template.GetMode; import com.linkedin.datahub.graphql.QueryContext; -import com.linkedin.datahub.graphql.generated.AuditStamp; import com.linkedin.datahub.graphql.generated.DataPlatform; import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.QueryEntity; -import com.linkedin.datahub.graphql.generated.QueryLanguage; -import com.linkedin.datahub.graphql.generated.QuerySource; -import com.linkedin.datahub.graphql.generated.QueryStatement; import com.linkedin.datahub.graphql.generated.QuerySubject; -import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.datahub.graphql.types.common.mappers.QueryPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.entity.EntityResponse; @@ -48,7 +43,10 @@ public QueryEntity apply( result.setType(EntityType.QUERY); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); - mappingHelper.mapToResult(context, QUERY_PROPERTIES_ASPECT_NAME, this::mapQueryProperties); + mappingHelper.mapToResult( + QUERY_PROPERTIES_ASPECT_NAME, + (entity, dataMap) -> + entity.setProperties(QueryPropertiesMapper.map(context, new QueryProperties(dataMap)))); mappingHelper.mapToResult(QUERY_SUBJECTS_ASPECT_NAME, this::mapQuerySubjects); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, this::mapPlatform); return mappingHelper.getResult(); @@ -64,37 +62,6 @@ private void mapPlatform(@Nonnull QueryEntity query, @Nonnull DataMap dataMap) { } } - private void mapQueryProperties( - @Nullable final QueryContext context, @Nonnull QueryEntity query, @Nonnull DataMap dataMap) { - QueryProperties queryProperties = new QueryProperties(dataMap); - com.linkedin.datahub.graphql.generated.QueryProperties res = - new com.linkedin.datahub.graphql.generated.QueryProperties(); - - // Query Source must be kept in sync. - res.setSource(QuerySource.valueOf(queryProperties.getSource().toString())); - res.setStatement( - new QueryStatement( - queryProperties.getStatement().getValue(), - QueryLanguage.valueOf(queryProperties.getStatement().getLanguage().toString()))); - res.setName(queryProperties.getName(GetMode.NULL)); - res.setDescription(queryProperties.getDescription(GetMode.NULL)); - if (queryProperties.hasOrigin() && queryProperties.getOrigin() != null) { - res.setOrigin(UrnToEntityMapper.map(context, queryProperties.getOrigin())); - } - - AuditStamp created = new AuditStamp(); - created.setTime(queryProperties.getCreated().getTime()); - created.setActor(queryProperties.getCreated().getActor(GetMode.NULL).toString()); - res.setCreated(created); - - AuditStamp lastModified = new AuditStamp(); - lastModified.setTime(queryProperties.getLastModified().getTime()); - lastModified.setActor(queryProperties.getLastModified().getActor(GetMode.NULL).toString()); - res.setLastModified(lastModified); - - query.setProperties(res); - } - @Nonnull private void mapQuerySubjects(@Nonnull QueryEntity query, @Nonnull DataMap dataMap) { QuerySubjects querySubjects = new QuerySubjects(dataMap); diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 9abf4e16f12dd..a5cb0893a64fa 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -6569,6 +6569,11 @@ type DataJob implements EntityWithRelationships & Entity & BrowsableEntity { The forms associated with the Dataset """ forms: Forms + + """ + Data Transform Logic associated with the Data Job + """ + dataTransformLogic: DataTransformLogic } """ @@ -6786,6 +6791,26 @@ type DataJobInputOutput { fineGrainedLineages: [FineGrainedLineage!] } +""" +Information about a transformation applied to data assets +""" +type DataTransform { + """ + The transformation may be defined by a query statement + """ + queryStatement: QueryStatement +} + +""" +Information about transformations applied to data assets +""" +type DataTransformLogic { + """ + List of transformations applied + """ + transforms: [DataTransform!]! +} + """ Information about individual user usage of a Dataset """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/DataTransformLogicMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/DataTransformLogicMapperTest.java new file mode 100644 index 0000000000000..f94738ff049ef --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/DataTransformLogicMapperTest.java @@ -0,0 +1,103 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +import com.linkedin.common.DataTransform; +import com.linkedin.common.DataTransformArray; +import com.linkedin.common.DataTransformLogic; +import com.linkedin.query.QueryLanguage; +import com.linkedin.query.QueryStatement; +import java.util.Arrays; +import org.testng.annotations.Test; + +public class DataTransformLogicMapperTest { + + @Test + public void testMapWithQueryStatement() throws Exception { + // Create test data + DataTransformLogic input = new DataTransformLogic(); + + // Create a transform with query statement + DataTransform transform1 = new DataTransform(); + QueryStatement statement = new QueryStatement(); + statement.setValue("SELECT * FROM source_table"); + statement.setLanguage(QueryLanguage.SQL); + transform1.setQueryStatement(statement); + + // Create another transform + DataTransform transform2 = new DataTransform(); + QueryStatement statement2 = new QueryStatement(); + statement2.setValue("INSERT INTO target_table SELECT * FROM temp_table"); + statement2.setLanguage(QueryLanguage.SQL); + transform2.setQueryStatement(statement2); + + // Set transforms + input.setTransforms(new DataTransformArray(Arrays.asList(transform1, transform2))); + + // Map the object + com.linkedin.datahub.graphql.generated.DataTransformLogic result = + DataTransformLogicMapper.map(null, input); + + // Verify result + assertNotNull(result); + assertEquals(result.getTransforms().size(), 2); + + // Verify first transform + com.linkedin.datahub.graphql.generated.DataTransform resultTransform1 = + result.getTransforms().get(0); + assertNotNull(resultTransform1.getQueryStatement()); + assertEquals(resultTransform1.getQueryStatement().getValue(), "SELECT * FROM source_table"); + assertEquals(resultTransform1.getQueryStatement().getLanguage().toString(), "SQL"); + + // Verify second transform + com.linkedin.datahub.graphql.generated.DataTransform resultTransform2 = + result.getTransforms().get(1); + assertNotNull(resultTransform2.getQueryStatement()); + assertEquals( + resultTransform2.getQueryStatement().getValue(), + "INSERT INTO target_table SELECT * FROM temp_table"); + assertEquals(resultTransform2.getQueryStatement().getLanguage().toString(), "SQL"); + } + + @Test + public void testMapWithoutQueryStatement() throws Exception { + // Create test data + DataTransformLogic input = new DataTransformLogic(); + + // Create a transform without query statement + DataTransform transform = new DataTransform(); + + // Set transforms + input.setTransforms(new DataTransformArray(Arrays.asList(transform))); + + // Map the object + com.linkedin.datahub.graphql.generated.DataTransformLogic result = + DataTransformLogicMapper.map(null, input); + + // Verify result + assertNotNull(result); + assertEquals(result.getTransforms().size(), 1); + + // Verify transform + com.linkedin.datahub.graphql.generated.DataTransform resultTransform = + result.getTransforms().get(0); + assertNull(resultTransform.getQueryStatement()); + } + + @Test + public void testMapWithEmptyTransforms() throws Exception { + // Create test data + DataTransformLogic input = new DataTransformLogic(); + input.setTransforms(new DataTransformArray(Arrays.asList())); + + // Map the object + com.linkedin.datahub.graphql.generated.DataTransformLogic result = + DataTransformLogicMapper.map(null, input); + + // Verify result + assertNotNull(result); + assertEquals(result.getTransforms().size(), 0); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapperTest.java new file mode 100644 index 0000000000000..756115cf2054a --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapperTest.java @@ -0,0 +1,121 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.query.QueryLanguage; +import com.linkedin.query.QueryProperties; +import com.linkedin.query.QuerySource; +import com.linkedin.query.QueryStatement; +import org.testng.annotations.Test; + +public class QueryPropertiesMapperTest { + + @Test + public void testMapWithRequiredFields() throws Exception { + // Create test data + QueryProperties input = new QueryProperties(); + + // Set required fields + QueryStatement statement = new QueryStatement(); + statement.setValue("SELECT * FROM table"); + statement.setLanguage(QueryLanguage.SQL); + input.setStatement(statement); + + input.setSource(QuerySource.MANUAL); + + Urn userUrn = Urn.createFromString("urn:li:corpuser:test"); + + AuditStamp created = new AuditStamp(); + created.setTime(1000L); + created.setActor(userUrn); + input.setCreated(created); + + AuditStamp lastModified = new AuditStamp(); + lastModified.setTime(2000L); + lastModified.setActor(userUrn); + input.setLastModified(lastModified); + + // Map the object + com.linkedin.datahub.graphql.generated.QueryProperties result = + QueryPropertiesMapper.map(null, input); + + // Verify required fields + assertNotNull(result); + assertEquals(result.getSource().toString(), "MANUAL"); + assertEquals(result.getStatement().getValue(), "SELECT * FROM table"); + assertEquals(result.getStatement().getLanguage().toString(), "SQL"); + + // Verify audit stamps + assertEquals(result.getCreated().getTime().longValue(), 1000L); + assertEquals(result.getCreated().getActor(), userUrn.toString()); + assertEquals(result.getLastModified().getTime().longValue(), 2000L); + assertEquals(result.getLastModified().getActor(), userUrn.toString()); + + // Verify createdOn resolved stamp + assertEquals(result.getCreatedOn().getTime().longValue(), 1000L); + assertEquals(result.getCreatedOn().getActor().getUrn(), userUrn.toString()); + + // Verify optional fields are null + assertNull(result.getName()); + assertNull(result.getDescription()); + assertNull(result.getOrigin()); + } + + @Test + public void testMapWithOptionalFields() throws Exception { + // Create test data + QueryProperties input = new QueryProperties(); + + // Set required fields + QueryStatement statement = new QueryStatement(); + statement.setValue("SELECT * FROM table"); + statement.setLanguage(QueryLanguage.SQL); + input.setStatement(statement); + + input.setSource(QuerySource.SYSTEM); + + Urn userUrn = Urn.createFromString("urn:li:corpuser:test"); + Urn originUrn = Urn.createFromString("urn:li:dataset:test"); + + AuditStamp created = new AuditStamp(); + created.setTime(1000L); + created.setActor(userUrn); + input.setCreated(created); + + AuditStamp lastModified = new AuditStamp(); + lastModified.setTime(2000L); + lastModified.setActor(userUrn); + input.setLastModified(lastModified); + + // Set optional fields + input.setName("Test Query"); + input.setDescription("Test Description"); + input.setOrigin(originUrn); + + // Map the object + com.linkedin.datahub.graphql.generated.QueryProperties result = + QueryPropertiesMapper.map(null, input); + + // Verify required fields + assertNotNull(result); + assertEquals(result.getSource().toString(), "SYSTEM"); + assertEquals(result.getStatement().getValue(), "SELECT * FROM table"); + assertEquals(result.getStatement().getLanguage().toString(), "SQL"); + + // Verify audit stamps + assertEquals(result.getCreated().getTime().longValue(), 1000L); + assertEquals(result.getCreated().getActor(), userUrn.toString()); + assertEquals(result.getLastModified().getTime().longValue(), 2000L); + assertEquals(result.getLastModified().getActor(), userUrn.toString()); + + // Verify optional fields + assertEquals(result.getName(), "Test Query"); + assertEquals(result.getDescription(), "Test Description"); + assertNotNull(result.getOrigin()); + assertEquals(result.getOrigin().getUrn(), originUrn.toString()); + } +} diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 09f873ebf7bc9..42080e4e17596 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -127,6 +127,7 @@ public class Constants { public static final String EMBED_ASPECT_NAME = "embed"; public static final String INCIDENTS_SUMMARY_ASPECT_NAME = "incidentsSummary"; public static final String DOCUMENTATION_ASPECT_NAME = "documentation"; + public static final String DATA_TRANSFORM_LOGIC_ASPECT_NAME = "dataTransformLogic"; // User public static final String CORP_USER_KEY_ASPECT_NAME = "corpUserKey"; diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/DataTransform.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/DataTransform.pdl new file mode 100644 index 0000000000000..adc8d693b28e2 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/DataTransform.pdl @@ -0,0 +1,13 @@ +namespace com.linkedin.common + +import com.linkedin.query.QueryStatement + +/** + * Information about a transformation. It may be a query, + */ +record DataTransform { + /** + * The data transform may be defined by a query statement + */ + queryStatement: optional QueryStatement +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/DataTransformLogic.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/DataTransformLogic.pdl new file mode 100644 index 0000000000000..431cebf436ffb --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/DataTransformLogic.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.common + +/** + * Information about a Query against one or more data assets (e.g. Tables or Views). + */ +@Aspect = { + "name": "dataTransformLogic" +} +record DataTransformLogic { + /** + * List of transformations applied + */ + transforms: array[DataTransform], +} diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 4fe170ced69f3..0193e5e2c5c6c 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -76,6 +76,7 @@ entities: - subTypes - incidentsSummary - testResults + - dataTransformLogic - name: dataFlow category: core keyAspect: dataFlowKey From abb64433fc7a1d5e7c852ee65675c8abebd8fda1 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Mon, 30 Dec 2024 13:32:28 -0800 Subject: [PATCH 27/27] fix(tests): fixing QueryPropertiesMapperTest.java (#12241) --- .../types/common/mappers/QueryPropertiesMapperTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapperTest.java index 756115cf2054a..a0251adca78f9 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapperTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapperTest.java @@ -55,10 +55,6 @@ public void testMapWithRequiredFields() throws Exception { assertEquals(result.getLastModified().getTime().longValue(), 2000L); assertEquals(result.getLastModified().getActor(), userUrn.toString()); - // Verify createdOn resolved stamp - assertEquals(result.getCreatedOn().getTime().longValue(), 1000L); - assertEquals(result.getCreatedOn().getActor().getUrn(), userUrn.toString()); - // Verify optional fields are null assertNull(result.getName()); assertNull(result.getDescription());