From a4f64fd49bc4895379d7508f57a5cbe0b1726e6d Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 4 Feb 2025 08:26:12 -0600 Subject: [PATCH] fix(misc-openapi): fix openlineage, platform events & swagger (#12539) --- .../openapi/config/SpringWebConfig.java | 2 +- .../controller/LineageApiImpl.java | 2 +- .../PlatformEntitiesController.java | 2 +- .../openapi/v3/OpenAPIV3Generator.java | 23 ++++++++----- .../config/SpringWebSchemaRegistryConfig.java | 6 ---- .../tests/openapi/openlineage/__init__.py | 0 .../openapi/openlineage/openlineage.json | 34 +++++++++++++++++++ .../tests/openapi/v2/platform_entities.json | 31 +++++++++++++++++ .../tests/utilities/concurrent_openapi.py | 7 ++-- 9 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 smoke-test/tests/openapi/openlineage/__init__.py create mode 100644 smoke-test/tests/openapi/openlineage/openlineage.json create mode 100644 smoke-test/tests/openapi/v2/platform_entities.json diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java index fb2dba8103d59..6f3f058e9d997 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java @@ -28,7 +28,7 @@ @EnableWebMvc @OpenAPIDefinition( info = @Info(title = "DataHub OpenAPI", version = "2.0.0"), - servers = {@Server(url = "/openapi/", description = "Default Server URL")}) + servers = {@Server(url = "/", description = "Default Server URL")}) @Order(2) @Configuration public class SpringWebConfig implements WebMvcConfigurer { diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/controller/LineageApiImpl.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/controller/LineageApiImpl.java index f1da20fefe6bf..61c4794bbfa58 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/controller/LineageApiImpl.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/controller/LineageApiImpl.java @@ -27,7 +27,7 @@ import org.springframework.web.bind.annotation.RestController; @RestController -@RequestMapping("/openlineage/api/v1") +@RequestMapping("/openapi/openlineage/api/v1") @Slf4j public class LineageApiImpl implements LineageApi { private static final ObjectMapper OBJECT_MAPPER = OpenLineageClientUtils.newObjectMapper(); diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/PlatformEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/PlatformEntitiesController.java index bfd1f952f95f7..79fe25411540b 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/PlatformEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/PlatformEntitiesController.java @@ -39,7 +39,7 @@ @RestController @RequiredArgsConstructor -@RequestMapping("/openapiv2/platform/entities/v1") +@RequestMapping("/openapi/v2/platform/entities/v1") @Slf4j @Tag( name = "Platform Entities", diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index f7764f2ddb39a..349672ef2dcec 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -35,6 +35,7 @@ @SuppressWarnings({"rawtypes", "unchecked"}) public class OpenAPIV3Generator { + private static final String PATH_PREFIX = "/openapi/v3"; private static final String MODEL_VERSION = "_v3"; private static final String TYPE_OBJECT = "object"; private static final String TYPE_BOOLEAN = "boolean"; @@ -171,18 +172,19 @@ public static OpenAPI generateOpenApiSpec( final Paths paths = new Paths(); // --> Cross-entity Paths - paths.addPathItem("/v3/entity/scroll", buildGenericListEntitiesPath()); + paths.addPathItem(PATH_PREFIX + "/entity/scroll", buildGenericListEntitiesPath()); // --> Entity Paths definedEntitySpecs.forEach( e -> { paths.addPathItem( - String.format("/v3/entity/%s", e.getName().toLowerCase()), buildListEntityPath(e)); + String.format(PATH_PREFIX + "/entity/%s", e.getName().toLowerCase()), + buildListEntityPath(e)); paths.addPathItem( - String.format("/v3/entity/%s/batchGet", e.getName().toLowerCase()), + String.format(PATH_PREFIX + "/entity/%s/batchGet", e.getName().toLowerCase()), buildBatchGetEntityPath(e)); paths.addPathItem( - String.format("/v3/entity/%s/{urn}", e.getName().toLowerCase()), + String.format(PATH_PREFIX + "/entity/%s/{urn}", e.getName().toLowerCase()), buildSingleEntityPath(e)); }); @@ -196,8 +198,9 @@ public static OpenAPI generateOpenApiSpec( a -> paths.addPathItem( String.format( - "/v3/entity/%s/{urn}/%s", - e.getName().toLowerCase(), a.getName().toLowerCase()), + PATH_PREFIX + "/entity/%s/{urn}/%s", + e.getName().toLowerCase(), + a.getName().toLowerCase()), buildSingleEntityAspectPath(e, a)))); definedEntitySpecs.forEach( e -> @@ -208,8 +211,9 @@ public static OpenAPI generateOpenApiSpec( a -> paths.addPathItem( String.format( - "/v3/entity/%s/{urn}/%s", - e.getName().toLowerCase(), a.getName().toLowerCase()), + PATH_PREFIX + "/entity/%s/{urn}/%s", + e.getName().toLowerCase(), + a.getName().toLowerCase()), buildSingleEntityAspectPath(e, a)))); // --> Link & Unlink APIs @@ -219,7 +223,8 @@ public static OpenAPI generateOpenApiSpec( .forEach( entitySpec -> { paths.addPathItem( - "/v3/entity/versioning/{versionSetUrn}/relationship/versionOf/{entityUrn}", + PATH_PREFIX + + "/entity/versioning/{versionSetUrn}/relationship/versionOf/{entityUrn}", buildVersioningRelationshipPath()); }); } diff --git a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java index 56b4e57530179..ef899d00ec936 100644 --- a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java +++ b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java @@ -4,9 +4,6 @@ import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory; import com.linkedin.metadata.registry.SchemaRegistryService; import io.datahubproject.openapi.schema.registry.SchemaRegistryController; -import io.swagger.v3.oas.annotations.OpenAPIDefinition; -import io.swagger.v3.oas.annotations.info.Info; -import io.swagger.v3.oas.annotations.servers.Server; import jakarta.servlet.http.HttpServletRequest; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; @@ -19,9 +16,6 @@ @Slf4j @EnableWebMvc -@OpenAPIDefinition( - info = @Info(title = "DataHub OpenAPI", version = "1.0.0"), - servers = {@Server(url = "/schema-registry/", description = "Schema Registry Server URL")}) @Order(3) @ConditionalOnProperty( name = "kafka.schemaRegistry.type", diff --git a/smoke-test/tests/openapi/openlineage/__init__.py b/smoke-test/tests/openapi/openlineage/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/smoke-test/tests/openapi/openlineage/openlineage.json b/smoke-test/tests/openapi/openlineage/openlineage.json new file mode 100644 index 0000000000000..f40fd110a1b1b --- /dev/null +++ b/smoke-test/tests/openapi/openlineage/openlineage.json @@ -0,0 +1,34 @@ +[ + { + "request": { + "url": "/openapi/openlineage/api/v1/lineage", + "description": "Post openlineage", + "json": { + "eventType": "START", + "eventTime": "2020-12-28T19:52:00.001+10:00", + "run": { + "runId": "d46e465b-d358-4d32-83d4-df660ff614dd" + }, + "job": { + "namespace": "workshop", + "name": "process_taxes" + }, + "inputs": [ + { + "namespace": "postgres://workshop-db:None", + "name": "workshop.public.taxes", + "facets": { + "dataSource": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.10.0/integration/airflow", + "_schemaURL": "https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/DataSourceDatasetFacet", + "name": "postgres://workshop-db:None", + "uri": "workshop-db" + } + } + } + ], + "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client" + } + } + } +] \ No newline at end of file diff --git a/smoke-test/tests/openapi/v2/platform_entities.json b/smoke-test/tests/openapi/v2/platform_entities.json new file mode 100644 index 0000000000000..206f5d0b152b8 --- /dev/null +++ b/smoke-test/tests/openapi/v2/platform_entities.json @@ -0,0 +1,31 @@ +[ + { + "request": { + "url": "/openapi/v2/platform/entities/v1/", + "description": "Create data product", + "params": { + "async": "false" + }, + "json": [ + { + "entityType": "dataProduct", + "entityUrn": "urn:li:dataProduct:4fd5aea7-a15f-4842-a2cc-e6562605ebf7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": { + "__type": "Status", + "removed": false + }, + "contentType": "application/json" + } + } + ] + }, + "response": { + "json": [ + "urn:li:dataProduct:4fd5aea7-a15f-4842-a2cc-e6562605ebf7" + ] + } + } +] \ No newline at end of file diff --git a/smoke-test/tests/utilities/concurrent_openapi.py b/smoke-test/tests/utilities/concurrent_openapi.py index 076cd12c7a3b3..03a7a00f005a3 100644 --- a/smoke-test/tests/utilities/concurrent_openapi.py +++ b/smoke-test/tests/utilities/concurrent_openapi.py @@ -57,13 +57,14 @@ def evaluate_test(auth_session, test_name, test_data): continue url = req_resp["request"]["url"] actual_resp = execute_request(auth_session, req_resp["request"]) + diff = None try: if "response" in req_resp and "status_codes" in req_resp["response"]: assert ( actual_resp.status_code in req_resp["response"]["status_codes"] ) else: - assert actual_resp.status_code in [200, 202, 204] + assert actual_resp.status_code in [200, 201, 202, 204] if "response" in req_resp: if "json" in req_resp["response"]: if "exclude_regex_paths" in req_resp["response"]: @@ -87,7 +88,9 @@ def evaluate_test(auth_session, test_name, test_data): ) if description: logger.error(f"Step {idx} Description: {description}") - logger.error(f"Response content: {actual_resp.content}") + if diff: + logger.error(f"Unexpected diff: {diff}") + logger.debug(f"Response content: {actual_resp.content}") raise e except Exception as e: logger.error(f"Error executing test: {test_name}")