fix(misc-openapi): fix openlineage, platform events & swagger (#12539)

datahub-project · Feb 4, 2025 · a4f64fd · a4f64fd
1 parent d1e8a0a
commit a4f64fd
Show file tree

Hide file tree

Showing 9 changed files with 87 additions and 20 deletions.
diff --git a/...rvice/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java b/...rvice/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java
@@ -28,7 +28,7 @@
 @EnableWebMvc
 @OpenAPIDefinition(
     info = @Info(title = "DataHub OpenAPI", version = "2.0.0"),
-    servers = {@Server(url = "/openapi/", description = "Default Server URL")})
+    servers = {@Server(url = "/", description = "Default Server URL")})
 @Order(2)
 @Configuration
 public class SpringWebConfig implements WebMvcConfigurer {

diff --git a/...ervlet/src/main/java/io/datahubproject/openapi/openlineage/controller/LineageApiImpl.java b/...ervlet/src/main/java/io/datahubproject/openapi/openlineage/controller/LineageApiImpl.java
@@ -27,7 +27,7 @@
 import org.springframework.web.bind.annotation.RestController;
 
 @RestController
-@RequestMapping("/openlineage/api/v1")
+@RequestMapping("/openapi/openlineage/api/v1")
 @Slf4j
 public class LineageApiImpl implements LineageApi {
   private static final ObjectMapper OBJECT_MAPPER = OpenLineageClientUtils.newObjectMapper();

diff --git a/...let/src/main/java/io/datahubproject/openapi/v2/controller/PlatformEntitiesController.java b/...let/src/main/java/io/datahubproject/openapi/v2/controller/PlatformEntitiesController.java
@@ -39,7 +39,7 @@
 
 @RestController
 @RequiredArgsConstructor
-@RequestMapping("/openapiv2/platform/entities/v1")
+@RequestMapping("/openapi/v2/platform/entities/v1")
 @Slf4j
 @Tag(
     name = "Platform Entities",

diff --git a/...ervice/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/...ervice/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java
@@ -35,6 +35,7 @@
 
 @SuppressWarnings({"rawtypes", "unchecked"})
 public class OpenAPIV3Generator {
+  private static final String PATH_PREFIX = "/openapi/v3";
   private static final String MODEL_VERSION = "_v3";
   private static final String TYPE_OBJECT = "object";
   private static final String TYPE_BOOLEAN = "boolean";
@@ -171,18 +172,19 @@ public static OpenAPI generateOpenApiSpec(
     final Paths paths = new Paths();
 
     // --> Cross-entity Paths
-    paths.addPathItem("/v3/entity/scroll", buildGenericListEntitiesPath());
+    paths.addPathItem(PATH_PREFIX + "/entity/scroll", buildGenericListEntitiesPath());
 
     // --> Entity Paths
     definedEntitySpecs.forEach(
         e -> {
           paths.addPathItem(
-              String.format("/v3/entity/%s", e.getName().toLowerCase()), buildListEntityPath(e));
+              String.format(PATH_PREFIX + "/entity/%s", e.getName().toLowerCase()),
+              buildListEntityPath(e));
           paths.addPathItem(
-              String.format("/v3/entity/%s/batchGet", e.getName().toLowerCase()),
+              String.format(PATH_PREFIX + "/entity/%s/batchGet", e.getName().toLowerCase()),
               buildBatchGetEntityPath(e));
           paths.addPathItem(
-              String.format("/v3/entity/%s/{urn}", e.getName().toLowerCase()),
+              String.format(PATH_PREFIX + "/entity/%s/{urn}", e.getName().toLowerCase()),
               buildSingleEntityPath(e));
         });
 
@@ -196,8 +198,9 @@ public static OpenAPI generateOpenApiSpec(
                     a ->
                         paths.addPathItem(
                             String.format(
-                                "/v3/entity/%s/{urn}/%s",
-                                e.getName().toLowerCase(), a.getName().toLowerCase()),
+                                PATH_PREFIX + "/entity/%s/{urn}/%s",
+                                e.getName().toLowerCase(),
+                                a.getName().toLowerCase()),
                             buildSingleEntityAspectPath(e, a))));
     definedEntitySpecs.forEach(
         e ->
@@ -208,8 +211,9 @@ public static OpenAPI generateOpenApiSpec(
                     a ->
                         paths.addPathItem(
                             String.format(
-                                "/v3/entity/%s/{urn}/%s",
-                                e.getName().toLowerCase(), a.getName().toLowerCase()),
+                                PATH_PREFIX + "/entity/%s/{urn}/%s",
+                                e.getName().toLowerCase(),
+                                a.getName().toLowerCase()),
                             buildSingleEntityAspectPath(e, a))));
 
     // --> Link & Unlink APIs
@@ -219,7 +223,8 @@ public static OpenAPI generateOpenApiSpec(
           .forEach(
               entitySpec -> {
                 paths.addPathItem(
-                    "/v3/entity/versioning/{versionSetUrn}/relationship/versionOf/{entityUrn}",
+                    PATH_PREFIX
+                        + "/entity/versioning/{versionSetUrn}/relationship/versionOf/{entityUrn}",
                     buildVersioningRelationshipPath());
               });
     }

diff --git a/.../java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java b/.../java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java
@@ -4,9 +4,6 @@
 import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory;
 import com.linkedin.metadata.registry.SchemaRegistryService;
 import io.datahubproject.openapi.schema.registry.SchemaRegistryController;
-import io.swagger.v3.oas.annotations.OpenAPIDefinition;
-import io.swagger.v3.oas.annotations.info.Info;
-import io.swagger.v3.oas.annotations.servers.Server;
 import jakarta.servlet.http.HttpServletRequest;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -19,9 +16,6 @@
 
 @Slf4j
 @EnableWebMvc
-@OpenAPIDefinition(
-    info = @Info(title = "DataHub OpenAPI", version = "1.0.0"),
-    servers = {@Server(url = "/schema-registry/", description = "Schema Registry Server URL")})
 @Order(3)
 @ConditionalOnProperty(
     name = "kafka.schemaRegistry.type",

diff --git a/smoke-test/tests/openapi/openlineage/__init__.py b/smoke-test/tests/openapi/openlineage/__init__.py
diff --git a/smoke-test/tests/openapi/openlineage/openlineage.json b/smoke-test/tests/openapi/openlineage/openlineage.json
@@ -0,0 +1,34 @@
+[
+  {
+    "request": {
+      "url": "/openapi/openlineage/api/v1/lineage",
+      "description": "Post openlineage",
+      "json": {
+        "eventType": "START",
+        "eventTime": "2020-12-28T19:52:00.001+10:00",
+        "run": {
+          "runId": "d46e465b-d358-4d32-83d4-df660ff614dd"
+        },
+        "job": {
+          "namespace": "workshop",
+          "name": "process_taxes"
+        },
+        "inputs": [
+          {
+            "namespace": "postgres://workshop-db:None",
+            "name": "workshop.public.taxes",
+            "facets": {
+              "dataSource": {
+                "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.10.0/integration/airflow",
+                "_schemaURL": "https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/DataSourceDatasetFacet",
+                "name": "postgres://workshop-db:None",
+                "uri": "workshop-db"
+              }
+            }
+          }
+        ],
+        "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client"
+      }
+    }
+  }
+]
diff --git a/smoke-test/tests/openapi/v2/platform_entities.json b/smoke-test/tests/openapi/v2/platform_entities.json
@@ -0,0 +1,31 @@
+[
+  {
+    "request": {
+      "url": "/openapi/v2/platform/entities/v1/",
+      "description": "Create data product",
+      "params": {
+        "async": "false"
+      },
+      "json": [
+        {
+          "entityType": "dataProduct",
+          "entityUrn": "urn:li:dataProduct:4fd5aea7-a15f-4842-a2cc-e6562605ebf7",
+          "changeType": "UPSERT",
+          "aspectName": "status",
+          "aspect": {
+            "value": {
+              "__type": "Status",
+              "removed": false
+            },
+            "contentType": "application/json"
+          }
+        }
+      ]
+    },
+    "response": {
+      "json": [
+        "urn:li:dataProduct:4fd5aea7-a15f-4842-a2cc-e6562605ebf7"
+      ]
+    }
+  }
+]
diff --git a/smoke-test/tests/utilities/concurrent_openapi.py b/smoke-test/tests/utilities/concurrent_openapi.py
@@ -57,13 +57,14 @@ def evaluate_test(auth_session, test_name, test_data):
                 continue
             url = req_resp["request"]["url"]
             actual_resp = execute_request(auth_session, req_resp["request"])
+            diff = None
             try:
                 if "response" in req_resp and "status_codes" in req_resp["response"]:
                     assert (
                         actual_resp.status_code in req_resp["response"]["status_codes"]
                     )
                 else:
-                    assert actual_resp.status_code in [200, 202, 204]
+                    assert actual_resp.status_code in [200, 201, 202, 204]
                 if "response" in req_resp:
                     if "json" in req_resp["response"]:
                         if "exclude_regex_paths" in req_resp["response"]:
@@ -87,7 +88,9 @@ def evaluate_test(auth_session, test_name, test_data):
                 )
                 if description:
                     logger.error(f"Step {idx} Description: {description}")
-                logger.error(f"Response content: {actual_resp.content}")
+                if diff:
+                    logger.error(f"Unexpected diff: {diff}")
+                logger.debug(f"Response content: {actual_resp.content}")
                 raise e
     except Exception as e:
         logger.error(f"Error executing test: {test_name}")