Implement support for TLS config, OAuth credentials and setup CP inte…

…gration tests (#201) * Add mTLS and OAuth 2.0 support for direct connections to Kafka and SR Resolves #125 #126 Adds preliminary support to allow Kafka and SR clients to use mTLS and OAuth 2.0 to authenticate for direct connections. These changes have been unit tested to verify the expected client configurations are generated. However, those expected configurations have NOT yet been tested against CCloud or CP via integration tests. * Add mTLS and OAuth 2.0 support for direct connections to Kafka and SR Resolves #125 #126 Adds preliminary support to allow Kafka and SR clients to use mTLS and OAuth 2.0 to authenticate for direct connections. These changes have been unit tested to verify the expected client configurations are generated. However, those expected configurations have NOT yet been tested against CCloud or CP via integration tests. * Add cp-demo testcontainer classes * Make some progress * Works with schemaless records tests (finally) * Tweak healthchecks * Allow reusing containers across test runs * wip stuff * wip * more wip * more wip * Setup tests for basic auth and mTLS * add tests * address feedback and add tests * add connection spec tests * Add scripts and mechanisms to start/stop/cleanup cp-demo * Shutdown cp-demo if running on CI * Fix all containers running check * Cache docker images and add TestLifecycle annotation to LocalIT * only run docker load if file exists * Use fully qualified name to avoid name conflicts in AbstractIT's connection cache * Disable surefire tests in the native profile * Store test results first before caching * Don't set id -- that makes the integration tests run twice. * Make delete topic test more robust * Remove confluentinc/common-parent as parent POM * Add back confluentinc/common-parent as parent POM * shutdown testcontainers by default * wip * Remove surefire config from native profile (move to followup) * Move TLS into Kafka/SR cluster configs * revert pom change * Handle no auth but SSL case * Cleanup * Refactor tests * Remove MutualTLSCredentials class * add openapi changes * Disable TLS * Fix CP tests * Fix CP tests * Disable TLS for confluent local direct connection * retry test * remove retrying annotation * remove integration-test id * Make produce tests more robust by retrying on 404 * remove unused imports * try `combine.self="override"` * set id * Address reviewer feedback * Harden shouldThrowNotImplementedForUnsupportedSchemaDetails * fix extra space and remove unnecessary changes from openapi files --------- Co-authored-by: Randall Hauch <[email protected]>
confluentinc · Dec 19, 2024 · 0ed73d1 · 0ed73d1
1 parent d80e49c
commit 0ed73d1
Show file tree

Hide file tree

Showing 50 changed files with 3,531 additions and 382 deletions.
diff --git a/.gitignore b/.gitignore
@@ -47,3 +47,5 @@ charts/package/
 # Ignore the templates.zip file created in main and test resources
 src/main/resources/static/templates.zip
 src/test/resources/static/templates.zip
+
+.cp-demo
diff --git a/.semaphore/multi-arch-builds-and-upload.yml b/.semaphore/multi-arch-builds-and-upload.yml
@@ -13,11 +13,13 @@ global_job_config:
       - checkout
       - make ci-bin-sem-cache-restore
       - make docker-login-ci
+      - make load-cached-docker-images
   epilogue:
     always:
       commands:
-        - make ci-bin-sem-cache-store
         - make store-test-results-to-semaphore
+        - make ci-bin-sem-cache-store
+        - make cache-docker-images
 
 blocks:
   - name: "Build Native Executable (MacOS AMD64)"

diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
@@ -27,11 +27,13 @@ global_job_config:
       - checkout
       - make ci-bin-sem-cache-restore
       - make docker-login-ci
+      - make load-cached-docker-images
   epilogue:
     always:
       commands:
-        - make ci-bin-sem-cache-store
         - make store-test-results-to-semaphore
+        - make ci-bin-sem-cache-store
+        - make cache-docker-images
 
 blocks:
   - name: "Build JARs and Unit Test"

diff --git a/Makefile b/Makefile
@@ -95,3 +95,93 @@ upload-artifacts-to-github-release:
 .PHONY: collect-notices-binary
 collect-notices-binary: clean mvn-package-native-sources-only
 	$(IDE_SIDECAR_SCRIPTS)/collect-notices-binary.sh target/native-sources/lib
+
+# Targets for managing cp-demo testcontainers used by the integration tests
+
+# Start the cp-demo testcontainers
+# Note: You do not need to run this in order to run the integration tests, however, if you want
+# to manually bring up the cp-demo environment, you may run this target. You will be
+# able to run the integration tests against the same environment, please keep that in mind!
+.PHONY: cp-demo-start
+cp-demo-start:
+	export TESTCONTAINERS_RYUK_DISABLED=true; \
+	./mvnw -s .mvn/settings.xml \
+		-Dexec.mainClass=io.confluent.idesidecar.restapi.util.CPDemoTestEnvironment \
+		-Dexec.classpathScope=test \
+		test-compile exec:java
+
+# Stop the cp-demo testcontainers
+.PHONY: cp-demo-stop
+cp-demo-stop:
+	./mvnw -s .mvn/settings.xml test-compile && \
+	./mvnw -s .mvn/settings.xml \
+		-Dexec.mainClass=io.confluent.idesidecar.restapi.util.CPDemoTestEnvironment \
+		-Dexec.classpathScope=test \
+		-Dexec.args=stop \
+		exec:java
+
+
+CONFLUENT_DOCKER_TAG = $(shell yq e '.ide-sidecar.integration-tests.cp-demo.tag' src/main/resources/application.yml | sed 's/^v//')
+# See io.confluent.idesidecar.restapi.util.ConfluentLocalKafkaWithRestProxyContainer
+CONFLUENT_LOCAL_DOCKER_TAG = "7.6.0"
+# See io.confluent.idesidecar.restapi.util.cpdemo.OpenldapContainer
+OSIXIA_OPENLDAP_DOCKER_TAG = "1.3.0"
+# See io.confluent.idesidecar.restapi.util.cpdemo.ToolsContainer
+CNFLDEMOS_TOOLS_DOCKER_TAG = "0.3"
+
+# Key for storing docker images in Semaphore CI cache
+SEMAPHORE_CP_ZOOKEEPER_DOCKER := ide-sidecar-docker-cp-zookeeper-$(CONFLUENT_DOCKER_TAG)
+SEMAPHORE_CP_SERVER_DOCKER := ide-sidecar-docker-cp-server-$(CONFLUENT_DOCKER_TAG)
+SEMAPHORE_OPENLDAP_DOCKER := ide-sidecar-docker-openldap-$(OSIXIA_OPENLDAP_DOCKER_TAG)
+SEMAPHORE_CNFLDEMOS_TOOLS_DOCKER := ide-sidecar-docker-cnfldemos-tools-$(CNFLDEMOS_TOOLS_DOCKER_TAG)
+SEMAPHORE_CONFLUENT_LOCAL_DOCKER := ide-sidecar-docker-confluent-local-$(CONFLUENT_LOCAL_DOCKER_TAG)
+
+## Cache docker images in Semaphore cache.
+.PHONY: cache-docker-images
+cache-docker-images:
+	cache has_key $(SEMAPHORE_CP_ZOOKEEPER_DOCKER) || (\
+		docker pull confluentinc/cp-zookeeper:$(CONFLUENT_DOCKER_TAG) && \
+		docker save confluentinc/cp-zookeeper:$(CONFLUENT_DOCKER_TAG) | gzip > cp-zookeeper.tgz && \
+		cache store $(SEMAPHORE_CP_ZOOKEEPER_DOCKER) cp-zookeeper.tgz && \
+		rm -rf cp-zookeeper.tgz)
+
+	cache has_key $(SEMAPHORE_CP_SERVER_DOCKER) || (\
+		docker pull confluentinc/cp-server:$(CONFLUENT_DOCKER_TAG) && \
+		docker save confluentinc/cp-server:$(CONFLUENT_DOCKER_TAG) | gzip > cp-server.tgz && \
+		cache store $(SEMAPHORE_CP_SERVER_DOCKER) cp-server.tgz && \
+		rm -rf cp-server.tgz)
+
+	cache has_key $(SEMAPHORE_OPENLDAP_DOCKER) || (\
+		docker pull osixia/openldap:$(OSIXIA_OPENLDAP_DOCKER_TAG) && \
+		docker save osixia/openldap:$(OSIXIA_OPENLDAP_DOCKER_TAG) | gzip > openldap.tgz && \
+		cache store $(SEMAPHORE_OPENLDAP_DOCKER) openldap.tgz && \
+		rm -rf openldap.tgz)
+
+	cache has_key $(SEMAPHORE_CNFLDEMOS_TOOLS_DOCKER) || (\
+		docker pull cnfldemos/tools:$(CNFLDEMOS_TOOLS_DOCKER_TAG) && \
+		docker save cnfldemos/tools:$(CNFLDEMOS_TOOLS_DOCKER_TAG) | gzip > cnfdemos-tools.tgz && \
+		cache store $(SEMAPHORE_CNFLDEMOS_TOOLS_DOCKER) cnfdemos-tools.tgz && \
+		rm -rf cnfdemos-tools.tgz)
+
+	cache has_key $(SEMAPHORE_CONFLUENT_LOCAL_DOCKER) || (\
+		docker pull confluentinc/cp-local:$(CONFLUENT_LOCAL_DOCKER_TAG) && \
+		docker save confluentinc/cp-local:$(CONFLUENT_LOCAL_DOCKER_TAG) | gzip > cp-local.tgz && \
+		cache store $(SEMAPHORE_CONFLUENT_LOCAL_DOCKER) cp-local.tgz && \
+		rm -rf cp-local.tgz)
+
+.PHONY: load-cached-docker-images
+load-cached-docker-images:
+	cache restore $(SEMAPHORE_CP_ZOOKEEPER_DOCKER) \
+    [ -f cp-zookeeper.tgz ] && docker load -i cp-zookeeper.tgz && rm -rf cp-zookeeper.tgz || true
+
+	cache restore $(SEMAPHORE_CP_SERVER_DOCKER) \
+	[ -f cp-server.tgz ] && docker load -i cp-server.tgz && rm -rf cp-server.tgz || true
+
+	cache restore $(SEMAPHORE_OPENLDAP_DOCKER) \
+	[ -f openldap.tgz ] && docker load -i openldap.tgz && rm -rf openldap.tgz || true
+
+	cache restore $(SEMAPHORE_CNFLDEMOS_TOOLS_DOCKER) \
+	[ -f cnfdemos-tools.tgz ] && docker load -i cnfdemos-tools.tgz && rm -rf cnfdemos-tools.tgz || true
+
+	cache restore $(SEMAPHORE_CONFLUENT_LOCAL_DOCKER) \
+	[ -f cp-local.tgz ] && docker load -i cp-local.tgz && rm -rf cp-local.tgz || true
diff --git a/pom.xml b/pom.xml
@@ -413,7 +413,7 @@
                   <goal>integration-test</goal>
                   <goal>verify</goal>
                 </goals>
-                <configuration combine.children="append">
+                <configuration combine.self="override" combine.children="append">
                   <failIfNoTests>true</failIfNoTests>
                   <systemPropertyVariables>
                     <native.image.path>

diff --git a/src/generated/resources/openapi.json b/src/generated/resources/openapi.json
@@ -659,7 +659,7 @@
   "components" : {
     "schemas" : {
       "ApiKeyAndSecret" : {
-        "description" : "Basic authentication credentials",
+        "description" : "API key and secret authentication credentials",
         "required" : [ "api_key", "api_secret" ],
         "type" : "object",
         "properties" : {
@@ -1121,19 +1121,17 @@
               "$ref" : "#/components/schemas/BasicCredentials"
             }, {
               "$ref" : "#/components/schemas/ApiKeyAndSecret"
+            }, {
+              "$ref" : "#/components/schemas/OAuthCredentials"
             } ],
             "nullable" : true
           },
           "ssl" : {
-            "description" : "Whether to communicate with the Kafka cluster over TLS/SSL. Defaults to 'true', but set to 'false' when the Kafka cluster does not support TLS/SSL.",
-            "default" : true,
-            "type" : "boolean",
-            "nullable" : true
-          },
-          "verify_ssl_certificates" : {
-            "description" : "Whether to verify the Kafka cluster certificates. Defaults to 'true', but set to 'false' when the Kafka cluster has self-signed certificates.",
-            "default" : true,
-            "type" : "boolean",
+            "description" : "The SSL configuration for connecting to the Kafka cluster. To disable, set `enabled` to false. To use the default SSL settings, set `enabled` to true and leave the `truststore` and `keystore` fields unset.",
+            "type" : "object",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/TLSConfig"
+            } ],
             "nullable" : true
           }
         }
@@ -1166,6 +1164,42 @@
           }
         }
       },
+      "KeyStore" : {
+        "required" : [ "path" ],
+        "type" : "object",
+        "properties" : {
+          "path" : {
+            "description" : "The path to the local key store file. Only specified if client needs to be authenticated by the server (mutual TLS).",
+            "maxLength" : 256,
+            "type" : "string"
+          },
+          "password" : {
+            "description" : "The password for the local key store file. If a password is not set, key store file configured will still be used, but integrity checking is disabled. A key store password is not supported for PEM format.",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/Password"
+            } ],
+            "nullable" : true
+          },
+          "type" : {
+            "description" : "The file format of the local key store file.",
+            "default" : "JKS",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/StoreType"
+            } ],
+            "nullable" : true
+          },
+          "key_password" : {
+            "description" : "The password of the private key in the local key store file.",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/Password"
+            } ],
+            "nullable" : true
+          }
+        }
+      },
       "LocalConfig" : {
         "description" : "Configuration when using Confluent Local and optionally a local Schema Registry.",
         "type" : "object",
@@ -1177,6 +1211,46 @@
           }
         }
       },
+      "OAuthCredentials" : {
+        "description" : "OAuth 2.0 authentication credentials",
+        "required" : [ "tokens_url", "client_id" ],
+        "type" : "object",
+        "properties" : {
+          "tokens_url" : {
+            "description" : "The URL of the OAuth 2.0 identity provider's token endpoint.",
+            "maxLength" : 256,
+            "type" : "string"
+          },
+          "client_id" : {
+            "description" : "The public identifier for the application as registered with the OAuth 2.0 identity provider.",
+            "maxLength" : 128,
+            "minLength" : 1,
+            "type" : "string"
+          },
+          "client_secret" : {
+            "description" : "The client secret known only to the application and the OAuth 2.0 identity provider.",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/Password"
+            } ]
+          },
+          "scope" : {
+            "description" : "The scope to use. The scope is optional and required only when your identity provider doesn't have a default scope or your groups claim is linked to a scope path to use when connecting to the external service.",
+            "maxLength" : 256,
+            "type" : "string"
+          },
+          "connect_timeout_millis" : {
+            "format" : "int32",
+            "description" : "The timeout in milliseconds when connecting to your identity provider.",
+            "minimum" : 0,
+            "type" : "integer"
+          },
+          "identityPool" : {
+            "description" : "Additional property that can be added in the request header to identify the principal ID for authorization. For example, this may bea Confluent Cloud identity pool.",
+            "type" : "string"
+          }
+        }
+      },
       "ObjectMetadata" : {
         "type" : "object",
         "properties" : {
@@ -1375,6 +1449,16 @@
               "$ref" : "#/components/schemas/BasicCredentials"
             }, {
               "$ref" : "#/components/schemas/ApiKeyAndSecret"
+            }, {
+              "$ref" : "#/components/schemas/OAuthCredentials"
+            } ],
+            "nullable" : true
+          },
+          "ssl" : {
+            "description" : "The SSL configuration for connecting to Schema Registry. If null, the connection will use SSL with the default settings. To disable, set `enabled` to false.",
+            "type" : "object",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/TLSConfig"
             } ],
             "nullable" : true
           }
@@ -1475,6 +1559,43 @@
         "enum" : [ "NO_TOKEN", "VALID_TOKEN", "INVALID_TOKEN", "FAILED" ],
         "type" : "string"
       },
+      "StoreType" : {
+        "enum" : [ "JKS", "PKCS12", "PEM", "UNKNOWN" ],
+        "type" : "string"
+      },
+      "TLSConfig" : {
+        "description" : "SSL configuration",
+        "required" : [ "enabled" ],
+        "type" : "object",
+        "properties" : {
+          "verify_hostname" : {
+            "description" : "Whether to verify the server certificate hostname. Defaults to true if not set.",
+            "default" : true,
+            "type" : "boolean"
+          },
+          "enabled" : {
+            "description" : "Whether SSL is enabled. If not set, defaults to true.",
+            "default" : true,
+            "type" : "boolean"
+          },
+          "truststore" : {
+            "description" : "The trust store configuration for authenticating the server's certificate.",
+            "type" : "object",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/TrustStore"
+            } ],
+            "nullable" : true
+          },
+          "keystore" : {
+            "description" : "The key store configuration that will identify and authenticate the client to the server, required for mutual TLS (mTLS)",
+            "type" : "object",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/KeyStore"
+            } ],
+            "nullable" : true
+          }
+        }
+      },
       "Template" : {
         "required" : [ "api_version", "kind", "id", "metadata", "spec" ],
         "type" : "object",
@@ -1557,6 +1678,34 @@
         "enum" : [ "NO_TIMESTAMP_TYPE", "CREATE_TIME", "LOG_APPEND_TIME" ],
         "type" : "string"
       },
+      "TrustStore" : {
+        "required" : [ "path" ],
+        "type" : "object",
+        "properties" : {
+          "path" : {
+            "description" : "The path to the local trust store file. Required for authenticating the server's certificate.",
+            "maxLength" : 256,
+            "type" : "string"
+          },
+          "password" : {
+            "description" : "The password for the local trust store file. If a password is not set, trust store file configured will still be used, but integrity checking is disabled. A trust store password is not supported for PEM format.",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/Password"
+            } ],
+            "nullable" : true
+          },
+          "type" : {
+            "description" : "The file format of the local trust store file",
+            "default" : "JKS",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/StoreType"
+            } ],
+            "nullable" : true
+          }
+        }
+      },
       "UserInfo" : {
         "type" : "object",
         "properties" : {