Implement support for TLS config, OAuth credentials and setup CP integration tests

.gitignore

@@ -47,3 +47,5 @@ charts/package/
 # Ignore the templates.zip file created in main and test resources
 src/main/resources/static/templates.zip
 src/test/resources/static/templates.zip
+
+.cp-demo

.semaphore/multi-arch-builds-and-upload.yml

@@ -13,11 +13,13 @@ global_job_config:
       - checkout
       - make ci-bin-sem-cache-restore
       - make docker-login-ci
+      - make load-cached-docker-images
   epilogue:
     always:
       commands:
-        - make ci-bin-sem-cache-store
         - make store-test-results-to-semaphore
+        - make ci-bin-sem-cache-store
+        - make cache-docker-images
 
 blocks:
   - name: "Build Native Executable (MacOS AMD64)"

.semaphore/semaphore.yml

@@ -27,11 +27,13 @@ global_job_config:
       - checkout
       - make ci-bin-sem-cache-restore
       - make docker-login-ci
+      - make load-cached-docker-images
   epilogue:
     always:
       commands:
-        - make ci-bin-sem-cache-store
         - make store-test-results-to-semaphore
+        - make ci-bin-sem-cache-store
+        - make cache-docker-images
 
 blocks:
   - name: "Build JARs and Unit Test"

Makefile

@@ -95,3 +95,93 @@ upload-artifacts-to-github-release:
 .PHONY: collect-notices-binary
 collect-notices-binary: clean mvn-package-native-sources-only
 	$(IDE_SIDECAR_SCRIPTS)/collect-notices-binary.sh target/native-sources/lib
+
+# Targets for managing cp-demo testcontainers used by the integration tests
+
+# Start the cp-demo testcontainers
+# Note: You do not need to run this in order to run the integration tests, however, if you want
+# to manually bring up the cp-demo environment, you may run this target. You will be
+# able to run the integration tests against the same environment, please keep that in mind!
+.PHONY: cp-demo-start
+cp-demo-start:
+	export TESTCONTAINERS_RYUK_DISABLED=true; \
+	./mvnw -s .mvn/settings.xml \
+		-Dexec.mainClass=io.confluent.idesidecar.restapi.util.CPDemoTestEnvironment \
+		-Dexec.classpathScope=test \
+		test-compile exec:java
+
+# Stop the cp-demo testcontainers
+.PHONY: cp-demo-stop
+cp-demo-stop:
+	./mvnw -s .mvn/settings.xml test-compile && \
+	./mvnw -s .mvn/settings.xml \
+		-Dexec.mainClass=io.confluent.idesidecar.restapi.util.CPDemoTestEnvironment \
+		-Dexec.classpathScope=test \
+		-Dexec.args=stop \
+		exec:java
+
+
+CONFLUENT_DOCKER_TAG = $(shell yq e '.ide-sidecar.integration-tests.cp-demo.tag' src/main/resources/application.yml | sed 's/^v//')
+# See io.confluent.idesidecar.restapi.util.ConfluentLocalKafkaWithRestProxyContainer
+CONFLUENT_LOCAL_DOCKER_TAG = "7.6.0"
+# See io.confluent.idesidecar.restapi.util.cpdemo.OpenldapContainer
+OSIXIA_OPENLDAP_DOCKER_TAG = "1.3.0"
+# See io.confluent.idesidecar.restapi.util.cpdemo.ToolsContainer
+CNFLDEMOS_TOOLS_DOCKER_TAG = "0.3"
+
+# Key for storing docker images in Semaphore CI cache
+SEMAPHORE_CP_ZOOKEEPER_DOCKER := ide-sidecar-docker-cp-zookeeper-$(CONFLUENT_DOCKER_TAG)
+SEMAPHORE_CP_SERVER_DOCKER := ide-sidecar-docker-cp-server-$(CONFLUENT_DOCKER_TAG)
+SEMAPHORE_OPENLDAP_DOCKER := ide-sidecar-docker-openldap-$(OSIXIA_OPENLDAP_DOCKER_TAG)
+SEMAPHORE_CNFLDEMOS_TOOLS_DOCKER := ide-sidecar-docker-cnfldemos-tools-$(CNFLDEMOS_TOOLS_DOCKER_TAG)
+SEMAPHORE_CONFLUENT_LOCAL_DOCKER := ide-sidecar-docker-confluent-local-$(CONFLUENT_LOCAL_DOCKER_TAG)
+
+## Cache docker images in Semaphore cache.
+.PHONY: cache-docker-images
+cache-docker-images:
+	cache has_key $(SEMAPHORE_CP_ZOOKEEPER_DOCKER) || (\
+		docker pull confluentinc/cp-zookeeper:$(CONFLUENT_DOCKER_TAG) && \
+		docker save confluentinc/cp-zookeeper:$(CONFLUENT_DOCKER_TAG) | gzip > cp-zookeeper.tgz && \
+		cache store $(SEMAPHORE_CP_ZOOKEEPER_DOCKER) cp-zookeeper.tgz && \
+		rm -rf cp-zookeeper.tgz)
+
+	cache has_key $(SEMAPHORE_CP_SERVER_DOCKER) || (\
+		docker pull confluentinc/cp-server:$(CONFLUENT_DOCKER_TAG) && \
+		docker save confluentinc/cp-server:$(CONFLUENT_DOCKER_TAG) | gzip > cp-server.tgz && \
+		cache store $(SEMAPHORE_CP_SERVER_DOCKER) cp-server.tgz && \
+		rm -rf cp-server.tgz)
+
+	cache has_key $(SEMAPHORE_OPENLDAP_DOCKER) || (\
+		docker pull osixia/openldap:$(OSIXIA_OPENLDAP_DOCKER_TAG) && \
+		docker save osixia/openldap:$(OSIXIA_OPENLDAP_DOCKER_TAG) | gzip > openldap.tgz && \
+		cache store $(SEMAPHORE_OPENLDAP_DOCKER) openldap.tgz && \
+		rm -rf openldap.tgz)
+
+	cache has_key $(SEMAPHORE_CNFLDEMOS_TOOLS_DOCKER) || (\
+		docker pull cnfldemos/tools:$(CNFLDEMOS_TOOLS_DOCKER_TAG) && \
+		docker save cnfldemos/tools:$(CNFLDEMOS_TOOLS_DOCKER_TAG) | gzip > cnfdemos-tools.tgz && \
+		cache store $(SEMAPHORE_CNFLDEMOS_TOOLS_DOCKER) cnfdemos-tools.tgz && \
+		rm -rf cnfdemos-tools.tgz)
+
+	cache has_key $(SEMAPHORE_CONFLUENT_LOCAL_DOCKER) || (\
+		docker pull confluentinc/cp-local:$(CONFLUENT_LOCAL_DOCKER_TAG) && \
+		docker save confluentinc/cp-local:$(CONFLUENT_LOCAL_DOCKER_TAG) | gzip > cp-local.tgz && \
+		cache store $(SEMAPHORE_CONFLUENT_LOCAL_DOCKER) cp-local.tgz && \
+		rm -rf cp-local.tgz)
+
+.PHONY: load-cached-docker-images
+load-cached-docker-images:
+	cache restore $(SEMAPHORE_CP_ZOOKEEPER_DOCKER) \
+    [ -f cp-zookeeper.tgz ] && docker load -i cp-zookeeper.tgz && rm -rf cp-zookeeper.tgz || true
+
+	cache restore $(SEMAPHORE_CP_SERVER_DOCKER) \
+	[ -f cp-server.tgz ] && docker load -i cp-server.tgz && rm -rf cp-server.tgz || true
+
+	cache restore $(SEMAPHORE_OPENLDAP_DOCKER) \
+	[ -f openldap.tgz ] && docker load -i openldap.tgz && rm -rf openldap.tgz || true
+
+	cache restore $(SEMAPHORE_CNFLDEMOS_TOOLS_DOCKER) \
+	[ -f cnfdemos-tools.tgz ] && docker load -i cnfdemos-tools.tgz && rm -rf cnfdemos-tools.tgz || true
+
+	cache restore $(SEMAPHORE_CONFLUENT_LOCAL_DOCKER) \
+	[ -f cp-local.tgz ] && docker load -i cp-local.tgz && rm -rf cp-local.tgz || true

pom.xml

@@ -408,7 +408,6 @@
             <artifactId>maven-failsafe-plugin</artifactId>
             <executions>
               <execution>
-                <id>integration-test</id>
                 <goals>
                   <goal>integration-test</goal>
                   <goal>verify</goal>

src/generated/resources/openapi.json

@@ -659,7 +659,7 @@
   "components" : {
     "schemas" : {
       "ApiKeyAndSecret" : {
-        "description" : "Basic authentication credentials",
+        "description" : "API key and secret authentication credentials",
         "required" : [ "api_key", "api_secret" ],
         "type" : "object",
         "properties" : {
@@ -1106,7 +1106,7 @@
       },
       "KafkaClusterConfig" : {
         "description" : "Kafka cluster configuration.",
-        "required" : [ "bootstrap_servers" ],
+        "required" : [ "bootstrap_servers", "ssl" ],
         "type" : "object",
         "properties" : {
           "bootstrap_servers" : {
@@ -1121,20 +1121,17 @@
               "$ref" : "#/components/schemas/BasicCredentials"
             }, {
               "$ref" : "#/components/schemas/ApiKeyAndSecret"
+            }, {
+              "$ref" : "#/components/schemas/OAuthCredentials"
             } ],
             "nullable" : true
           },
           "ssl" : {
-            "description" : "Whether to communicate with the Kafka cluster over TLS/SSL. Defaults to 'true', but set to 'false' when the Kafka cluster does not support TLS/SSL.",
-            "default" : true,
-            "type" : "boolean",
-            "nullable" : true
-          },
-          "verify_ssl_certificates" : {
-            "description" : "Whether to verify the Kafka cluster certificates. Defaults to 'true', but set to 'false' when the Kafka cluster has self-signed certificates.",
-            "default" : true,
-            "type" : "boolean",
-            "nullable" : true
+            "description" : "The SSL configuration for connecting to the Kafka cluster. To disable, set `enabled` to false. To use the default SSL settings, set `enabled` to true and leave the `truststore` and `keystore` fields unset.",
+            "type" : "object",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/TLSConfig"
+            } ]
           }
         }
       },
@@ -1166,6 +1163,42 @@
           }
         }
       },
+      "KeyStore" : {
+        "required" : [ "path" ],
+        "type" : "object",
+        "properties" : {
+          "path" : {
+            "description" : "The path to the local key store file. Only specified if client needs to be authenticated by the server (mutual TLS).",
+            "maxLength" : 256,
+            "type" : "string"
+          },
+          "password" : {
+            "description" : "The password for the local key store file. If a password is not set, trust  store file configured will still be used, but integrity checking is disabled. A key store password is not supported for PEM format.",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/Password"
+            } ],
+            "nullable" : true
+          },
+          "type" : {
+            "description" : "The file format of the local key store file.",
+            "default" : "JKS",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/StoreType"
+            } ],
+            "nullable" : true
+          },
+          "key_password" : {
+            "description" : "The password of the private key in the local key store file.",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/Password"
+            } ],
+            "nullable" : true
+          }
+        }
+      },
       "LocalConfig" : {
         "description" : "Configuration when using Confluent Local and optionally a local Schema Registry.",
         "type" : "object",
@@ -1177,6 +1210,46 @@
           }
         }
       },
+      "OAuthCredentials" : {
+        "description" : "OAuth 2.0 authentication credentials",
+        "required" : [ "tokens_url", "client_id" ],
+        "type" : "object",
+        "properties" : {
+          "tokens_url" : {
+            "description" : "The URL of the OAuth 2.0 identity provider's token endpoint.",
+            "maxLength" : 256,
+            "type" : "string"
+          },
+          "client_id" : {
+            "description" : "The public identifier for the application as registered with the OAuth 2.0 identity provider.",
+            "maxLength" : 128,
+            "minLength" : 1,
+            "type" : "string"
+          },
+          "client_secret" : {
+            "description" : "The client secret known only to the application and the OAuth 2.0 identity provider.",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/Password"
+            } ]
+          },
+          "scope" : {
+            "description" : "The scope to use. The scope is optional and required only when your identity provider doesn't have a default scope or your groups claim is linked to a scope path to use when connecting to the external service.",
+            "maxLength" : 256,
+            "type" : "string"
+          },
+          "connect_timeout_millis" : {
+            "format" : "int32",
+            "description" : "The timeout in milliseconds when connecting to your identity provider.",
+            "minimum" : 0,
+            "type" : "integer"
+          },
+          "identityPool" : {
+            "description" : "Additional property that can be added in the request header to identify the principal ID for authorization. For example, this may bea Confluent Cloud identity pool.",
+            "type" : "string"
+          }
+        }
+      },
       "ObjectMetadata" : {
         "type" : "object",
         "properties" : {
@@ -1375,6 +1448,16 @@
               "$ref" : "#/components/schemas/BasicCredentials"
             }, {
               "$ref" : "#/components/schemas/ApiKeyAndSecret"
+            }, {
+              "$ref" : "#/components/schemas/OAuthCredentials"
+            } ],
+            "nullable" : true
+          },
+          "ssl" : {
+            "description" : "The SSL configuration for connecting to Schema Registry. If null, the connection will use SSL with the default settings. To disable, set `enabled` to false.",
+            "type" : "object",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/TLSConfig"
             } ],
             "nullable" : true
           }
@@ -1475,6 +1558,43 @@
         "enum" : [ "NO_TOKEN", "VALID_TOKEN", "INVALID_TOKEN", "FAILED" ],
         "type" : "string"
       },
+      "StoreType" : {
+        "enum" : [ "JKS", "PKCS12", "PEM", "UNKNOWN" ],
+        "type" : "string"
+      },
+      "TLSConfig" : {
+        "description" : "SSL configuration",
+        "required" : [ "enabled" ],
+        "type" : "object",
+        "properties" : {
+          "verify_hostname" : {
+            "description" : "Whether to verify the server certificate hostname. Defaults to true if not set.",
+            "default" : true,
+            "type" : "boolean"
+          },
+          "enabled" : {
+            "description" : "Whether SSL is enabled. If not set, defaults to true.",
+            "default" : true,
+            "type" : "boolean"
+          },
+          "truststore" : {
+            "description" : "The trust store configuration for authenticating the server's certificate.",
+            "type" : "object",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/TrustStore"
+            } ],
+            "nullable" : true
+          },
+          "keystore" : {
+            "description" : "The key store configuration that will identify and authenticate the client to the server, required for mutual TLS (mTLS)",
+            "type" : "object",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/KeyStore"
+            } ],
+            "nullable" : true
+          }
+        }
+      },
       "Template" : {
         "required" : [ "api_version", "kind", "id", "metadata", "spec" ],
         "type" : "object",
@@ -1557,6 +1677,34 @@
         "enum" : [ "NO_TIMESTAMP_TYPE", "CREATE_TIME", "LOG_APPEND_TIME" ],
         "type" : "string"
       },
+      "TrustStore" : {
+        "required" : [ "path" ],
+        "type" : "object",
+        "properties" : {
+          "path" : {
+            "description" : "The path to the local trust store file. Required for authenticating the server's certificate.",
+            "maxLength" : 256,
+            "type" : "string"
+          },
+          "password" : {
+            "description" : "The password for the local trust store file. If a password is not set, trust store file configured will still be used, but integrity checking is disabled. A trust store password is not supported for PEM format.",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/Password"
+            } ],
+            "nullable" : true
+          },
+          "type" : {
+            "description" : "The file format of the local trust store file",
+            "default" : "JKS",
+            "type" : "string",
+            "allOf" : [ {
+              "$ref" : "#/components/schemas/StoreType"
+            } ],
+            "nullable" : true
+          }
+        }
+      },
       "UserInfo" : {
         "type" : "object",
         "properties" : {