feat: Add Kafka-based sources to IngestionDataSourceSettings proto and IngestionFailureEvent proto

.jsdoc.js

@@ -1,4 +1,4 @@
-// Copyright 2024 Google LLC
+// Copyright 2025 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@ module.exports = {
     includePattern: '\\.js$'
   },
   templates: {
-    copyright: 'Copyright 2024 Google LLC',
+    copyright: 'Copyright 2025 Google LLC',
     includeDate: false,
     sourceFiles: false,
     systemName: '@google-cloud/pubsub',

protos/google/pubsub/v1/pubsub.proto

@@ -323,13 +323,181 @@ message IngestionDataSourceSettings {
     string match_glob = 9 [(google.api.field_behavior) = OPTIONAL];
   }
 
+  // Ingestion settings for Azure Event Hubs.
+  message AzureEventHubs {
+    // Possible states for managed ingestion from Event Hubs.
+    enum State {
+      // Default value. This value is unused.
+      STATE_UNSPECIFIED = 0;
+
+      // Ingestion is active.
+      ACTIVE = 1;
+
+      // Permission denied encountered while consuming data from Event Hubs.
+      // This can happen when `client_id`, or `tenant_id` are invalid. Or the
+      // right permissions haven't been granted.
+      EVENT_HUBS_PERMISSION_DENIED = 2;
+
+      // Permission denied encountered while publishing to the topic.
+      PUBLISH_PERMISSION_DENIED = 3;
+
+      // The provided Event Hubs namespace couldn't be found.
+      NAMESPACE_NOT_FOUND = 4;
+
+      // The provided Event Hub couldn't be found.
+      EVENT_HUB_NOT_FOUND = 5;
+
+      // The provided Event Hubs subscription couldn't be found.
+      SUBSCRIPTION_NOT_FOUND = 6;
+
+      // The provided Event Hubs resource group couldn't be found.
+      RESOURCE_GROUP_NOT_FOUND = 7;
+    }
+
+    // Output only. An output-only field that indicates the state of the Event
+    // Hubs ingestion source.
+    State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Optional. Name of the resource group within the azure subscription.
+    string resource_group = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the Event Hubs namespace.
+    string namespace = 3 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the Event Hub.
+    string event_hub = 4 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The client id of the Azure application that is being used to
+    // authenticate Pub/Sub.
+    string client_id = 5 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The tenant id of the Azure application that is being used to
+    // authenticate Pub/Sub.
+    string tenant_id = 6 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The Azure subscription id.
+    string subscription_id = 7 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The GCP service account to be used for Federated Identity
+    // authentication.
+    string gcp_service_account = 8 [(google.api.field_behavior) = OPTIONAL];
+  }
+
+  // Ingestion settings for Amazon MSK.
+  message AwsMsk {
+    // Possible states for managed ingestion from Amazon MSK.
+    enum State {
+      // Default value. This value is unused.
+      STATE_UNSPECIFIED = 0;
+
+      // Ingestion is active.
+      ACTIVE = 1;
+
+      // Permission denied encountered while consuming data from Amazon MSK.
+      MSK_PERMISSION_DENIED = 2;
+
+      // Permission denied encountered while publishing to the topic.
+      PUBLISH_PERMISSION_DENIED = 3;
+
+      // The provided MSK cluster wasn't found.
+      CLUSTER_NOT_FOUND = 4;
+
+      // The provided topic wasn't found.
+      TOPIC_NOT_FOUND = 5;
+    }
+
+    // Output only. An output-only field that indicates the state of the Amazon
+    // MSK ingestion source.
+    State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Required. The Amazon Resource Name (ARN) that uniquely identifies the
+    // cluster.
+    string cluster_arn = 2 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The name of the topic in the Amazon MSK cluster that Pub/Sub
+    // will import from.
+    string topic = 3 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. AWS role ARN to be used for Federated Identity authentication
+    // with Amazon MSK. Check the Pub/Sub docs for how to set up this role and
+    // the required permissions that need to be attached to it.
+    string aws_role_arn = 4 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The GCP service account to be used for Federated Identity
+    // authentication with Amazon MSK (via a `AssumeRoleWithWebIdentity` call
+    // for the provided role). The `aws_role_arn` must be set up with
+    // `accounts.google.com:sub` equals to this service account number.
+    string gcp_service_account = 5 [(google.api.field_behavior) = REQUIRED];
+  }
+
+  // Ingestion settings for Confluent Cloud.
+  message ConfluentCloud {
+    // Possible states for managed ingestion from Confluent Cloud.
+    enum State {
+      // Default value. This value is unused.
+      STATE_UNSPECIFIED = 0;
+
+      // Ingestion is active.
+      ACTIVE = 1;
+
+      // Permission denied encountered while consuming data from Confluent
+      // Cloud.
+      CONFLUENT_CLOUD_PERMISSION_DENIED = 2;
+
+      // Permission denied encountered while publishing to the topic.
+      PUBLISH_PERMISSION_DENIED = 3;
+
+      // The provided bootstrap server address is unreachable.
+      UNREACHABLE_BOOTSTRAP_SERVER = 4;
+
+      // The provided cluster wasn't found.
+      CLUSTER_NOT_FOUND = 5;
+
+      // The provided topic wasn't found.
+      TOPIC_NOT_FOUND = 6;
+    }
+
+    // Output only. An output-only field that indicates the state of the
+    // Confluent Cloud ingestion source.
+    State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Required. The address of the bootstrap server. The format is url:port.
+    string bootstrap_server = 2 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The id of the cluster.
+    string cluster_id = 3 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The name of the topic in the Confluent Cloud cluster that
+    // Pub/Sub will import from.
+    string topic = 4 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The id of the identity pool to be used for Federated Identity
+    // authentication with Confluent Cloud. See
+    // https://docs.confluent.io/cloud/current/security/authenticate/workload-identities/identity-providers/oauth/identity-pools.html#add-oauth-identity-pools.
+    string identity_pool_id = 5 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The GCP service account to be used for Federated Identity
+    // authentication with `identity_pool_id`.
+    string gcp_service_account = 6 [(google.api.field_behavior) = REQUIRED];
+  }
+
   // Only one source type can have settings set.
   oneof source {
     // Optional. Amazon Kinesis Data Streams.
     AwsKinesis aws_kinesis = 1 [(google.api.field_behavior) = OPTIONAL];
 
     // Optional. Cloud Storage.
     CloudStorage cloud_storage = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Azure Event Hubs.
+    AzureEventHubs azure_event_hubs = 3
+        [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Amazon MSK.
+    AwsMsk aws_msk = 5 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Confluent Cloud.
+    ConfluentCloud confluent_cloud = 6 [(google.api.field_behavior) = OPTIONAL];
   }
 
   // Optional. Platform Logs settings. If unset, no Platform Logs will be
@@ -408,6 +576,78 @@ message IngestionFailureEvent {
     }
   }
 
+  // Failure when ingesting from an Amazon MSK source.
+  message AwsMskFailureReason {
+    // Optional. The ARN of the cluster of the topic being ingested from.
+    string cluster_arn = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the Kafka topic being ingested from.
+    string kafka_topic = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The partition ID of the message that failed to be ingested.
+    int64 partition_id = 3 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The offset within the partition of the message that failed to
+    // be ingested.
+    int64 offset = 4 [(google.api.field_behavior) = OPTIONAL];
+
+    // Reason why ingestion failed for the specified message.
+    oneof reason {
+      // Optional. The Pub/Sub API limits prevented the desired message from
+      // being published.
+      ApiViolationReason api_violation_reason = 5
+          [(google.api.field_behavior) = OPTIONAL];
+    }
+  }
+
+  // Failure when ingesting from an Azure Event Hubs source.
+  message AzureEventHubsFailureReason {
+    // Optional. The namespace containing the event hub being ingested from.
+    string namespace = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the event hub being ingested from.
+    string event_hub = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The partition ID of the message that failed to be ingested.
+    int64 partition_id = 3 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The offset within the partition of the message that failed to
+    // be ingested.
+    int64 offset = 4 [(google.api.field_behavior) = OPTIONAL];
+
+    // Reason why ingestion failed for the specified message.
+    oneof reason {
+      // Optional. The Pub/Sub API limits prevented the desired message from
+      // being published.
+      ApiViolationReason api_violation_reason = 5
+          [(google.api.field_behavior) = OPTIONAL];
+    }
+  }
+
+  // Failure when ingesting from a Confluent Cloud source.
+  message ConfluentCloudFailureReason {
+    // Optional. The cluster ID containing the topic being ingested from.
+    string cluster_id = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the Kafka topic being ingested from.
+    string kafka_topic = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The partition ID of the message that failed to be ingested.
+    int64 partition_id = 3 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The offset within the partition of the message that failed to
+    // be ingested.
+    int64 offset = 4 [(google.api.field_behavior) = OPTIONAL];
+
+    // Reason why ingestion failed for the specified message.
+    oneof reason {
+      // Optional. The Pub/Sub API limits prevented the desired message from
+      // being published.
+      ApiViolationReason api_violation_reason = 5
+          [(google.api.field_behavior) = OPTIONAL];
+    }
+  }
+
   // Required. Name of the import topic. Format is:
   // projects/{project_name}/topics/{topic_name}.
   string topic = 1 [(google.api.field_behavior) = REQUIRED];
@@ -419,6 +659,18 @@ message IngestionFailureEvent {
     // Optional. Failure when ingesting from Cloud Storage.
     CloudStorageFailure cloud_storage_failure = 3
         [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Failure when ingesting from Amazon MSK.
+    AwsMskFailureReason aws_msk_failure = 4
+        [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Failure when ingesting from Azure Event Hubs.
+    AzureEventHubsFailureReason azure_event_hubs_failure = 5
+        [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Failure when ingesting from Confluent Cloud.
+    ConfluentCloudFailureReason confluent_cloud_failure = 6
+        [(google.api.field_behavior) = OPTIONAL];
   }
 }