Skip to content

Commit

Permalink
[Feature] add avro format document
Browse files Browse the repository at this point in the history
  • Loading branch information
liunaijie committed Jan 5, 2024
1 parent ac6e2d1 commit b891202
Showing 1 changed file with 111 additions and 0 deletions.
111 changes: 111 additions & 0 deletions docs/en/connector-v2/formats/avro.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
# Avro format

Avro is very popular in streaming data pipeline. Now seatunnel supports Avro format in kafka connector.

# How to use Avro format

## Kafka uses example

- This is an example to generate data from fake source and sink to kafka with avro format.

```bash
env {
parallelism = 1
job.mode = "BATCH"
}

source {
FakeSource {
row.num = 90
schema = {
fields {
c_map = "map<string, string>"
c_array = "array<int>"
c_string = string
c_boolean = boolean
c_tinyint = tinyint
c_smallint = smallint
c_int = int
c_bigint = bigint
c_float = float
c_double = double
c_bytes = bytes
c_date = date
c_decimal = "decimal(38, 18)"
c_timestamp = timestamp
c_row = {
c_map = "map<string, string>"
c_array = "array<int>"
c_string = string
c_boolean = boolean
c_tinyint = tinyint
c_smallint = smallint
c_int = int
c_bigint = bigint
c_float = float
c_double = double
c_bytes = bytes
c_date = date
c_decimal = "decimal(38, 18)"
c_timestamp = timestamp
}
}
}
result_table_name = "fake"
}
}

sink {
Kafka {
bootstrap.servers = "kafkaCluster:9092"
topic = "test_avro_topic_fake_source"
format = avro
}
}
```

- This is an example read data from kafka with avro format and print to console.

```bash
env {
parallelism = 1
job.mode = "BATCH"
}

source {
Kafka {
bootstrap.servers = "kafkaCluster:9092"
topic = "test_avro_topic"
result_table_name = "kafka_table"
kafka.auto.offset.reset = "earliest"
format = avro
format_error_handle_way = skip
schema = {
fields {
id = bigint
c_map = "map<string, smallint>"
c_array = "array<tinyint>"
c_string = string
c_boolean = boolean
c_tinyint = tinyint
c_smallint = smallint
c_int = int
c_bigint = bigint
c_float = float
c_double = double
c_decimal = "decimal(2, 1)"
c_bytes = bytes
c_date = date
c_timestamp = timestamp
}
}
}
}

sink {
Console {
source_table_name = "kafka_table"
}
}
```

0 comments on commit b891202

Please sign in to comment.