From bcc6473ee289179c89698a530716b60e7ad8776f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20M=C3=B6ll?= Date: Thu, 24 Aug 2023 17:04:56 +0200 Subject: [PATCH 1/2] ClickHouse: Made tags optional and adjusted TABLE creation for newer version --- cmd/tsbs_load_clickhouse/main.go | 1 + pkg/targets/clickhouse/creator.go | 14 ++++++++++---- pkg/targets/clickhouse/implemented_target.go | 1 + 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/tsbs_load_clickhouse/main.go b/cmd/tsbs_load_clickhouse/main.go index ac7181d0c..b3f058e93 100644 --- a/cmd/tsbs_load_clickhouse/main.go +++ b/cmd/tsbs_load_clickhouse/main.go @@ -45,6 +45,7 @@ func init() { Password: viper.GetString("password"), LogBatches: viper.GetBool("log-batches"), Debug: viper.GetInt("debug"), + InTableTag: viper.GetBool("in-table-partition-tag"), DbName: loaderConf.DBName, } diff --git a/pkg/targets/clickhouse/creator.go b/pkg/targets/clickhouse/creator.go index 9e991d36b..3e8f04ee3 100644 --- a/pkg/targets/clickhouse/creator.go +++ b/pkg/targets/clickhouse/creator.go @@ -127,12 +127,17 @@ func createMetricsTable(conf *ClickhouseConfig, db *sqlx.DB, tableName string, f // columnsWithType - column specifications with type. Ex.: "cpu_usage Float64" var columnsWithType []string - for _, column := range columnNames { + for idx, column := range columnNames { if len(column) == 0 { // Skip nameless columns continue } - columnsWithType = append(columnsWithType, fmt.Sprintf("%s Nullable(Float64)", column)) + if conf.InTableTag && idx == 0 { + columnsWithType = append(columnsWithType, fmt.Sprintf("%s Nullable(String)", column)) + } else { + columnsWithType = append(columnsWithType, fmt.Sprintf("%s Nullable(Float64)", column)) + } + } sql := fmt.Sprintf(` @@ -143,7 +148,7 @@ func createMetricsTable(conf *ClickhouseConfig, db *sqlx.DB, tableName string, f tags_id UInt32, %s, additional_tags String DEFAULT '' - ) ENGINE = MergeTree(created_date, (tags_id, created_at), 8192) + ) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_date) PRIMARY KEY (tags_id, created_at) `, tableName, strings.Join(columnsWithType, ",")) @@ -180,7 +185,8 @@ func generateTagsTableQuery(tagNames, tagTypes []string) string { "created_at DateTime DEFAULT now(),\n"+ "id UInt32,\n"+ "%s"+ - ") ENGINE = MergeTree(created_date, (%s), 8192)", + ") ENGINE = MergeTree()"+ + "PARTITION BY toYYYYMM(created_date) PRIMARY KEY %s", cols, index) } diff --git a/pkg/targets/clickhouse/implemented_target.go b/pkg/targets/clickhouse/implemented_target.go index 27cb53411..809e2b4a5 100644 --- a/pkg/targets/clickhouse/implemented_target.go +++ b/pkg/targets/clickhouse/implemented_target.go @@ -30,6 +30,7 @@ func (c clickhouseTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag. flagSet.String(flagPrefix+"password", "", "Password to connect to ClickHouse") flagSet.Bool(flagPrefix+"log-batches", false, "Whether to time individual batches.") flagSet.Int(flagPrefix+"debug", 0, "Debug printing (choices: 0, 1, 2). (default 0)") + flagSet.Bool(flagPrefix+"in-table-partition-tag", false, "Whether the partition key (e.g. hostname) should also be in the metrics hypertable") } func (c clickhouseTarget) TargetName() string { From dddc2a0dc83a3bb9fe02bca9a65e5ead3adb6095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20M=C3=B6ll?= Date: Thu, 31 Aug 2023 11:13:13 +0200 Subject: [PATCH 2/2] Update creator_test.go --- pkg/targets/clickhouse/creator_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/targets/clickhouse/creator_test.go b/pkg/targets/clickhouse/creator_test.go index 6c329ef71..880a66827 100644 --- a/pkg/targets/clickhouse/creator_test.go +++ b/pkg/targets/clickhouse/creator_test.go @@ -18,7 +18,7 @@ func TestGenerateTagsTableQuery(t *testing.T) { "created_at DateTime DEFAULT now(),\n" + "id UInt32,\n" + "tag1 Nullable(String)" + - ") ENGINE = MergeTree(created_date, (id), 8192)"}, { + ") ENGINE = MergeTree()PARTITION BY toYYYYMM(created_date) PRIMARY KEY id"}, { inTagNames: []string{"tag1", "tag2", "tag3", "tag4"}, inTagTypes: []string{"int32", "int64", "float32", "float64"}, out: "CREATE TABLE tags(\n" + @@ -29,7 +29,7 @@ func TestGenerateTagsTableQuery(t *testing.T) { "tag2 Nullable(Int64),\n" + "tag3 Nullable(Float32),\n" + "tag4 Nullable(Float64)" + - ") ENGINE = MergeTree(created_date, (id), 8192)"}, + ") ENGINE = MergeTree()PARTITION BY toYYYYMM(created_date) PRIMARY KEY id"}, } for _, tc := range testCases { t.Run(fmt.Sprintf("tags table for %v", tc.inTagNames), func(t *testing.T) {