From 10a44bccd2a883e64b03fcd5c87cd96a9884d36e Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Wed, 21 Jul 2021 21:14:27 -0400
Subject: [PATCH 01/41] Add copyright notice to the otel test.

---
 otel/conf_test.go | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/otel/conf_test.go b/otel/conf_test.go
index 9caa2ac366..8a5d960fa1 100644
--- a/otel/conf_test.go
+++ b/otel/conf_test.go
@@ -1,3 +1,17 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package otel
 
 import (

From 207bf723d39543e2bac793b172f18a2aaa8e149c Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 29 Jul 2021 20:38:25 -0400
Subject: [PATCH 02/41] Fix invalid receiver type in test input.

---
 .../input.yaml                                                  | 2 --
 .../golden_error                                                | 2 +-
 .../input.yaml                                                  | 2 +-
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_invalid_parameter_include_paths_is_empty_string/input.yaml b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_invalid_parameter_include_paths_is_empty_string/input.yaml
index 22eef91cb3..ab98b92fe3 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_invalid_parameter_include_paths_is_empty_string/input.yaml
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_invalid_parameter_include_paths_is_empty_string/input.yaml
@@ -3,8 +3,6 @@ logging:
     receiver_1:
       type: files
       include_paths: ""
-    google:
-      type: google_cloud_logging
   service:
     pipelines:
       test_pipeline:
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
index 92721f642d..796893bbe3 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field unsupported_paramater not found in type confgenerator.LoggingReceiver
\ No newline at end of file
+  line 8: field unsupported_parameter not found in type confgenerator.LoggingReceiver
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/input.yaml b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/input.yaml
index 798be75e79..ff927d3a63 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/input.yaml
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/input.yaml
@@ -5,7 +5,7 @@ logging:
       listen_host: 1.1.1.1
       listen_port: 1111
       transport_protocol: tcp
-      unsupported_paramater: value_1
+      unsupported_parameter: value_1
   service:
     pipelines:
       default_pipeline:

From 81d08a7934f705aa414d627edf5fce67dff40a96 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Sat, 31 Jul 2021 16:09:55 -0400
Subject: [PATCH 03/41] Fix typos in exporters warnings.

---
 confgenerator/config.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/confgenerator/config.go b/confgenerator/config.go
index 2471b82dd0..7866a62e5e 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -210,7 +210,7 @@ func (l *Logging) Validate(platform string) error {
 		return err
 	}
 	if len(l.Exporters) > 0 {
-		log.Print(`The "metrics.exporters" field is no longer needed and will be ignored. This does not change any functionality. Please remove it from your configuration.`)
+		log.Print(`The "logging.exporters" field is no longer needed and will be ignored. This does not change any functionality. Please remove it from your configuration.`)
 	}
 	for id, r := range l.Receivers {
 		if err := r.ValidateType(subagent, "receiver", id, platform); err != nil {
@@ -318,7 +318,7 @@ func (m *Metrics) Validate(platform string) error {
 			return err
 		}
 		if len(p.ExporterIDs) > 0 {
-			log.Printf(`The "logging.service.pipelines.%s.exporters" field is deprecated and will be ignored. Please remove it from your configuration.`, id)
+			log.Printf(`The "metrics.service.pipelines.%s.exporters" field is deprecated and will be ignored. Please remove it from your configuration.`, id)
 		}
 	}
 	return nil

From 045a1e98db17d112ed338382b5eda49f7534abdb Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Fri, 30 Jul 2021 23:44:04 -0400
Subject: [PATCH 04/41] Add missing test for invalid logging and metrics
 receiver types on Linux.

---
 .../golden_error                                         | 1 +
 .../input.yaml                                           | 9 +++++++++
 .../linux/metrics-receiver_invalid_type_iis/golden_error | 1 +
 .../linux/metrics-receiver_invalid_type_iis/input.yaml   | 9 +++++++++
 .../metrics-receiver_invalid_type_mssql/golden_error     | 1 +
 .../linux/metrics-receiver_invalid_type_mssql/input.yaml | 9 +++++++++
 6 files changed, 30 insertions(+)
 create mode 100644 confgenerator/testdata/invalid/linux/logging-receiver_invalid_type_windows_event_log/golden_error
 create mode 100644 confgenerator/testdata/invalid/linux/logging-receiver_invalid_type_windows_event_log/input.yaml
 create mode 100644 confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_iis/golden_error
 create mode 100644 confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_iis/input.yaml
 create mode 100644 confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_mssql/golden_error
 create mode 100644 confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_mssql/input.yaml

diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_invalid_type_windows_event_log/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_invalid_type_windows_event_log/golden_error
new file mode 100644
index 0000000000..518f7d7ced
--- /dev/null
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_invalid_type_windows_event_log/golden_error
@@ -0,0 +1 @@
+logging receiver "receiver_1" with type "windows_event_log" is not supported. Supported logging receiver types: [files, syslog].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_invalid_type_windows_event_log/input.yaml b/confgenerator/testdata/invalid/linux/logging-receiver_invalid_type_windows_event_log/input.yaml
new file mode 100644
index 0000000000..b09fa0e6d9
--- /dev/null
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_invalid_type_windows_event_log/input.yaml
@@ -0,0 +1,9 @@
+logging:
+  receivers:
+    receiver_1:
+      type: windows_event_log
+      channels: [System,Application,Security]
+  service:
+    pipelines:
+      default_pipeline:
+        receivers: [receiver_1]
diff --git a/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_iis/golden_error b/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_iis/golden_error
new file mode 100644
index 0000000000..7b451cbbf2
--- /dev/null
+++ b/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_iis/golden_error
@@ -0,0 +1 @@
+metrics receiver "receiver_1" with type "iis" is not supported. Supported metrics receiver types: [hostmetrics].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_iis/input.yaml b/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_iis/input.yaml
new file mode 100644
index 0000000000..345e0423ca
--- /dev/null
+++ b/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_iis/input.yaml
@@ -0,0 +1,9 @@
+metrics:
+  receivers:
+    receiver_1:
+      type: iis
+      collection_interval: 60s
+  service:
+    pipelines:
+      custom_pipeline:
+        receivers: [receiver_1]
diff --git a/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_mssql/golden_error b/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_mssql/golden_error
new file mode 100644
index 0000000000..eb15fac5c5
--- /dev/null
+++ b/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_mssql/golden_error
@@ -0,0 +1 @@
+metrics receiver "receiver_1" with type "mssql" is not supported. Supported metrics receiver types: [hostmetrics].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_mssql/input.yaml b/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_mssql/input.yaml
new file mode 100644
index 0000000000..355f9d85df
--- /dev/null
+++ b/confgenerator/testdata/invalid/linux/metrics-receiver_invalid_type_mssql/input.yaml
@@ -0,0 +1,9 @@
+metrics:
+  receivers:
+    receiver_1:
+      type: mssql
+      collection_interval: 60s
+  service:
+    pipelines:
+      custom_pipeline:
+        receivers: [receiver_1]

From f5032930601b25ed972c2035f4c732df284608e8 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Fri, 9 Jul 2021 14:43:35 -0400
Subject: [PATCH 05/41] Add flow annotations to make the generated configs a
 bit nicer.

---
 confgenerator/built-in-config-linux.yaml   |  9 +++------
 confgenerator/built-in-config-windows.yaml | 16 ++++------------
 confgenerator/config.go                    | 16 ++++++++--------
 3 files changed, 15 insertions(+), 26 deletions(-)

diff --git a/confgenerator/built-in-config-linux.yaml b/confgenerator/built-in-config-linux.yaml
index 3be425f612..cd84fc39b6 100644
--- a/confgenerator/built-in-config-linux.yaml
+++ b/confgenerator/built-in-config-linux.yaml
@@ -8,8 +8,7 @@ logging:
   service:
     pipelines:
       default_pipeline:
-        receivers:
-        - syslog
+        receivers: [syslog]
 metrics:
   receivers:
     hostmetrics:
@@ -22,7 +21,5 @@ metrics:
   service:
     pipelines:
       default_pipeline:
-        receivers:
-        - hostmetrics
-        processors:
-        - metrics_filter
+        receivers: [hostmetrics]
+        processors: [metrics_filter]
diff --git a/confgenerator/built-in-config-windows.yaml b/confgenerator/built-in-config-windows.yaml
index 8f5a495857..ddd74d9d93 100644
--- a/confgenerator/built-in-config-windows.yaml
+++ b/confgenerator/built-in-config-windows.yaml
@@ -2,15 +2,11 @@ logging:
   receivers:
     windows_event_log:
       type: windows_event_log
-      channels:
-      - System
-      - Application
-      - Security
+      channels: [System, Application, Security]
   service:
     pipelines:
       default_pipeline:
-        receivers:
-        - windows_event_log
+        receivers: [windows_event_log]
 metrics:
   receivers:
     hostmetrics:
@@ -29,9 +25,5 @@ metrics:
   service:
     pipelines:
       default_pipeline:
-        receivers:
-        - hostmetrics
-        - iis
-        - mssql
-        processors:
-        - metrics_filter
+        receivers: [hostmetrics, iis, mssql]
+        processors: [metrics_filter]
diff --git a/confgenerator/config.go b/confgenerator/config.go
index 7866a62e5e..9cc26e9e57 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -106,7 +106,7 @@ type LoggingReceiverSyslog struct {
 }
 
 type LoggingReceiverWinevtlog struct {
-	Channels []string `yaml:"channels,omitempty"`
+	Channels []string `yaml:"channels,omitempty,flow"`
 }
 
 type LoggingReceiver struct {
@@ -144,9 +144,9 @@ type LoggingService struct {
 }
 
 type LoggingPipeline struct {
-	ReceiverIDs  []string `yaml:"receivers,omitempty"`
-	ProcessorIDs []string `yaml:"processors,omitempty"`
-	ExporterIDs  []string `yaml:"exporters,omitempty"`
+	ReceiverIDs  []string `yaml:"receivers,omitempty,flow"`
+	ProcessorIDs []string `yaml:"processors,omitempty,flow"`
+	ExporterIDs  []string `yaml:"exporters,omitempty,flow"`
 }
 
 // Ops Agent metrics config.
@@ -164,7 +164,7 @@ type MetricsReceiver struct {
 }
 
 type MetricsProcessorExcludeMetrics struct {
-	MetricsPattern []string `yaml:"metrics_pattern"`
+	MetricsPattern []string `yaml:"metrics_pattern,flow"`
 }
 
 type MetricsProcessor struct {
@@ -182,9 +182,9 @@ type MetricsService struct {
 }
 
 type MetricsPipeline struct {
-	ReceiverIDs  []string `yaml:"receivers"`
-	ProcessorIDs []string `yaml:"processors"`
-	ExporterIDs  []string `yaml:"exporters,omitempty"`
+	ReceiverIDs  []string `yaml:"receivers,flow"`
+	ProcessorIDs []string `yaml:"processors,flow"`
+	ExporterIDs  []string `yaml:"exporters,omitempty,flow"`
 }
 
 func (uc *UnifiedConfig) Validate(platform string) error {

From 5f31d91e910465e9617c87349ecc34692508c1ad Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Wed, 21 Jul 2021 21:10:45 -0400
Subject: [PATCH 06/41] Move fluentbit and otel config generation under
 confgenerator. Rename fluentbit package to "fluentbit".

---
 confgenerator/confgenerator.go                | 90 +++++++++----------
 .../conf => confgenerator/fluentbit}/conf.go  |  4 +-
 .../fluentbit}/conf_test.go                   |  2 +-
 {otel => confgenerator/otel}/conf.go          |  0
 {otel => confgenerator/otel}/conf_test.go     |  0
 5 files changed, 48 insertions(+), 48 deletions(-)
 rename {fluentbit/conf => confgenerator/fluentbit}/conf.go (99%)
 rename {fluentbit/conf => confgenerator/fluentbit}/conf_test.go (99%)
 rename {otel => confgenerator/otel}/conf.go (100%)
 rename {otel => confgenerator/otel}/conf_test.go (100%)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index f2485cb1f6..f7ba7e58ed 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -22,9 +22,9 @@ import (
 	"strings"
 	"text/template"
 
-	"github.com/GoogleCloudPlatform/ops-agent/fluentbit/conf"
+	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit"
+	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel"
 	"github.com/GoogleCloudPlatform/ops-agent/internal/version"
-	"github.com/GoogleCloudPlatform/ops-agent/otel"
 	"github.com/shirou/gopsutil/host"
 )
 
@@ -137,14 +137,14 @@ func generateOtelServices(receiverNameMap map[string]string, exporterNameMap map
 }
 
 // defaultTails returns the default Tail sections for the agents' own logs.
-func defaultTails(logsDir string, stateDir string, hostInfo *host.InfoStat) (tails []*conf.Tail) {
-	tails = []*conf.Tail{}
-	tailFluentbit := conf.Tail{
+func defaultTails(logsDir string, stateDir string, hostInfo *host.InfoStat) (tails []*fluentbit.Tail) {
+	tails = []*fluentbit.Tail{}
+	tailFluentbit := fluentbit.Tail{
 		Tag:  "ops-agent-fluent-bit",
 		DB:   filepathJoin(hostInfo.OS, stateDir, "buffers", "ops-agent-fluent-bit"),
 		Path: filepathJoin(hostInfo.OS, logsDir, "logging-module.log"),
 	}
-	tailCollectd := conf.Tail{
+	tailCollectd := fluentbit.Tail{
 		Tag:  "ops-agent-collectd",
 		DB:   filepathJoin(hostInfo.OS, stateDir, "buffers", "ops-agent-collectd"),
 		Path: filepathJoin(hostInfo.OS, logsDir, "metrics-module.log"),
@@ -158,8 +158,8 @@ func defaultTails(logsDir string, stateDir string, hostInfo *host.InfoStat) (tai
 }
 
 // defaultStackdriverOutputs returns the default Stackdriver sections for the agents' own logs.
-func defaultStackdriverOutputs(hostInfo *host.InfoStat) (stackdrivers []*conf.Stackdriver) {
-	return []*conf.Stackdriver{
+func defaultStackdriverOutputs(hostInfo *host.InfoStat) (stackdrivers []*fluentbit.Stackdriver) {
+	return []*fluentbit.Stackdriver{
 		{
 			Match:   "ops-agent-fluent-bit|ops-agent-collectd",
 			Workers: getWorkers(hostInfo),
@@ -216,14 +216,14 @@ func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir strin
 	fbTails := defaultTails(logsDir, stateDir, hostInfo)
 	userAgent, _ := getUserAgent("Google-Cloud-Ops-Agent-Logging", hostInfo)
 	fbStackdrivers := defaultStackdriverOutputs(hostInfo)
-	fbSyslogs := []*conf.Syslog{}
-	fbWinEventlogs := []*conf.WindowsEventlog{}
-	fbFilterParserGroups := []conf.FilterParserGroup{}
-	fbFilterAddLogNames := []*conf.FilterModifyAddLogName{}
-	fbFilterRewriteTags := []*conf.FilterRewriteTag{}
-	fbFilterRemoveLogNames := []*conf.FilterModifyRemoveLogName{}
-	jsonParsers := []*conf.ParserJSON{}
-	regexParsers := []*conf.ParserRegex{}
+	fbSyslogs := []*fluentbit.Syslog{}
+	fbWinEventlogs := []*fluentbit.WindowsEventlog{}
+	fbFilterParserGroups := []fluentbit.FilterParserGroup{}
+	fbFilterAddLogNames := []*fluentbit.FilterModifyAddLogName{}
+	fbFilterRewriteTags := []*fluentbit.FilterRewriteTag{}
+	fbFilterRemoveLogNames := []*fluentbit.FilterModifyRemoveLogName{}
+	jsonParsers := []*fluentbit.ParserJSON{}
+	regexParsers := []*fluentbit.ParserRegex{}
 
 	if logging != nil && logging.Service != nil {
 		// Override any user-specified exporters
@@ -237,7 +237,7 @@ func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir strin
 			p.ExporterIDs = []string{"google"}
 		}
 
-		extractedTails := []*conf.Tail{}
+		extractedTails := []*fluentbit.Tail{}
 		var err error
 		extractedTails, fbSyslogs, fbWinEventlogs, err = generateFluentBitInputs(logging.Receivers, logging.Service.Pipelines, stateDir, hostInfo)
 		if err != nil {
@@ -248,7 +248,7 @@ func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir strin
 		if err != nil {
 			return "", "", err
 		}
-		extractedStackdrivers := []*conf.Stackdriver{}
+		extractedStackdrivers := []*fluentbit.Stackdriver{}
 		fbFilterAddLogNames, fbFilterRewriteTags, fbFilterRemoveLogNames, extractedStackdrivers, err = extractExporterPlugins(logging.Exporters, logging.Service.Pipelines, hostInfo)
 		if err != nil {
 			return "", "", err
@@ -259,11 +259,11 @@ func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir strin
 			return "", "", err
 		}
 	}
-	mainConfig, err := conf.GenerateFluentBitMainConfig(fbTails, fbSyslogs, fbWinEventlogs, fbFilterParserGroups, fbFilterAddLogNames, fbFilterRewriteTags, fbFilterRemoveLogNames, fbStackdrivers, userAgent)
+	mainConfig, err := fluentbit.GenerateFluentBitMainConfig(fbTails, fbSyslogs, fbWinEventlogs, fbFilterParserGroups, fbFilterAddLogNames, fbFilterRewriteTags, fbFilterRemoveLogNames, fbStackdrivers, userAgent)
 	if err != nil {
 		return "", "", err
 	}
-	parserConfig, err := conf.GenerateFluentBitParserConfig(jsonParsers, regexParsers)
+	parserConfig, err := fluentbit.GenerateFluentBitParserConfig(jsonParsers, regexParsers)
 	if err != nil {
 		return "", "", err
 	}
@@ -468,10 +468,10 @@ func generateOtelProcessors(processors map[string]*MetricsProcessor, pipelines m
 	return excludeMetricsList, processorNameMap, nil
 }
 
-func generateFluentBitInputs(receivers map[string]*LoggingReceiver, pipelines map[string]*LoggingPipeline, stateDir string, hostInfo *host.InfoStat) ([]*conf.Tail, []*conf.Syslog, []*conf.WindowsEventlog, error) {
-	fbTails := []*conf.Tail{}
-	fbSyslogs := []*conf.Syslog{}
-	fbWinEventlogs := []*conf.WindowsEventlog{}
+func generateFluentBitInputs(receivers map[string]*LoggingReceiver, pipelines map[string]*LoggingPipeline, stateDir string, hostInfo *host.InfoStat) ([]*fluentbit.Tail, []*fluentbit.Syslog, []*fluentbit.WindowsEventlog, error) {
+	fbTails := []*fluentbit.Tail{}
+	fbSyslogs := []*fluentbit.Syslog{}
+	fbWinEventlogs := []*fluentbit.WindowsEventlog{}
 	fileReceiverFactories, syslogReceiverFactories, wineventlogReceiverFactories, err := extractReceiverFactories(receivers)
 	if err != nil {
 		return nil, nil, nil, err
@@ -480,7 +480,7 @@ func generateFluentBitInputs(receivers map[string]*LoggingReceiver, pipelines ma
 		p := pipelines[pID]
 		for _, rID := range p.ReceiverIDs {
 			if f, ok := fileReceiverFactories[rID]; ok {
-				fbTail := conf.Tail{
+				fbTail := fluentbit.Tail{
 					Tag:  fmt.Sprintf("%s.%s", pID, rID),
 					DB:   filepathJoin(hostInfo.OS, stateDir, "buffers", pID+"_"+rID),
 					Path: strings.Join(f.IncludePaths, ","),
@@ -492,7 +492,7 @@ func generateFluentBitInputs(receivers map[string]*LoggingReceiver, pipelines ma
 				continue
 			}
 			if f, ok := syslogReceiverFactories[rID]; ok {
-				fbSyslog := conf.Syslog{
+				fbSyslog := fluentbit.Syslog{
 					Tag:    fmt.Sprintf("%s.%s", pID, rID),
 					Listen: f.ListenHost,
 					Mode:   f.TransportProtocol,
@@ -502,7 +502,7 @@ func generateFluentBitInputs(receivers map[string]*LoggingReceiver, pipelines ma
 				continue
 			}
 			if f, ok := wineventlogReceiverFactories[rID]; ok {
-				fbWinlog := conf.WindowsEventlog{
+				fbWinlog := fluentbit.WindowsEventlog{
 					Tag:          fmt.Sprintf("%s.%s", pID, rID),
 					Channels:     strings.Join(f.Channels, ","),
 					Interval_Sec: "1",
@@ -516,17 +516,17 @@ func generateFluentBitInputs(receivers map[string]*LoggingReceiver, pipelines ma
 	return fbTails, fbSyslogs, fbWinEventlogs, nil
 }
 
-func generateFluentBitFilters(processors map[string]*LoggingProcessor, pipelines map[string]*LoggingPipeline) ([]conf.FilterParserGroup, error) {
+func generateFluentBitFilters(processors map[string]*LoggingProcessor, pipelines map[string]*LoggingPipeline) ([]fluentbit.FilterParserGroup, error) {
 	// Note: Keep each pipeline's filters in a separate group, because
 	// the order within that group is important, even though the order
 	// of the groups themselves does not matter.
-	groups := []conf.FilterParserGroup{}
+	groups := []fluentbit.FilterParserGroup{}
 	for _, pID := range sortedKeys(pipelines) {
-		fbFilterParsers := []*conf.FilterParser{}
+		fbFilterParsers := []*fluentbit.FilterParser{}
 		pipeline := pipelines[pID]
 		for _, processorID := range pipeline.ProcessorIDs {
 			p, ok := processors[processorID]
-			fbFilterParser := conf.FilterParser{
+			fbFilterParser := fluentbit.FilterParser{
 				Match:   fmt.Sprintf("%s.*", pID),
 				Parser:  processorID,
 				KeyName: "message",
@@ -544,26 +544,26 @@ func generateFluentBitFilters(processors map[string]*LoggingProcessor, pipelines
 }
 
 func extractExporterPlugins(exporters map[string]*LoggingExporter, pipelines map[string]*LoggingPipeline, hostInfo *host.InfoStat) (
-	[]*conf.FilterModifyAddLogName, []*conf.FilterRewriteTag, []*conf.FilterModifyRemoveLogName, []*conf.Stackdriver, error) {
-	fbFilterModifyAddLogNames := []*conf.FilterModifyAddLogName{}
-	fbFilterRewriteTags := []*conf.FilterRewriteTag{}
-	fbFilterModifyRemoveLogNames := []*conf.FilterModifyRemoveLogName{}
-	fbStackdrivers := []*conf.Stackdriver{}
+	[]*fluentbit.FilterModifyAddLogName, []*fluentbit.FilterRewriteTag, []*fluentbit.FilterModifyRemoveLogName, []*fluentbit.Stackdriver, error) {
+	fbFilterModifyAddLogNames := []*fluentbit.FilterModifyAddLogName{}
+	fbFilterRewriteTags := []*fluentbit.FilterRewriteTag{}
+	fbFilterModifyRemoveLogNames := []*fluentbit.FilterModifyRemoveLogName{}
+	fbStackdrivers := []*fluentbit.Stackdriver{}
 	stackdriverExporters := make(map[string][]string)
 	for _, pID := range sortedKeys(pipelines) {
 		pipeline := pipelines[pID]
 		for _, exporterID := range pipeline.ExporterIDs {
 			// for each receiver, generate a output plugin with the specified receiver id
 			for _, rID := range pipeline.ReceiverIDs {
-				fbFilterModifyAddLogNames = append(fbFilterModifyAddLogNames, &conf.FilterModifyAddLogName{
+				fbFilterModifyAddLogNames = append(fbFilterModifyAddLogNames, &fluentbit.FilterModifyAddLogName{
 					Match:   fmt.Sprintf("%s.%s", pID, rID),
 					LogName: rID,
 				})
 				// generate single rewriteTag for this pipeline
-				fbFilterRewriteTags = append(fbFilterRewriteTags, &conf.FilterRewriteTag{
+				fbFilterRewriteTags = append(fbFilterRewriteTags, &fluentbit.FilterRewriteTag{
 					Match: fmt.Sprintf("%s.%s", pID, rID),
 				})
-				fbFilterModifyRemoveLogNames = append(fbFilterModifyRemoveLogNames, &conf.FilterModifyRemoveLogName{
+				fbFilterModifyRemoveLogNames = append(fbFilterModifyRemoveLogNames, &fluentbit.FilterModifyRemoveLogName{
 					Match: rID,
 				})
 				stackdriverExporters[exporterID] = append(stackdriverExporters[exporterID], rID)
@@ -571,7 +571,7 @@ func extractExporterPlugins(exporters map[string]*LoggingExporter, pipelines map
 		}
 	}
 	for _, tags := range stackdriverExporters {
-		fbStackdrivers = append(fbStackdrivers, &conf.Stackdriver{
+		fbStackdrivers = append(fbStackdrivers, &fluentbit.Stackdriver{
 			Match:   strings.Join(tags, "|"),
 			Workers: getWorkers(hostInfo),
 		})
@@ -579,21 +579,21 @@ func extractExporterPlugins(exporters map[string]*LoggingExporter, pipelines map
 	return fbFilterModifyAddLogNames, fbFilterRewriteTags, fbFilterModifyRemoveLogNames, fbStackdrivers, nil
 }
 
-func extractFluentBitParsers(processors map[string]*LoggingProcessor) ([]*conf.ParserJSON, []*conf.ParserRegex, error) {
-	fbJSONParsers := []*conf.ParserJSON{}
-	fbRegexParsers := []*conf.ParserRegex{}
+func extractFluentBitParsers(processors map[string]*LoggingProcessor) ([]*fluentbit.ParserJSON, []*fluentbit.ParserRegex, error) {
+	fbJSONParsers := []*fluentbit.ParserJSON{}
+	fbRegexParsers := []*fluentbit.ParserRegex{}
 	for _, name := range sortedKeys(processors) {
 		p := processors[name]
 		switch t := p.Type; t {
 		case "parse_json":
-			fbJSONParser := conf.ParserJSON{
+			fbJSONParser := fluentbit.ParserJSON{
 				Name:       name,
 				TimeKey:    p.TimeKey,
 				TimeFormat: p.TimeFormat,
 			}
 			fbJSONParsers = append(fbJSONParsers, &fbJSONParser)
 		case "parse_regex":
-			fbRegexParser := conf.ParserRegex{
+			fbRegexParser := fluentbit.ParserRegex{
 				Name:       name,
 				Regex:      p.Regex,
 				TimeKey:    p.TimeKey,
diff --git a/fluentbit/conf/conf.go b/confgenerator/fluentbit/conf.go
similarity index 99%
rename from fluentbit/conf/conf.go
rename to confgenerator/fluentbit/conf.go
index 9a96202a65..f1a955fa31 100644
--- a/fluentbit/conf/conf.go
+++ b/confgenerator/fluentbit/conf.go
@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package conf provides data structures to represent and generate fluentBit configuration.
-package conf
+// Package fluentbit provides data structures to represent and generate fluentBit configuration.
+package fluentbit
 
 import (
 	"fmt"
diff --git a/fluentbit/conf/conf_test.go b/confgenerator/fluentbit/conf_test.go
similarity index 99%
rename from fluentbit/conf/conf_test.go
rename to confgenerator/fluentbit/conf_test.go
index 4b8281fab2..fc7a9166d4 100644
--- a/fluentbit/conf/conf_test.go
+++ b/confgenerator/fluentbit/conf_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package conf
+package fluentbit
 
 import (
 	"testing"
diff --git a/otel/conf.go b/confgenerator/otel/conf.go
similarity index 100%
rename from otel/conf.go
rename to confgenerator/otel/conf.go
diff --git a/otel/conf_test.go b/confgenerator/otel/conf_test.go
similarity index 100%
rename from otel/conf_test.go
rename to confgenerator/otel/conf_test.go

From 1201e3d9cbc6440305d1c272e0df5f6b446413e1 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 29 Jul 2021 01:17:01 -0400
Subject: [PATCH 07/41] Annotate optional fields with the `config:"optional"`
 tag.

---
 confgenerator/config.go | 26 ++++++++++----------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/confgenerator/config.go b/confgenerator/config.go
index 9cc26e9e57..65b35739bd 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -26,16 +26,6 @@ import (
 	yaml "gopkg.in/yaml.v2"
 )
 
-// TODO(lingshi): Figure out a cleaner way to do "required" validation.
-// The "omitempty" annotation is reserved to make YAML marshal/unmarshal results reasonable.
-var requiredFields = []string{
-	"channels",
-	"include_paths",
-	"listen_host",
-	"listen_port",
-	"regex",
-}
-
 // Ops Agent config.
 type UnifiedConfig struct {
 	Logging *Logging `yaml:"logging"`
@@ -96,11 +86,11 @@ type Logging struct {
 
 type LoggingReceiverFiles struct {
 	IncludePaths []string `yaml:"include_paths,omitempty"`
-	ExcludePaths []string `yaml:"exclude_paths,omitempty"` // optional
+	ExcludePaths []string `yaml:"exclude_paths,omitempty" config:"optional"`
 }
 
 type LoggingReceiverSyslog struct {
-	TransportProtocol string `yaml:"transport_protocol,omitempty"` // one of "tcp" or "udp"
+	TransportProtocol string `yaml:"transport_protocol,omitempty" config:"optional"` // one of "tcp" or "udp"
 	ListenHost        string `yaml:"listen_host,omitempty"`
 	ListenPort        uint16 `yaml:"listen_port,omitempty"`
 }
@@ -118,9 +108,9 @@ type LoggingReceiver struct {
 }
 
 type LoggingProcessorParseJson struct {
-	Field      string `yaml:"field,omitempty"`       // optional, default to "message"
-	TimeKey    string `yaml:"time_key,omitempty"`    // optional, by default does not parse timestamp
-	TimeFormat string `yaml:"time_format,omitempty"` // optional, must be provided if time_key is present
+	Field      string `yaml:"field,omitempty" config:"optional"`       // default to "message"
+	TimeKey    string `yaml:"time_key,omitempty" config:"optional"`    // by default does not parse timestamp
+	TimeFormat string `yaml:"time_format,omitempty" config:"optional"` // must be provided if time_key is present
 }
 
 type LoggingProcessorParseRegex struct {
@@ -395,9 +385,13 @@ func collectYamlFields(s interface{}) []yamlField {
 				// Expand inline structs.
 				parameters = append(parameters, recurse(v)...)
 			} else if f.PkgPath == "" { // skip private non-struct fields
+				t, e := f.Tag.Lookup("config")
+				if !e {
+					t = ""
+				}
 				parameters = append(parameters, yamlField{
 					Name:     n,
-					Required: sliceContains(requiredFields, n),
+					Required: t != "optional",
 					Value:    v.Interface(),
 					IsZero:   v.IsZero(),
 				})

From 9311420b37e7d75d6c4ec4067ded2b3368166987 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 29 Jul 2021 20:19:44 -0400
Subject: [PATCH 08/41] Rename "component" to "kind" everywhere.

---
 confgenerator/config.go | 48 ++++++++++++++++++++---------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/confgenerator/config.go b/confgenerator/config.go
index 65b35739bd..2750348ac0 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -323,31 +323,31 @@ func sliceContains(slice []string, value string) bool {
 	return false
 }
 
-func (c *configComponent) ValidateType(subagent string, component string, id string, platform string) error {
-	supportedTypes := supportedComponentTypes[platform+"_"+subagent+"_"+component]
+func (c *configComponent) ValidateType(subagent string, kind string, id string, platform string) error {
+	supportedTypes := supportedComponentTypes[platform+"_"+subagent+"_"+kind]
 	if !sliceContains(supportedTypes, c.Type) {
 		// e.g. metrics receiver "receiver_1" with type "unsupported_type" is not supported.
 		// Supported metrics receiver types: [hostmetrics, iis, mssql].
 		return fmt.Errorf(`%s %s %q with type %q is not supported. Supported %s %s types: [%s].`,
-			subagent, component, id, c.Type, subagent, component, strings.Join(supportedTypes, ", "))
+			subagent, kind, id, c.Type, subagent, kind, strings.Join(supportedTypes, ", "))
 	}
 	return nil
 }
 
-func (r *LoggingReceiver) ValidateParameters(subagent string, component string, id string) error {
-	return validateParameters(*r, subagent, component, id, r.Type)
+func (r *LoggingReceiver) ValidateParameters(subagent string, kind string, id string) error {
+	return validateParameters(*r, subagent, kind, id, r.Type)
 }
 
-func (p *LoggingProcessor) ValidateParameters(subagent string, component string, id string) error {
-	return validateParameters(*p, subagent, component, id, p.Type)
+func (p *LoggingProcessor) ValidateParameters(subagent string, kind string, id string) error {
+	return validateParameters(*p, subagent, kind, id, p.Type)
 }
 
-func (r *MetricsReceiver) ValidateParameters(subagent string, component string, id string) error {
-	return validateParameters(*r, subagent, component, id, r.Type)
+func (r *MetricsReceiver) ValidateParameters(subagent string, kind string, id string) error {
+	return validateParameters(*r, subagent, kind, id, r.Type)
 }
 
-func (p *MetricsProcessor) ValidateParameters(subagent string, component string, id string) error {
-	return validateParameters(*p, subagent, component, id, p.Type)
+func (p *MetricsProcessor) ValidateParameters(subagent string, kind string, id string) error {
+	return validateParameters(*p, subagent, kind, id, p.Type)
 }
 
 type yamlField struct {
@@ -402,7 +402,7 @@ func collectYamlFields(s interface{}) []yamlField {
 	return recurse(reflect.ValueOf(s))
 }
 
-func validateParameters(s interface{}, subagent string, component string, id string, componentType string) error {
+func validateParameters(s interface{}, subagent string, kind string, id string, componentType string) error {
 	supportedParameters := supportedParameters[componentType]
 	// Include type when checking.
 	allParameters := []string{"type"}
@@ -415,20 +415,20 @@ func validateParameters(s interface{}, subagent string, component string, id str
 				// e.g. parameter "transport_protocol" in "files" type logging receiver "receiver_1" is not supported.
 				// Supported parameters: [include_paths, exclude_paths].
 				return fmt.Errorf(`%s is not supported. Supported parameters: [%s].`,
-					parameterErrorPrefix(subagent, component, id, componentType, p.Name), strings.Join(supportedParameters, ", "))
+					parameterErrorPrefix(subagent, kind, id, componentType, p.Name), strings.Join(supportedParameters, ", "))
 			}
 			continue
 		}
 		if p.IsZero && p.Required {
 			// e.g. parameter "include_paths" in "files" type logging receiver "receiver_1" is required.
-			return fmt.Errorf(`%s is required.`, parameterErrorPrefix(subagent, component, id, componentType, p.Name))
+			return fmt.Errorf(`%s is required.`, parameterErrorPrefix(subagent, kind, id, componentType, p.Name))
 		}
 		if hasAdditionalValidation {
 			if f, ok := additionalValidation[p.Name]; ok {
 				if err := f(p.Value); err != nil {
 					// e.g. parameter "collection_interval" in "hostmetrics" type metrics receiver "receiver_1"
 					// has invalid value "1s": below the minimum threshold of "10s".
-					return fmt.Errorf(`%s has invalid value %q: %s`, parameterErrorPrefix(subagent, component, id, componentType, p.Name), p.Value, err)
+					return fmt.Errorf(`%s has invalid value %q: %s`, parameterErrorPrefix(subagent, kind, id, componentType, p.Name), p.Value, err)
 				}
 			}
 		}
@@ -592,26 +592,26 @@ func findInvalid(actual []string, allowed map[string]bool) []string {
 	return invalid
 }
 
-func validateComponentIds(components interface{}, subagent string, component string) error {
+func validateComponentIds(components interface{}, subagent string, kind string) error {
 	for _, id := range sortedKeys(components) {
 		if strings.HasPrefix(id, "lib:") {
 			// e.g. logging receiver id "lib:abc" is not allowed because prefix 'lib:' is reserved for pre-defined receivers.
 			return fmt.Errorf(`%s %s id %q is not allowed because prefix 'lib:' is reserved for pre-defined %ss.`,
-				subagent, component, id, component)
+				subagent, kind, id, kind)
 		}
 	}
 	return nil
 }
 
-func validateComponentKeys(components interface{}, refs []string, subagent string, component string, pipeline string) error {
+func validateComponentKeys(components interface{}, refs []string, subagent string, kind string, pipeline string) error {
 	invalid := findInvalid(refs, mapKeys(components))
 	if len(invalid) > 0 {
-		return fmt.Errorf("%s %s %q from pipeline %q is not defined.", subagent, component, invalid[0], pipeline)
+		return fmt.Errorf("%s %s %q from pipeline %q is not defined.", subagent, kind, invalid[0], pipeline)
 	}
 	return nil
 }
 
-func validateComponentTypeCounts(components interface{}, refs []string, subagent string, component string) error {
+func validateComponentTypeCounts(components interface{}, refs []string, subagent string, kind string) error {
 	r := map[string]int{}
 	cm := reflect.ValueOf(components)
 	for _, id := range refs {
@@ -627,9 +627,9 @@ func validateComponentTypeCounts(components interface{}, refs []string, subagent
 		}
 		if limit, ok := componentTypeLimits[t]; ok && r[t] > limit {
 			if limit == 1 {
-				return fmt.Errorf("at most one %s %s with type %q is allowed.", subagent, component, t)
+				return fmt.Errorf("at most one %s %s with type %q is allowed.", subagent, kind, t)
 			}
-			return fmt.Errorf("at most %d %s %ss with type %q are allowed.", limit, subagent, component, t)
+			return fmt.Errorf("at most %d %s %ss with type %q are allowed.", limit, subagent, kind, t)
 		}
 	}
 	return nil
@@ -639,6 +639,6 @@ func validateComponentTypeCounts(components interface{}, refs []string, subagent
 // id is the id of the receiver, processor, or exporter.
 // componentType is the type of the receiver, processor, or exporter, e.g., "hostmetrics".
 // parameter is name of the parameter.
-func parameterErrorPrefix(subagent string, component string, id string, componentType string, parameter string) string {
-	return fmt.Sprintf(`parameter %q in %q type %s %s %q`, parameter, componentType, subagent, component, id)
+func parameterErrorPrefix(subagent string, kind string, id string, componentType string, parameter string) string {
+	return fmt.Sprintf(`parameter %q in %q type %s %s %q`, parameter, componentType, subagent, kind, id)
 }

From 5288ebf18c8235e507afba2a457839c295e36f41 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 29 Jul 2021 21:17:35 -0400
Subject: [PATCH 09/41] Use a custom unmarshaler for the logging receiver
 types.

---
 confgenerator/confgenerator.go                |  21 +--
 confgenerator/config.go                       | 120 +++++++++++++++---
 confgenerator/confmerger.go                   |  32 ++---
 .../golden_error                              |   2 +-
 .../golden_error                              |   3 +-
 .../golden_error                              |   3 +-
 .../golden_error                              |   2 +-
 .../golden_error                              |   3 +-
 .../golden_error                              |   3 +-
 .../golden_error                              |   3 +-
 .../golden_error                              |   3 +-
 .../golden_error                              |   2 +-
 .../golden_error                              |   2 +-
 .../golden_error                              |   3 +-
 .../golden_error                              |   3 +-
 .../golden_error                              |   3 +-
 .../golden_error                              |   3 +-
 .../golden_error                              |   2 +-
 .../golden_error                              |   3 +-
 19 files changed, 154 insertions(+), 62 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index f7ba7e58ed..b46c866212 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -54,7 +54,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 		// TODO: Refactor remaining code to not consult these fields
 		metrics.Exporters = map[string]*MetricsExporter{
 			"google": &MetricsExporter{
-				configComponent: configComponent{Type: "google_cloud_monitoring"},
+				configComponent: configComponent{ComponentType: "google_cloud_monitoring"},
 			},
 		}
 		for _, p := range metrics.Service.Pipelines {
@@ -230,7 +230,7 @@ func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir strin
 		// TODO: Refactor remaining code to not consult these fields
 		logging.Exporters = map[string]*LoggingExporter{
 			"google": &LoggingExporter{
-				configComponent: configComponent{Type: "google_cloud_logging"},
+				configComponent: configComponent{ComponentType: "google_cloud_logging"},
 			},
 		}
 		for _, p := range logging.Service.Pipelines {
@@ -306,7 +306,7 @@ func extractOtelReceiverFactories(receivers map[string]*MetricsReceiver) (map[st
 	mssqlReceiverFactories := map[string]*mssqlReceiverFactory{}
 	iisReceiverFactories := map[string]*iisReceiverFactory{}
 	for n, r := range receivers {
-		switch r.Type {
+		switch r.Type() {
 		case "hostmetrics":
 			hostmetricsReceiverFactories[n] = &hostmetricsReceiverFactory{
 				CollectionInterval: r.CollectionInterval,
@@ -327,7 +327,7 @@ func extractOtelReceiverFactories(receivers map[string]*MetricsReceiver) (map[st
 func extractOtelProcessorFactories(processors map[string]*MetricsProcessor) (map[string]*excludemetricsProcessorFactory, error) {
 	excludemetricsProcessorFactories := map[string]*excludemetricsProcessorFactory{}
 	for n, p := range processors {
-		switch p.Type {
+		switch p.Type() {
 		case "exclude_metrics":
 			excludemetricsProcessorFactories[n] = &excludemetricsProcessorFactory{
 				MetricsPattern: p.MetricsPattern,
@@ -337,24 +337,27 @@ func extractOtelProcessorFactories(processors map[string]*MetricsProcessor) (map
 	return excludemetricsProcessorFactories, nil
 }
 
-func extractReceiverFactories(receivers map[string]*LoggingReceiver) (map[string]*fileReceiverFactory, map[string]*syslogReceiverFactory, map[string]*wineventlogReceiverFactory, error) {
+func extractReceiverFactories(receivers map[string]LoggingReceiver) (map[string]*fileReceiverFactory, map[string]*syslogReceiverFactory, map[string]*wineventlogReceiverFactory, error) {
 	fileReceiverFactories := map[string]*fileReceiverFactory{}
 	syslogReceiverFactories := map[string]*syslogReceiverFactory{}
 	wineventlogReceiverFactories := map[string]*wineventlogReceiverFactory{}
 	for rID, r := range receivers {
-		switch r.Type {
+		switch r.Type() {
 		case "files":
+			r := r.(*LoggingReceiverFiles)
 			fileReceiverFactories[rID] = &fileReceiverFactory{
 				IncludePaths: r.IncludePaths,
 				ExcludePaths: r.ExcludePaths,
 			}
 		case "syslog":
+			r := r.(*LoggingReceiverSyslog)
 			syslogReceiverFactories[rID] = &syslogReceiverFactory{
 				TransportProtocol: r.TransportProtocol,
 				ListenHost:        r.ListenHost,
 				ListenPort:        r.ListenPort,
 			}
 		case "windows_event_log":
+			r := r.(*LoggingReceiverWinevtlog)
 			wineventlogReceiverFactories[rID] = &wineventlogReceiverFactory{
 				Channels: r.Channels,
 			}
@@ -415,7 +418,7 @@ func generateOtelExporters(exporters map[string]*MetricsExporter, pipelines map[
 			if !ok {
 				continue
 			}
-			switch exporter.Type {
+			switch exporter.Type() {
 			case "google_cloud_monitoring":
 				if _, ok := exportNameMap[eID]; !ok {
 					stackdriver := otel.Stackdriver{
@@ -468,7 +471,7 @@ func generateOtelProcessors(processors map[string]*MetricsProcessor, pipelines m
 	return excludeMetricsList, processorNameMap, nil
 }
 
-func generateFluentBitInputs(receivers map[string]*LoggingReceiver, pipelines map[string]*LoggingPipeline, stateDir string, hostInfo *host.InfoStat) ([]*fluentbit.Tail, []*fluentbit.Syslog, []*fluentbit.WindowsEventlog, error) {
+func generateFluentBitInputs(receivers map[string]LoggingReceiver, pipelines map[string]*LoggingPipeline, stateDir string, hostInfo *host.InfoStat) ([]*fluentbit.Tail, []*fluentbit.Syslog, []*fluentbit.WindowsEventlog, error) {
 	fbTails := []*fluentbit.Tail{}
 	fbSyslogs := []*fluentbit.Syslog{}
 	fbWinEventlogs := []*fluentbit.WindowsEventlog{}
@@ -584,7 +587,7 @@ func extractFluentBitParsers(processors map[string]*LoggingProcessor) ([]*fluent
 	fbRegexParsers := []*fluentbit.ParserRegex{}
 	for _, name := range sortedKeys(processors) {
 		p := processors[name]
-		switch t := p.Type; t {
+		switch t := p.Type(); t {
 		case "parse_json":
 			fbJSONParser := fluentbit.ParserJSON{
 				Name:       name,
diff --git a/confgenerator/config.go b/confgenerator/config.go
index 2750348ac0..80cf450ff5 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -72,39 +72,109 @@ func ParseUnifiedConfigAndValidate(input []byte, platform string) (UnifiedConfig
 	return config, nil
 }
 
+type component interface {
+	Type() string
+	ValidateType(subagent string, component string, id string, platform string) error
+	ValidateParameters(subagent string, component string, id string) error
+	//Validate() error
+}
+
 type configComponent struct {
-	Type string `yaml:"type"`
+	ComponentType string `yaml:"type"`
+}
+
+func (c *configComponent) Type() string {
+	return c.ComponentType
+}
+
+func unmarshalComponentYaml(typeMap map[string]func() component, inner *interface{}, unmarshal func(interface{}) error) error {
+	c := configComponent{}
+	unmarshal(&c) // Get the type; ignore the error
+	f := typeMap[c.Type()]
+	if f == nil {
+		return fmt.Errorf("Unknown type %q", c.Type())
+	}
+	*inner = f()
+	return unmarshal(*inner)
 }
 
 // Ops Agent logging config.
+type loggingReceiverMap map[string]LoggingReceiver
 type Logging struct {
-	Receivers  map[string]*LoggingReceiver  `yaml:"receivers,omitempty"`
+	Receivers  loggingReceiverMap           `yaml:"receivers,omitempty"`
 	Processors map[string]*LoggingProcessor `yaml:"processors,omitempty"`
 	Exporters  map[string]*LoggingExporter  `yaml:"exporters,omitempty"`
 	Service    *LoggingService              `yaml:"service"`
 }
 
-type LoggingReceiverFiles struct {
+type LoggingReceiver interface {
+	component
+}
+
+type LoggingReceiverFiles struct { // Type "files"
+	configComponent `yaml:",inline"`
+
 	IncludePaths []string `yaml:"include_paths,omitempty"`
 	ExcludePaths []string `yaml:"exclude_paths,omitempty" config:"optional"`
 }
 
-type LoggingReceiverSyslog struct {
+func init() {
+	registerLoggingReceiverType("files", func() component { return &LoggingReceiverFiles{} })
+}
+
+type LoggingReceiverSyslog struct { // Type "syslog"
+	configComponent `yaml:",inline"`
+
 	TransportProtocol string `yaml:"transport_protocol,omitempty" config:"optional"` // one of "tcp" or "udp"
 	ListenHost        string `yaml:"listen_host,omitempty"`
 	ListenPort        uint16 `yaml:"listen_port,omitempty"`
 }
 
-type LoggingReceiverWinevtlog struct {
-	Channels []string `yaml:"channels,omitempty,flow"`
+func init() {
+	registerLoggingReceiverType("syslog", func() component { return &LoggingReceiverSyslog{} })
 }
 
-type LoggingReceiver struct {
+type LoggingReceiverWinevtlog struct { // Type "windows_event_log"
 	configComponent `yaml:",inline"`
 
-	LoggingReceiverSyslog    `yaml:",inline"` // Type "syslog"
-	LoggingReceiverFiles     `yaml:",inline"` // Type "files"
-	LoggingReceiverWinevtlog `yaml:",inline"` // Type "windows_event_log"
+	Channels []string `yaml:"channels,omitempty,flow"`
+}
+
+func init() {
+	registerLoggingReceiverType("windows_event_log", func() component { return &LoggingReceiverWinevtlog{} })
+}
+
+var loggingReceiverTypes = map[string]func() component{}
+
+func registerLoggingReceiverType(name string, constructor func() component) error {
+	if _, ok := loggingReceiverTypes[name]; ok {
+		return fmt.Errorf("Duplicate receiver type: %q", name)
+	}
+	loggingReceiverTypes[name] = constructor
+	return nil
+}
+
+// Wrapper type to store the unmarshaled YAML value.
+type loggingReceiverWrapper struct {
+	inner interface{}
+}
+
+func (l *loggingReceiverWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	return unmarshalComponentYaml(loggingReceiverTypes, &l.inner, unmarshal)
+}
+
+func (m *loggingReceiverMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// Unmarshal into a temporary map to capture types.
+	tm := map[string]loggingReceiverWrapper{}
+	if err := unmarshal(&tm); err != nil {
+		return err
+	}
+	// Unwrap the structs.
+	*m = loggingReceiverMap{}
+	for k, r := range tm {
+		(*m)[k] = r.inner.(LoggingReceiver)
+	}
+	return nil
 }
 
 type LoggingProcessorParseJson struct {
@@ -325,29 +395,37 @@ func sliceContains(slice []string, value string) bool {
 
 func (c *configComponent) ValidateType(subagent string, kind string, id string, platform string) error {
 	supportedTypes := supportedComponentTypes[platform+"_"+subagent+"_"+kind]
-	if !sliceContains(supportedTypes, c.Type) {
+	if !sliceContains(supportedTypes, c.Type()) {
 		// e.g. metrics receiver "receiver_1" with type "unsupported_type" is not supported.
 		// Supported metrics receiver types: [hostmetrics, iis, mssql].
 		return fmt.Errorf(`%s %s %q with type %q is not supported. Supported %s %s types: [%s].`,
-			subagent, kind, id, c.Type, subagent, kind, strings.Join(supportedTypes, ", "))
+			subagent, kind, id, c.Type(), subagent, kind, strings.Join(supportedTypes, ", "))
 	}
 	return nil
 }
 
-func (r *LoggingReceiver) ValidateParameters(subagent string, kind string, id string) error {
-	return validateParameters(*r, subagent, kind, id, r.Type)
+func (r *LoggingReceiverFiles) ValidateParameters(subagent string, kind string, id string) error {
+	return validateParameters(*r, subagent, kind, id, r.Type())
+}
+
+func (r *LoggingReceiverSyslog) ValidateParameters(subagent string, kind string, id string) error {
+	return validateParameters(*r, subagent, kind, id, r.Type())
+}
+
+func (r *LoggingReceiverWinevtlog) ValidateParameters(subagent string, kind string, id string) error {
+	return validateParameters(*r, subagent, kind, id, r.Type())
 }
 
 func (p *LoggingProcessor) ValidateParameters(subagent string, kind string, id string) error {
-	return validateParameters(*p, subagent, kind, id, p.Type)
+	return validateParameters(*p, subagent, kind, id, p.Type())
 }
 
 func (r *MetricsReceiver) ValidateParameters(subagent string, kind string, id string) error {
-	return validateParameters(*r, subagent, kind, id, r.Type)
+	return validateParameters(*r, subagent, kind, id, r.Type())
 }
 
 func (p *MetricsProcessor) ValidateParameters(subagent string, kind string, id string) error {
-	return validateParameters(*p, subagent, kind, id, p.Type)
+	return validateParameters(*p, subagent, kind, id, p.Type())
 }
 
 type yamlField struct {
@@ -533,7 +611,11 @@ var (
 func mapKeys(m interface{}) map[string]bool {
 	keys := map[string]bool{}
 	switch m := m.(type) {
-	case map[string]*LoggingReceiver:
+	case map[string]LoggingReceiver:
+		for k := range m {
+			keys[k] = true
+		}
+	case loggingReceiverMap:
 		for k := range m {
 			keys[k] = true
 		}
@@ -619,7 +701,7 @@ func validateComponentTypeCounts(components interface{}, refs []string, subagent
 		if !v.IsValid() {
 			continue // Some reserved ids don't map to components.
 		}
-		t := v.Elem().FieldByName("Type").String()
+		t := v.Interface().(component).Type()
 		if _, ok := r[t]; ok {
 			r[t] += 1
 		} else {
diff --git a/confgenerator/confmerger.go b/confgenerator/confmerger.go
index b11853749c..c7e947ba35 100644
--- a/confgenerator/confmerger.go
+++ b/confgenerator/confmerger.go
@@ -28,12 +28,10 @@ var (
 	builtInConfStructs = map[string]*UnifiedConfig{
 		"linux": &UnifiedConfig{
 			Logging: &Logging{
-				Receivers: map[string]*LoggingReceiver{
-					"syslog": &LoggingReceiver{
-						configComponent: configComponent{Type: "files"},
-						LoggingReceiverFiles: LoggingReceiverFiles{
-							IncludePaths: []string{"/var/log/messages", "/var/log/syslog"},
-						},
+				Receivers: map[string]LoggingReceiver{
+					"syslog": &LoggingReceiverFiles{
+						configComponent: configComponent{ComponentType: "files"},
+						IncludePaths:    []string{"/var/log/messages", "/var/log/syslog"},
 					},
 				},
 				Processors: map[string]*LoggingProcessor{},
@@ -48,13 +46,13 @@ var (
 			Metrics: &Metrics{
 				Receivers: map[string]*MetricsReceiver{
 					"hostmetrics": &MetricsReceiver{
-						configComponent:    configComponent{Type: "hostmetrics"},
+						configComponent:    configComponent{ComponentType: "hostmetrics"},
 						CollectionInterval: "60s",
 					},
 				},
 				Processors: map[string]*MetricsProcessor{
 					"metrics_filter": &MetricsProcessor{
-						configComponent: configComponent{Type: "exclude_metrics"},
+						configComponent: configComponent{ComponentType: "exclude_metrics"},
 					},
 				},
 				Service: &MetricsService{
@@ -69,12 +67,10 @@ var (
 		},
 		"windows": &UnifiedConfig{
 			Logging: &Logging{
-				Receivers: map[string]*LoggingReceiver{
-					"windows_event_log": &LoggingReceiver{
-						configComponent: configComponent{Type: "windows_event_log"},
-						LoggingReceiverWinevtlog: LoggingReceiverWinevtlog{
-							Channels: []string{"System", "Application", "Security"},
-						},
+				Receivers: map[string]LoggingReceiver{
+					"windows_event_log": &LoggingReceiverWinevtlog{
+						configComponent: configComponent{ComponentType: "windows_event_log"},
+						Channels:        []string{"System", "Application", "Security"},
 					},
 				},
 				Processors: map[string]*LoggingProcessor{},
@@ -89,21 +85,21 @@ var (
 			Metrics: &Metrics{
 				Receivers: map[string]*MetricsReceiver{
 					"hostmetrics": &MetricsReceiver{
-						configComponent:    configComponent{Type: "hostmetrics"},
+						configComponent:    configComponent{ComponentType: "hostmetrics"},
 						CollectionInterval: "60s",
 					},
 					"iis": &MetricsReceiver{
-						configComponent:    configComponent{Type: "iis"},
+						configComponent:    configComponent{ComponentType: "iis"},
 						CollectionInterval: "60s",
 					},
 					"mssql": &MetricsReceiver{
-						configComponent:    configComponent{Type: "mssql"},
+						configComponent:    configComponent{ComponentType: "mssql"},
 						CollectionInterval: "60s",
 					},
 				},
 				Processors: map[string]*MetricsProcessor{
 					"metrics_filter": &MetricsProcessor{
-						configComponent: configComponent{Type: "exclude_metrics"},
+						configComponent: configComponent{ComponentType: "exclude_metrics"},
 					},
 				},
 				Service: &MetricsService{
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error
index b5de85b7f5..e31d67aed5 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: cannot unmarshal !!str `value_1` into []string
\ No newline at end of file
+  line 8: field channels not found in type confgenerator.LoggingReceiverFiles
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error
index 54720290bc..c3cce76ba1 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error
@@ -1 +1,2 @@
-parameter "listen_host" in "files" type logging receiver "receiver_1" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 8: field listen_host not found in type confgenerator.LoggingReceiverFiles
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error
index bfa2e4c43b..b27adabc01 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error
@@ -1 +1,2 @@
-parameter "listen_port" in "files" type logging receiver "receiver_1" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 8: field listen_port not found in type confgenerator.LoggingReceiverFiles
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error
index a4bb1801dd..c8851b03ee 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field field_1 not found in type confgenerator.LoggingReceiver
\ No newline at end of file
+  line 8: field field_1 not found in type confgenerator.LoggingReceiverFiles
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error
index 51fd8f1464..854454d3de 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error
@@ -1 +1,2 @@
-parameter "transport_protocol" in "files" type logging receiver "receiver_1" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 8: field transport_protocol not found in type confgenerator.LoggingReceiverFiles
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error
index 174edf95d1..43b582afee 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error
@@ -1 +1,2 @@
-parameter "channels" in "syslog" type logging receiver "receiver_1" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 8: field channels not found in type confgenerator.LoggingReceiverSyslog
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error
index 37fada5e83..18dd4af940 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error
@@ -1 +1,2 @@
-parameter "exclude_paths" in "syslog" type logging receiver "receiver_1" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 8: field exclude_paths not found in type confgenerator.LoggingReceiverSyslog
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error
index 13de1aee25..d6fa1c9d70 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error
@@ -1 +1,2 @@
-parameter "include_paths" in "syslog" type logging receiver "receiver_1" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 8: field include_paths not found in type confgenerator.LoggingReceiverSyslog
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
index 796893bbe3..a0fb365def 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field unsupported_parameter not found in type confgenerator.LoggingReceiver
\ No newline at end of file
+  line 8: field unsupported_parameter not found in type confgenerator.LoggingReceiverSyslog
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
index 651eda5f6a..56a44eb816 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-logging receiver "receiver_1" with type "unsupported_type" is not supported. Supported logging receiver types: [files, syslog].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: Unknown type "unsupported_type"
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error
index afc3e100fe..55455817c4 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error
@@ -1 +1,2 @@
-parameter "exclude_paths" in "windows_event_log" type logging receiver "receiver_1" is not supported. Supported parameters: [channels].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 6: field exclude_paths not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error
index e1960b755b..bf4b26635f 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error
@@ -1 +1,2 @@
-parameter "include_paths" in "windows_event_log" type logging receiver "receiver_1" is not supported. Supported parameters: [channels].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 6: field include_paths not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error
index ec9886ebc6..4d68bd71c9 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error
@@ -1 +1,2 @@
-parameter "listen_host" in "windows_event_log" type logging receiver "receiver_1" is not supported. Supported parameters: [channels].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 6: field listen_host not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error
index 9416172109..1c3456feb4 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error
@@ -1 +1,2 @@
-parameter "listen_port" in "windows_event_log" type logging receiver "receiver_1" is not supported. Supported parameters: [channels].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 6: field listen_port not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error
index 516f0dba8c..191fc797ac 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 6: field unsupported_parameter not found in type confgenerator.LoggingReceiver
\ No newline at end of file
+  line 6: field unsupported_parameter not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error
index 550faacb05..e3cdc24243 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error
@@ -1 +1,2 @@
-parameter "transport_protocol" in "windows_event_log" type logging receiver "receiver_1" is not supported. Supported parameters: [channels].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+  line 6: field transport_protocol not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file

From 5fd483260c740a2c4e53b6152bf3fc01d57d85ca Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Fri, 30 Jul 2021 19:25:49 -0400
Subject: [PATCH 10/41] Rename type map to registry; embed subagent and kind in
 the registry.

---
 confgenerator/config.go | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/confgenerator/config.go b/confgenerator/config.go
index 80cf450ff5..c28f506ee9 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -87,10 +87,16 @@ func (c *configComponent) Type() string {
 	return c.ComponentType
 }
 
-func unmarshalComponentYaml(typeMap map[string]func() component, inner *interface{}, unmarshal func(interface{}) error) error {
+type componentTypeRegistry struct {
+	Subagent string
+	Kind     string
+	TypeMap  map[string]func() component
+}
+
+func unmarshalComponentYaml(registry *componentTypeRegistry, inner *interface{}, unmarshal func(interface{}) error) error {
 	c := configComponent{}
 	unmarshal(&c) // Get the type; ignore the error
-	f := typeMap[c.Type()]
+	f := registry.TypeMap[c.Type()]
 	if f == nil {
 		return fmt.Errorf("Unknown type %q", c.Type())
 	}
@@ -144,13 +150,16 @@ func init() {
 	registerLoggingReceiverType("windows_event_log", func() component { return &LoggingReceiverWinevtlog{} })
 }
 
-var loggingReceiverTypes = map[string]func() component{}
+var loggingReceiverTypes = &componentTypeRegistry{
+	Subagent: "logging", Kind: "receiver",
+	TypeMap: map[string]func() component{},
+}
 
 func registerLoggingReceiverType(name string, constructor func() component) error {
-	if _, ok := loggingReceiverTypes[name]; ok {
+	if _, ok := loggingReceiverTypes.TypeMap[name]; ok {
 		return fmt.Errorf("Duplicate receiver type: %q", name)
 	}
-	loggingReceiverTypes[name] = constructor
+	loggingReceiverTypes.TypeMap[name] = constructor
 	return nil
 }
 

From 6441afba14eada74e0a4280ea2c6c738b36e5ddf Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Fri, 30 Jul 2021 19:51:45 -0400
Subject: [PATCH 11/41] Embed Type in LoggingReceiver.

---
 confgenerator/config.go | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/confgenerator/config.go b/confgenerator/config.go
index c28f506ee9..186115fc3f 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -117,18 +117,22 @@ type LoggingReceiver interface {
 	component
 }
 
-type LoggingReceiverFiles struct { // Type "files"
+type LoggingReceiverFiles struct {
 	configComponent `yaml:",inline"`
 
 	IncludePaths []string `yaml:"include_paths,omitempty"`
 	ExcludePaths []string `yaml:"exclude_paths,omitempty" config:"optional"`
 }
 
+func (r LoggingReceiverFiles) Type() string {
+	return "files"
+}
+
 func init() {
-	registerLoggingReceiverType("files", func() component { return &LoggingReceiverFiles{} })
+	registerLoggingReceiverType(func() component { return &LoggingReceiverFiles{} })
 }
 
-type LoggingReceiverSyslog struct { // Type "syslog"
+type LoggingReceiverSyslog struct {
 	configComponent `yaml:",inline"`
 
 	TransportProtocol string `yaml:"transport_protocol,omitempty" config:"optional"` // one of "tcp" or "udp"
@@ -136,18 +140,26 @@ type LoggingReceiverSyslog struct { // Type "syslog"
 	ListenPort        uint16 `yaml:"listen_port,omitempty"`
 }
 
+func (r LoggingReceiverSyslog) Type() string {
+	return "syslog"
+}
+
 func init() {
-	registerLoggingReceiverType("syslog", func() component { return &LoggingReceiverSyslog{} })
+	registerLoggingReceiverType(func() component { return &LoggingReceiverSyslog{} })
 }
 
-type LoggingReceiverWinevtlog struct { // Type "windows_event_log"
+type LoggingReceiverWinevtlog struct {
 	configComponent `yaml:",inline"`
 
 	Channels []string `yaml:"channels,omitempty,flow"`
 }
 
+func (r LoggingReceiverWinevtlog) Type() string {
+	return "windows_event_log"
+}
+
 func init() {
-	registerLoggingReceiverType("windows_event_log", func() component { return &LoggingReceiverWinevtlog{} })
+	registerLoggingReceiverType(func() component { return &LoggingReceiverWinevtlog{} })
 }
 
 var loggingReceiverTypes = &componentTypeRegistry{
@@ -155,7 +167,8 @@ var loggingReceiverTypes = &componentTypeRegistry{
 	TypeMap: map[string]func() component{},
 }
 
-func registerLoggingReceiverType(name string, constructor func() component) error {
+func registerLoggingReceiverType(constructor func() component) error {
+	name := constructor().(LoggingReceiver).Type()
 	if _, ok := loggingReceiverTypes.TypeMap[name]; ok {
 		return fmt.Errorf("Duplicate receiver type: %q", name)
 	}

From 3872f378e8b04d45bab91b74f2ec1def451824fa Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Fri, 30 Jul 2021 23:44:28 -0400
Subject: [PATCH 12/41] Move custom parameter validation for logging receivers
 into ValidateParameters.

---
 confgenerator/config.go | 46 ++++++++++++++++++-----------------------
 1 file changed, 20 insertions(+), 26 deletions(-)

diff --git a/confgenerator/config.go b/confgenerator/config.go
index 186115fc3f..a4f362e99f 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -431,7 +431,19 @@ func (r *LoggingReceiverFiles) ValidateParameters(subagent string, kind string,
 }
 
 func (r *LoggingReceiverSyslog) ValidateParameters(subagent string, kind string, id string) error {
-	return validateParameters(*r, subagent, kind, id, r.Type())
+	if err := validateParameters(*r, subagent, kind, id, r.Type()); err != nil {
+		return err
+	}
+	validProtocolValues := []string{"tcp", "udp"}
+	if !sliceContains(validProtocolValues, r.TransportProtocol) {
+		err := fmt.Errorf(`must be one of [%s].`, strings.Join(validProtocolValues, ", "))
+		return fmt.Errorf(`%s has invalid value %q: %s`, parameterErrorPrefix(subagent, kind, id, r.Type(), "transport_protocol"), r.TransportProtocol, err)
+	}
+	if net.ParseIP(r.ListenHost) == nil {
+		err := fmt.Errorf(`must be a valid IP.`)
+		return fmt.Errorf(`%s has invalid value %q: %s`, parameterErrorPrefix(subagent, kind, id, r.Type(), "listen_host"), r.ListenHost, err)
+	}
+	return nil
 }
 
 func (r *LoggingReceiverWinevtlog) ValidateParameters(subagent string, kind string, id string) error {
@@ -510,7 +522,7 @@ func validateParameters(s interface{}, subagent string, kind string, id string,
 	additionalValidation, hasAdditionalValidation := additionalParameterValidation[componentType]
 	parameters := collectYamlFields(s)
 	for _, p := range parameters {
-		if !sliceContains(allParameters, p.Name) {
+		if supportedParameters != nil && !sliceContains(allParameters, p.Name) {
 			if !p.IsZero {
 				// e.g. parameter "transport_protocol" in "files" type logging receiver "receiver_1" is not supported.
 				// Supported parameters: [include_paths, exclude_paths].
@@ -564,15 +576,12 @@ var (
 	}
 
 	supportedParameters = map[string][]string{
-		"files":             []string{"include_paths", "exclude_paths"},
-		"syslog":            []string{"transport_protocol", "listen_host", "listen_port"},
-		"windows_event_log": []string{"channels"},
-		"parse_json":        []string{"field", "time_key", "time_format"},
-		"parse_regex":       []string{"field", "time_key", "time_format", "regex"},
-		"hostmetrics":       []string{"collection_interval"},
-		"iis":               []string{"collection_interval"},
-		"mssql":             []string{"collection_interval"},
-		"exclude_metrics":   []string{"metrics_pattern"},
+		"parse_json":      []string{"field", "time_key", "time_format"},
+		"parse_regex":     []string{"field", "time_key", "time_format", "regex"},
+		"hostmetrics":     []string{"collection_interval"},
+		"iis":             []string{"collection_interval"},
+		"mssql":           []string{"collection_interval"},
+		"exclude_metrics": []string{"metrics_pattern"},
 	}
 
 	collectionIntervalValidation = map[string]func(interface{}) error{
@@ -590,21 +599,6 @@ var (
 	}
 
 	additionalParameterValidation = map[string]map[string]func(interface{}) error{
-		"syslog": map[string]func(interface{}) error{
-			"transport_protocol": func(v interface{}) error {
-				validValues := []string{"tcp", "udp"}
-				if !sliceContains(validValues, v.(string)) {
-					return fmt.Errorf(`must be one of [%s].`, strings.Join(validValues, ", "))
-				}
-				return nil
-			},
-			"listen_host": func(v interface{}) error {
-				if net.ParseIP(v.(string)) == nil {
-					return fmt.Errorf(`must be a valid IP.`)
-				}
-				return nil
-			},
-		},
 		"hostmetrics": collectionIntervalValidation,
 		"iis":         collectionIntervalValidation,
 		"mssql":       collectionIntervalValidation,

From cce48d90410c2fd4ca400e496e38e26109c8adca Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Fri, 30 Jul 2021 21:45:24 -0400
Subject: [PATCH 13/41] Use a custom unmarshaler for the logging processor
 types.

---
 confgenerator/confgenerator.go                | 10 +-
 confgenerator/config.go                       | 97 ++++++++++++++++---
 confgenerator/confmerger.go                   |  6 +-
 .../golden_error                              |  2 +-
 4 files changed, 93 insertions(+), 22 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index b46c866212..8341fb1756 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -519,7 +519,7 @@ func generateFluentBitInputs(receivers map[string]LoggingReceiver, pipelines map
 	return fbTails, fbSyslogs, fbWinEventlogs, nil
 }
 
-func generateFluentBitFilters(processors map[string]*LoggingProcessor, pipelines map[string]*LoggingPipeline) ([]fluentbit.FilterParserGroup, error) {
+func generateFluentBitFilters(processors map[string]LoggingProcessor, pipelines map[string]*LoggingPipeline) ([]fluentbit.FilterParserGroup, error) {
 	// Note: Keep each pipeline's filters in a separate group, because
 	// the order within that group is important, even though the order
 	// of the groups themselves does not matter.
@@ -534,8 +534,8 @@ func generateFluentBitFilters(processors map[string]*LoggingProcessor, pipelines
 				Parser:  processorID,
 				KeyName: "message",
 			}
-			if ok && p.Field != "" {
-				fbFilterParser.KeyName = p.Field
+			if ok && p.GetField() != "" {
+				fbFilterParser.KeyName = p.GetField()
 			}
 			fbFilterParsers = append(fbFilterParsers, &fbFilterParser)
 		}
@@ -582,13 +582,14 @@ func extractExporterPlugins(exporters map[string]*LoggingExporter, pipelines map
 	return fbFilterModifyAddLogNames, fbFilterRewriteTags, fbFilterModifyRemoveLogNames, fbStackdrivers, nil
 }
 
-func extractFluentBitParsers(processors map[string]*LoggingProcessor) ([]*fluentbit.ParserJSON, []*fluentbit.ParserRegex, error) {
+func extractFluentBitParsers(processors map[string]LoggingProcessor) ([]*fluentbit.ParserJSON, []*fluentbit.ParserRegex, error) {
 	fbJSONParsers := []*fluentbit.ParserJSON{}
 	fbRegexParsers := []*fluentbit.ParserRegex{}
 	for _, name := range sortedKeys(processors) {
 		p := processors[name]
 		switch t := p.Type(); t {
 		case "parse_json":
+			p := p.(*LoggingProcessorParseJson)
 			fbJSONParser := fluentbit.ParserJSON{
 				Name:       name,
 				TimeKey:    p.TimeKey,
@@ -596,6 +597,7 @@ func extractFluentBitParsers(processors map[string]*LoggingProcessor) ([]*fluent
 			}
 			fbJSONParsers = append(fbJSONParsers, &fbJSONParser)
 		case "parse_regex":
+			p := p.(*LoggingProcessorParseRegex)
 			fbRegexParser := fluentbit.ParserRegex{
 				Name:       name,
 				Regex:      p.Regex,
diff --git a/confgenerator/config.go b/confgenerator/config.go
index a4f362e99f..4e5eb17f2d 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -106,11 +106,12 @@ func unmarshalComponentYaml(registry *componentTypeRegistry, inner *interface{},
 
 // Ops Agent logging config.
 type loggingReceiverMap map[string]LoggingReceiver
+type loggingProcessorMap map[string]LoggingProcessor
 type Logging struct {
-	Receivers  loggingReceiverMap           `yaml:"receivers,omitempty"`
-	Processors map[string]*LoggingProcessor `yaml:"processors,omitempty"`
-	Exporters  map[string]*LoggingExporter  `yaml:"exporters,omitempty"`
-	Service    *LoggingService              `yaml:"service"`
+	Receivers  loggingReceiverMap          `yaml:"receivers,omitempty"`
+	Processors loggingProcessorMap         `yaml:"processors,omitempty"`
+	Exporters  map[string]*LoggingExporter `yaml:"exporters,omitempty"`
+	Service    *LoggingService             `yaml:"service"`
 }
 
 type LoggingReceiver interface {
@@ -199,22 +200,84 @@ func (m *loggingReceiverMap) UnmarshalYAML(unmarshal func(interface{}) error) er
 	return nil
 }
 
-type LoggingProcessorParseJson struct {
+type LoggingProcessor interface {
+	component
+	GetField() string
+}
+
+type loggingProcessorParseShared struct {
 	Field      string `yaml:"field,omitempty" config:"optional"`       // default to "message"
 	TimeKey    string `yaml:"time_key,omitempty" config:"optional"`    // by default does not parse timestamp
 	TimeFormat string `yaml:"time_format,omitempty" config:"optional"` // must be provided if time_key is present
 }
 
+func (p loggingProcessorParseShared) GetField() string {
+	return p.Field
+}
+
+type LoggingProcessorParseJson struct {
+	configComponent             `yaml:",inline"`
+	loggingProcessorParseShared `yaml:",inline"`
+}
+
+func (r LoggingProcessorParseJson) Type() string {
+	return "parse_json"
+}
+
+func init() {
+	registerLoggingProcessorType(func() component { return &LoggingProcessorParseJson{} })
+}
+
 type LoggingProcessorParseRegex struct {
+	configComponent             `yaml:",inline"`
+	loggingProcessorParseShared `yaml:",inline"`
+
 	Regex string `yaml:"regex,omitempty"`
+}
 
-	LoggingProcessorParseJson `yaml:",inline"` // Type "parse_json"
+func (r LoggingProcessorParseRegex) Type() string {
+	return "parse_regex"
 }
 
-type LoggingProcessor struct {
-	configComponent `yaml:",inline"`
+func init() {
+	registerLoggingProcessorType(func() component { return &LoggingProcessorParseRegex{} })
+}
+
+var loggingProcessorTypes = &componentTypeRegistry{
+	Subagent: "logging", Kind: "processor",
+	TypeMap: map[string]func() component{},
+}
+
+func registerLoggingProcessorType(constructor func() component) error {
+	name := constructor().(LoggingProcessor).Type()
+	if _, ok := loggingProcessorTypes.TypeMap[name]; ok {
+		return fmt.Errorf("Duplicate processor type: %q", name)
+	}
+	loggingProcessorTypes.TypeMap[name] = constructor
+	return nil
+}
+
+// Wrapper type to store the unmarshaled YAML value.
+type loggingProcessorWrapper struct {
+	inner interface{}
+}
 
-	LoggingProcessorParseRegex `yaml:",inline"` // Type "parse_json" or "parse_regex"
+func (l *loggingProcessorWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	return unmarshalComponentYaml(loggingProcessorTypes, &l.inner, unmarshal)
+}
+
+func (m *loggingProcessorMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// Unmarshal into a temporary map to capture types.
+	tm := map[string]loggingProcessorWrapper{}
+	if err := unmarshal(&tm); err != nil {
+		return err
+	}
+	// Unwrap the structs.
+	*m = loggingProcessorMap{}
+	for k, r := range tm {
+		(*m)[k] = r.inner.(LoggingProcessor)
+	}
+	return nil
 }
 
 type LoggingExporter struct {
@@ -325,7 +388,7 @@ func (l *Logging) Validate(platform string) error {
 		if err := validateComponentKeys(l.Receivers, p.ReceiverIDs, subagent, "receiver", id); err != nil {
 			return err
 		}
-		validProcessors := map[string]*LoggingProcessor{}
+		validProcessors := map[string]LoggingProcessor{}
 		for k, v := range l.Processors {
 			validProcessors[k] = v
 		}
@@ -450,7 +513,11 @@ func (r *LoggingReceiverWinevtlog) ValidateParameters(subagent string, kind stri
 	return validateParameters(*r, subagent, kind, id, r.Type())
 }
 
-func (p *LoggingProcessor) ValidateParameters(subagent string, kind string, id string) error {
+func (p *LoggingProcessorParseJson) ValidateParameters(subagent string, kind string, id string) error {
+	return validateParameters(*p, subagent, kind, id, p.Type())
+}
+
+func (p *LoggingProcessorParseRegex) ValidateParameters(subagent string, kind string, id string) error {
 	return validateParameters(*p, subagent, kind, id, p.Type())
 }
 
@@ -576,8 +643,6 @@ var (
 	}
 
 	supportedParameters = map[string][]string{
-		"parse_json":      []string{"field", "time_key", "time_format"},
-		"parse_regex":     []string{"field", "time_key", "time_format", "regex"},
 		"hostmetrics":     []string{"collection_interval"},
 		"iis":             []string{"collection_interval"},
 		"mssql":           []string{"collection_interval"},
@@ -635,7 +700,11 @@ func mapKeys(m interface{}) map[string]bool {
 		for k := range m {
 			keys[k] = true
 		}
-	case map[string]*LoggingProcessor:
+	case map[string]LoggingProcessor:
+		for k := range m {
+			keys[k] = true
+		}
+	case loggingProcessorMap:
 		for k := range m {
 			keys[k] = true
 		}
diff --git a/confgenerator/confmerger.go b/confgenerator/confmerger.go
index c7e947ba35..0db89e3612 100644
--- a/confgenerator/confmerger.go
+++ b/confgenerator/confmerger.go
@@ -34,7 +34,7 @@ var (
 						IncludePaths:    []string{"/var/log/messages", "/var/log/syslog"},
 					},
 				},
-				Processors: map[string]*LoggingProcessor{},
+				Processors: map[string]LoggingProcessor{},
 				Service: &LoggingService{
 					Pipelines: map[string]*LoggingPipeline{
 						"default_pipeline": &LoggingPipeline{
@@ -73,7 +73,7 @@ var (
 						Channels:        []string{"System", "Application", "Security"},
 					},
 				},
-				Processors: map[string]*LoggingProcessor{},
+				Processors: map[string]LoggingProcessor{},
 				Service: &LoggingService{
 					Pipelines: map[string]*LoggingPipeline{
 						"default_pipeline": &LoggingPipeline{
@@ -184,7 +184,7 @@ func mergeConfigs(original, overrides *UnifiedConfig) {
 		}
 
 		// Overrides logging.processors.
-		original.Logging.Processors = map[string]*LoggingProcessor{}
+		original.Logging.Processors = map[string]LoggingProcessor{}
 		for k, v := range overrides.Logging.Processors {
 			original.Logging.Processors[k] = v
 		}
diff --git a/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
index 646f3afc25..56a44eb816 100644
--- a/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
@@ -1 +1 @@
-logging processor "processor_1" with type "unsupported_type" is not supported. Supported logging processor types: [parse_json, parse_regex].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: Unknown type "unsupported_type"
\ No newline at end of file

From dd625ccbc269bfe6453e88bf335251dc62c4d7a4 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Sat, 31 Jul 2021 00:34:07 -0400
Subject: [PATCH 14/41] Use a custom unmarshaler for the logging exporter
 types.

---
 confgenerator/confgenerator.go |  6 +--
 confgenerator/config.go        | 70 +++++++++++++++++++++++++++++++---
 2 files changed, 67 insertions(+), 9 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 8341fb1756..97d79add09 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -228,8 +228,8 @@ func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir strin
 	if logging != nil && logging.Service != nil {
 		// Override any user-specified exporters
 		// TODO: Refactor remaining code to not consult these fields
-		logging.Exporters = map[string]*LoggingExporter{
-			"google": &LoggingExporter{
+		logging.Exporters = map[string]LoggingExporter{
+			"google": &LoggingExporterGoogleCloudLogging{
 				configComponent: configComponent{ComponentType: "google_cloud_logging"},
 			},
 		}
@@ -546,7 +546,7 @@ func generateFluentBitFilters(processors map[string]LoggingProcessor, pipelines
 	return groups, nil
 }
 
-func extractExporterPlugins(exporters map[string]*LoggingExporter, pipelines map[string]*LoggingPipeline, hostInfo *host.InfoStat) (
+func extractExporterPlugins(exporters map[string]LoggingExporter, pipelines map[string]*LoggingPipeline, hostInfo *host.InfoStat) (
 	[]*fluentbit.FilterModifyAddLogName, []*fluentbit.FilterRewriteTag, []*fluentbit.FilterModifyRemoveLogName, []*fluentbit.Stackdriver, error) {
 	fbFilterModifyAddLogNames := []*fluentbit.FilterModifyAddLogName{}
 	fbFilterRewriteTags := []*fluentbit.FilterRewriteTag{}
diff --git a/confgenerator/config.go b/confgenerator/config.go
index 4e5eb17f2d..ce0d95543a 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -107,11 +107,12 @@ func unmarshalComponentYaml(registry *componentTypeRegistry, inner *interface{},
 // Ops Agent logging config.
 type loggingReceiverMap map[string]LoggingReceiver
 type loggingProcessorMap map[string]LoggingProcessor
+type loggingExporterMap map[string]LoggingExporter
 type Logging struct {
-	Receivers  loggingReceiverMap          `yaml:"receivers,omitempty"`
-	Processors loggingProcessorMap         `yaml:"processors,omitempty"`
-	Exporters  map[string]*LoggingExporter `yaml:"exporters,omitempty"`
-	Service    *LoggingService             `yaml:"service"`
+	Receivers  loggingReceiverMap  `yaml:"receivers,omitempty"`
+	Processors loggingProcessorMap `yaml:"processors,omitempty"`
+	Exporters  loggingExporterMap  `yaml:"exporters,omitempty"`
+	Service    *LoggingService     `yaml:"service"`
 }
 
 type LoggingReceiver interface {
@@ -280,10 +281,59 @@ func (m *loggingProcessorMap) UnmarshalYAML(unmarshal func(interface{}) error) e
 	return nil
 }
 
-type LoggingExporter struct {
+type LoggingExporter interface {
+	component
+}
+
+type LoggingExporterGoogleCloudLogging struct {
 	configComponent `yaml:",inline"`
 }
 
+func (r LoggingExporterGoogleCloudLogging) Type() string {
+	return "google_cloud_logging"
+}
+
+func init() {
+	registerLoggingExporterType(func() component { return &LoggingExporterGoogleCloudLogging{} })
+}
+
+var loggingExporterTypes = &componentTypeRegistry{
+	Subagent: "logging", Kind: "exporter",
+	TypeMap: map[string]func() component{},
+}
+
+func registerLoggingExporterType(constructor func() component) error {
+	name := constructor().(LoggingExporter).Type()
+	if _, ok := loggingExporterTypes.TypeMap[name]; ok {
+		return fmt.Errorf("Duplicate exporter type: %q", name)
+	}
+	loggingExporterTypes.TypeMap[name] = constructor
+	return nil
+}
+
+// Wrapper type to store the unmarshaled YAML value.
+type loggingExporterWrapper struct {
+	inner interface{}
+}
+
+func (l *loggingExporterWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	return unmarshalComponentYaml(loggingExporterTypes, &l.inner, unmarshal)
+}
+
+func (m *loggingExporterMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// Unmarshal into a temporary map to capture types.
+	tm := map[string]loggingExporterWrapper{}
+	if err := unmarshal(&tm); err != nil {
+		return err
+	}
+	// Unwrap the structs.
+	*m = loggingExporterMap{}
+	for k, r := range tm {
+		(*m)[k] = r.inner.(LoggingExporter)
+	}
+	return nil
+}
+
 type LoggingService struct {
 	Pipelines map[string]*LoggingPipeline
 }
@@ -521,6 +571,10 @@ func (p *LoggingProcessorParseRegex) ValidateParameters(subagent string, kind st
 	return validateParameters(*p, subagent, kind, id, p.Type())
 }
 
+func (r *LoggingExporterGoogleCloudLogging) ValidateParameters(subagent string, kind string, id string) error {
+	panic("Should never be called")
+}
+
 func (r *MetricsReceiver) ValidateParameters(subagent string, kind string, id string) error {
 	return validateParameters(*r, subagent, kind, id, r.Type())
 }
@@ -708,7 +762,11 @@ func mapKeys(m interface{}) map[string]bool {
 		for k := range m {
 			keys[k] = true
 		}
-	case map[string]*LoggingExporter:
+	case map[string]LoggingExporter:
+		for k := range m {
+			keys[k] = true
+		}
+	case loggingExporterMap:
 		for k := range m {
 			keys[k] = true
 		}

From 261dbb5e728ef94c3df1f8c20d021a58e9f15efb Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Sat, 31 Jul 2021 14:08:41 -0400
Subject: [PATCH 15/41] Use a custom unmarshaler for the metrics receiver
 types.

---
 confgenerator/confgenerator.go                |   7 +-
 confgenerator/config.go                       | 161 +++++++++++++++---
 confgenerator/confmerger.go                   |  28 +--
 .../golden_error                              |   2 +-
 .../golden_error                              |   2 +-
 5 files changed, 155 insertions(+), 45 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 97d79add09..cc826aca2a 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -301,21 +301,24 @@ type excludemetricsProcessorFactory struct {
 	MetricsPattern []string
 }
 
-func extractOtelReceiverFactories(receivers map[string]*MetricsReceiver) (map[string]*hostmetricsReceiverFactory, map[string]*mssqlReceiverFactory, map[string]*iisReceiverFactory, error) {
+func extractOtelReceiverFactories(receivers map[string]MetricsReceiver) (map[string]*hostmetricsReceiverFactory, map[string]*mssqlReceiverFactory, map[string]*iisReceiverFactory, error) {
 	hostmetricsReceiverFactories := map[string]*hostmetricsReceiverFactory{}
 	mssqlReceiverFactories := map[string]*mssqlReceiverFactory{}
 	iisReceiverFactories := map[string]*iisReceiverFactory{}
 	for n, r := range receivers {
 		switch r.Type() {
 		case "hostmetrics":
+			r := r.(*MetricsReceiverHostmetrics)
 			hostmetricsReceiverFactories[n] = &hostmetricsReceiverFactory{
 				CollectionInterval: r.CollectionInterval,
 			}
 		case "mssql":
+			r := r.(*MetricsReceiverMssql)
 			mssqlReceiverFactories[n] = &mssqlReceiverFactory{
 				CollectionInterval: r.CollectionInterval,
 			}
 		case "iis":
+			r := r.(*MetricsReceiverIis)
 			iisReceiverFactories[n] = &iisReceiverFactory{
 				CollectionInterval: r.CollectionInterval,
 			}
@@ -366,7 +369,7 @@ func extractReceiverFactories(receivers map[string]LoggingReceiver) (map[string]
 	return fileReceiverFactories, syslogReceiverFactories, wineventlogReceiverFactories, nil
 }
 
-func generateOtelReceivers(receivers map[string]*MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]*otel.HostMetrics, []*otel.MSSQL, []*otel.IIS, map[string]string, error) {
+func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]*otel.HostMetrics, []*otel.MSSQL, []*otel.IIS, map[string]string, error) {
 	hostMetricsList := []*otel.HostMetrics{}
 	mssqlList := []*otel.MSSQL{}
 	iisList := []*otel.IIS{}
diff --git a/confgenerator/config.go b/confgenerator/config.go
index ce0d95543a..2d3dd3e3de 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -345,19 +345,106 @@ type LoggingPipeline struct {
 }
 
 // Ops Agent metrics config.
+type metricsReceiverMap map[string]MetricsReceiver
 type Metrics struct {
-	Receivers  map[string]*MetricsReceiver  `yaml:"receivers"`
+	Receivers  metricsReceiverMap           `yaml:"receivers"`
 	Processors map[string]*MetricsProcessor `yaml:"processors"`
 	Exporters  map[string]*MetricsExporter  `yaml:"exporters,omitempty"`
 	Service    *MetricsService              `yaml:"service"`
 }
 
-type MetricsReceiver struct {
-	configComponent `yaml:",inline"`
+type MetricsReceiver interface {
+	component
+	GetCollectionInterval() string
+}
 
+type metricsReceiverShared struct {
 	CollectionInterval string `yaml:"collection_interval"` // time.Duration format
 }
 
+func (r *metricsReceiverShared) GetCollectionInterval() string {
+	return r.CollectionInterval
+}
+
+type MetricsReceiverHostmetrics struct {
+	configComponent `yaml:",inline"`
+
+	metricsReceiverShared `yaml:",inline"`
+}
+
+func (r MetricsReceiverHostmetrics) Type() string {
+	return "hostmetrics"
+}
+
+func init() {
+	registerMetricsReceiverType(func() component { return &MetricsReceiverHostmetrics{} })
+}
+
+type MetricsReceiverIis struct {
+	configComponent `yaml:",inline"`
+
+	metricsReceiverShared `yaml:",inline"`
+}
+
+func (r MetricsReceiverIis) Type() string {
+	return "iis"
+}
+
+func init() {
+	registerMetricsReceiverType(func() component { return &MetricsReceiverIis{} })
+}
+
+type MetricsReceiverMssql struct {
+	configComponent `yaml:",inline"`
+
+	metricsReceiverShared `yaml:",inline"`
+}
+
+func (r MetricsReceiverMssql) Type() string {
+	return "mssql"
+}
+
+func init() {
+	registerMetricsReceiverType(func() component { return &MetricsReceiverMssql{} })
+}
+
+var metricsReceiverTypes = &componentTypeRegistry{
+	Subagent: "metrics", Kind: "receiver",
+	TypeMap: map[string]func() component{},
+}
+
+func registerMetricsReceiverType(constructor func() component) error {
+	name := constructor().(MetricsReceiver).Type()
+	if _, ok := metricsReceiverTypes.TypeMap[name]; ok {
+		return fmt.Errorf("Duplicate receiver type: %q", name)
+	}
+	metricsReceiverTypes.TypeMap[name] = constructor
+	return nil
+}
+
+// Wrapper type to store the unmarshaled YAML value.
+type metricsReceiverWrapper struct {
+	inner interface{}
+}
+
+func (m *metricsReceiverWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	return unmarshalComponentYaml(metricsReceiverTypes, &m.inner, unmarshal)
+}
+
+func (m *metricsReceiverMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// Unmarshal into a temporary map to capture types.
+	tm := map[string]metricsReceiverWrapper{}
+	if err := unmarshal(&tm); err != nil {
+		return err
+	}
+	// Unwrap the structs.
+	*m = metricsReceiverMap{}
+	for k, r := range tm {
+		(*m)[k] = r.inner.(MetricsReceiver)
+	}
+	return nil
+}
+
 type MetricsProcessorExcludeMetrics struct {
 	MetricsPattern []string `yaml:"metrics_pattern,flow"`
 }
@@ -575,8 +662,40 @@ func (r *LoggingExporterGoogleCloudLogging) ValidateParameters(subagent string,
 	panic("Should never be called")
 }
 
-func (r *MetricsReceiver) ValidateParameters(subagent string, kind string, id string) error {
-	return validateParameters(*r, subagent, kind, id, r.Type())
+func validateSharedParameters(r MetricsReceiver, subagent string, kind string, id string) error {
+	if err := validateParameters(r, subagent, kind, id, r.Type()); err != nil {
+		return err
+	}
+
+	validateCollectionInterval := func(collectionInterval string) error {
+		t, err := time.ParseDuration(collectionInterval)
+		if err != nil {
+			return fmt.Errorf(`not an interval (e.g. "60s"). Detailed error: %s`, err)
+		}
+		interval := t.Seconds()
+		if interval < 10 {
+			return fmt.Errorf(`below the minimum threshold of "10s".`)
+		}
+		return nil
+	}
+
+	collectionInterval := r.GetCollectionInterval()
+	if err := validateCollectionInterval(collectionInterval); err != nil {
+		return fmt.Errorf(`%s has invalid value %q: %s`, parameterErrorPrefix(subagent, kind, id, r.Type(), "collection_interval"), collectionInterval, err)
+	}
+	return nil
+}
+
+func (r *MetricsReceiverHostmetrics) ValidateParameters(subagent string, kind string, id string) error {
+	return validateSharedParameters(r, subagent, kind, id)
+}
+
+func (r *MetricsReceiverIis) ValidateParameters(subagent string, kind string, id string) error {
+	return validateSharedParameters(r, subagent, kind, id)
+}
+
+func (r *MetricsReceiverMssql) ValidateParameters(subagent string, kind string, id string) error {
+	return validateSharedParameters(r, subagent, kind, id)
 }
 
 func (p *MetricsProcessor) ValidateParameters(subagent string, kind string, id string) error {
@@ -632,7 +751,11 @@ func collectYamlFields(s interface{}) []yamlField {
 		}
 		return parameters
 	}
-	return recurse(reflect.ValueOf(s))
+	v := reflect.ValueOf(s)
+	if v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+	return recurse(v)
 }
 
 func validateParameters(s interface{}, subagent string, kind string, id string, componentType string) error {
@@ -697,30 +820,10 @@ var (
 	}
 
 	supportedParameters = map[string][]string{
-		"hostmetrics":     []string{"collection_interval"},
-		"iis":             []string{"collection_interval"},
-		"mssql":           []string{"collection_interval"},
 		"exclude_metrics": []string{"metrics_pattern"},
 	}
 
-	collectionIntervalValidation = map[string]func(interface{}) error{
-		"collection_interval": func(v interface{}) error {
-			t, err := time.ParseDuration(v.(string))
-			if err != nil {
-				return fmt.Errorf(`not an interval (e.g. "60s"). Detailed error: %s`, err)
-			}
-			interval := t.Seconds()
-			if interval < 10 {
-				return fmt.Errorf(`below the minimum threshold of "10s".`)
-			}
-			return nil
-		},
-	}
-
 	additionalParameterValidation = map[string]map[string]func(interface{}) error{
-		"hostmetrics": collectionIntervalValidation,
-		"iis":         collectionIntervalValidation,
-		"mssql":       collectionIntervalValidation,
 		"exclude_metrics": map[string]func(interface{}) error{
 			"metrics_pattern": func(v interface{}) error {
 				var errors []string
@@ -774,7 +877,11 @@ func mapKeys(m interface{}) map[string]bool {
 		for k := range m {
 			keys[k] = true
 		}
-	case map[string]*MetricsReceiver:
+	case map[string]MetricsReceiver:
+		for k := range m {
+			keys[k] = true
+		}
+	case metricsReceiverMap:
 		for k := range m {
 			keys[k] = true
 		}
diff --git a/confgenerator/confmerger.go b/confgenerator/confmerger.go
index 0db89e3612..ae812ab7c3 100644
--- a/confgenerator/confmerger.go
+++ b/confgenerator/confmerger.go
@@ -44,10 +44,10 @@ var (
 				},
 			},
 			Metrics: &Metrics{
-				Receivers: map[string]*MetricsReceiver{
-					"hostmetrics": &MetricsReceiver{
-						configComponent:    configComponent{ComponentType: "hostmetrics"},
-						CollectionInterval: "60s",
+				Receivers: map[string]MetricsReceiver{
+					"hostmetrics": &MetricsReceiverHostmetrics{
+						configComponent:       configComponent{ComponentType: "hostmetrics"},
+						metricsReceiverShared: metricsReceiverShared{CollectionInterval: "60s"},
 					},
 				},
 				Processors: map[string]*MetricsProcessor{
@@ -83,18 +83,18 @@ var (
 				},
 			},
 			Metrics: &Metrics{
-				Receivers: map[string]*MetricsReceiver{
-					"hostmetrics": &MetricsReceiver{
-						configComponent:    configComponent{ComponentType: "hostmetrics"},
-						CollectionInterval: "60s",
+				Receivers: map[string]MetricsReceiver{
+					"hostmetrics": &MetricsReceiverHostmetrics{
+						configComponent:       configComponent{ComponentType: "hostmetrics"},
+						metricsReceiverShared: metricsReceiverShared{CollectionInterval: "60s"},
 					},
-					"iis": &MetricsReceiver{
-						configComponent:    configComponent{ComponentType: "iis"},
-						CollectionInterval: "60s",
+					"iis": &MetricsReceiverIis{
+						configComponent:       configComponent{ComponentType: "iis"},
+						metricsReceiverShared: metricsReceiverShared{CollectionInterval: "60s"},
 					},
-					"mssql": &MetricsReceiver{
-						configComponent:    configComponent{ComponentType: "mssql"},
-						CollectionInterval: "60s",
+					"mssql": &MetricsReceiverMssql{
+						configComponent:       configComponent{ComponentType: "mssql"},
+						metricsReceiverShared: metricsReceiverShared{CollectionInterval: "60s"},
 					},
 				},
 				Processors: map[string]*MetricsProcessor{
diff --git a/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
index e4523719b3..56a44eb816 100644
--- a/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-metrics receiver "receiver_1" with type "unsupported_type" is not supported. Supported metrics receiver types: [hostmetrics].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: Unknown type "unsupported_type"
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
index 8915ae7a8c..56a44eb816 100644
--- a/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-metrics receiver "receiver_1" with type "unsupported_type" is not supported. Supported metrics receiver types: [hostmetrics, iis, mssql].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: Unknown type "unsupported_type"
\ No newline at end of file

From cfbbafebd9826a673cfe3de544f5d1e1729e46c9 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Sat, 31 Jul 2021 14:31:25 -0400
Subject: [PATCH 16/41] Use a custom unmarshaler for the metrics processor
 types.

---
 confgenerator/confgenerator.go                |   5 +-
 confgenerator/config.go                       | 136 ++++++++++--------
 confgenerator/confmerger.go                   |   8 +-
 .../golden_error                              |   2 +-
 4 files changed, 86 insertions(+), 65 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index cc826aca2a..6c2fda834e 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -327,11 +327,12 @@ func extractOtelReceiverFactories(receivers map[string]MetricsReceiver) (map[str
 	return hostmetricsReceiverFactories, mssqlReceiverFactories, iisReceiverFactories, nil
 }
 
-func extractOtelProcessorFactories(processors map[string]*MetricsProcessor) (map[string]*excludemetricsProcessorFactory, error) {
+func extractOtelProcessorFactories(processors map[string]MetricsProcessor) (map[string]*excludemetricsProcessorFactory, error) {
 	excludemetricsProcessorFactories := map[string]*excludemetricsProcessorFactory{}
 	for n, p := range processors {
 		switch p.Type() {
 		case "exclude_metrics":
+			p := p.(*MetricsProcessorExcludeMetrics)
 			excludemetricsProcessorFactories[n] = &excludemetricsProcessorFactory{
 				MetricsPattern: p.MetricsPattern,
 			}
@@ -437,7 +438,7 @@ func generateOtelExporters(exporters map[string]*MetricsExporter, pipelines map[
 	return stackdriverList, exportNameMap, nil
 }
 
-func generateOtelProcessors(processors map[string]*MetricsProcessor, pipelines map[string]*MetricsPipeline) ([]*otel.ExcludeMetrics, map[string]string, error) {
+func generateOtelProcessors(processors map[string]MetricsProcessor, pipelines map[string]*MetricsPipeline) ([]*otel.ExcludeMetrics, map[string]string, error) {
 	excludeMetricsList := []*otel.ExcludeMetrics{}
 	processorNameMap := make(map[string]string)
 	excludemetricsProcessorFactories, err := extractOtelProcessorFactories(processors)
diff --git a/confgenerator/config.go b/confgenerator/config.go
index 2d3dd3e3de..3ccf988be5 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -346,11 +346,12 @@ type LoggingPipeline struct {
 
 // Ops Agent metrics config.
 type metricsReceiverMap map[string]MetricsReceiver
+type metricsProcessorMap map[string]MetricsProcessor
 type Metrics struct {
-	Receivers  metricsReceiverMap           `yaml:"receivers"`
-	Processors map[string]*MetricsProcessor `yaml:"processors"`
-	Exporters  map[string]*MetricsExporter  `yaml:"exporters,omitempty"`
-	Service    *MetricsService              `yaml:"service"`
+	Receivers  metricsReceiverMap          `yaml:"receivers"`
+	Processors metricsProcessorMap         `yaml:"processors"`
+	Exporters  map[string]*MetricsExporter `yaml:"exporters,omitempty"`
+	Service    *MetricsService             `yaml:"service"`
 }
 
 type MetricsReceiver interface {
@@ -445,14 +446,59 @@ func (m *metricsReceiverMap) UnmarshalYAML(unmarshal func(interface{}) error) er
 	return nil
 }
 
+type MetricsProcessor interface {
+	component
+}
+
 type MetricsProcessorExcludeMetrics struct {
+	configComponent `yaml:",inline"`
+
 	MetricsPattern []string `yaml:"metrics_pattern,flow"`
 }
 
-type MetricsProcessor struct {
-	configComponent `yaml:",inline"`
+func (r MetricsProcessorExcludeMetrics) Type() string {
+	return "exclude_metrics"
+}
 
-	MetricsProcessorExcludeMetrics `yaml:",inline"` // Type "exclude_metrics"
+func init() {
+	registerMetricsProcessorType(func() component { return &MetricsProcessorExcludeMetrics{} })
+}
+
+var metricsProcessorTypes = &componentTypeRegistry{
+	Subagent: "metrics", Kind: "processor",
+	TypeMap: map[string]func() component{},
+}
+
+func registerMetricsProcessorType(constructor func() component) error {
+	name := constructor().(MetricsProcessor).Type()
+	if _, ok := metricsProcessorTypes.TypeMap[name]; ok {
+		return fmt.Errorf("Duplicate processor type: %q", name)
+	}
+	metricsProcessorTypes.TypeMap[name] = constructor
+	return nil
+}
+
+// Wrapper type to store the unmarshaled YAML value.
+type metricsProcessorWrapper struct {
+	inner interface{}
+}
+
+func (m *metricsProcessorWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	return unmarshalComponentYaml(metricsProcessorTypes, &m.inner, unmarshal)
+}
+
+func (m *metricsProcessorMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// Unmarshal into a temporary map to capture types.
+	tm := map[string]metricsProcessorWrapper{}
+	if err := unmarshal(&tm); err != nil {
+		return err
+	}
+	// Unwrap the structs.
+	*m = metricsProcessorMap{}
+	for k, r := range tm {
+		(*m)[k] = r.inner.(MetricsProcessor)
+	}
+	return nil
 }
 
 type MetricsExporter struct {
@@ -698,8 +744,25 @@ func (r *MetricsReceiverMssql) ValidateParameters(subagent string, kind string,
 	return validateSharedParameters(r, subagent, kind, id)
 }
 
-func (p *MetricsProcessor) ValidateParameters(subagent string, kind string, id string) error {
-	return validateParameters(*p, subagent, kind, id, p.Type())
+func (p *MetricsProcessorExcludeMetrics) ValidateParameters(subagent string, kind string, id string) error {
+	if err := validateParameters(*p, subagent, kind, id, p.Type()); err != nil {
+		return err
+	}
+	var errors []string
+	for _, prefix := range p.MetricsPattern {
+		if !strings.HasSuffix(prefix, "/*") {
+			errors = append(errors, fmt.Sprintf(`%q must end with "/*"`, prefix))
+		}
+		// TODO: Relax the prefix check when we support metrics with other prefixes.
+		if !strings.HasPrefix(prefix, "agent.googleapis.com/") {
+			errors = append(errors, fmt.Sprintf(`%q must start with "agent.googleapis.com/"`, prefix))
+		}
+	}
+	if len(errors) > 0 {
+		err := fmt.Errorf(strings.Join(errors, " | "))
+		return fmt.Errorf(`%s has invalid value %q: %s`, parameterErrorPrefix(subagent, kind, id, p.Type(), "metrics_pattern"), p.MetricsPattern, err)
+	}
+	return nil
 }
 
 type yamlField struct {
@@ -759,35 +822,13 @@ func collectYamlFields(s interface{}) []yamlField {
 }
 
 func validateParameters(s interface{}, subagent string, kind string, id string, componentType string) error {
-	supportedParameters := supportedParameters[componentType]
-	// Include type when checking.
-	allParameters := []string{"type"}
-	allParameters = append(allParameters, supportedParameters...)
-	additionalValidation, hasAdditionalValidation := additionalParameterValidation[componentType]
+	// Check for required parameters.
 	parameters := collectYamlFields(s)
 	for _, p := range parameters {
-		if supportedParameters != nil && !sliceContains(allParameters, p.Name) {
-			if !p.IsZero {
-				// e.g. parameter "transport_protocol" in "files" type logging receiver "receiver_1" is not supported.
-				// Supported parameters: [include_paths, exclude_paths].
-				return fmt.Errorf(`%s is not supported. Supported parameters: [%s].`,
-					parameterErrorPrefix(subagent, kind, id, componentType, p.Name), strings.Join(supportedParameters, ", "))
-			}
-			continue
-		}
 		if p.IsZero && p.Required {
 			// e.g. parameter "include_paths" in "files" type logging receiver "receiver_1" is required.
 			return fmt.Errorf(`%s is required.`, parameterErrorPrefix(subagent, kind, id, componentType, p.Name))
 		}
-		if hasAdditionalValidation {
-			if f, ok := additionalValidation[p.Name]; ok {
-				if err := f(p.Value); err != nil {
-					// e.g. parameter "collection_interval" in "hostmetrics" type metrics receiver "receiver_1"
-					// has invalid value "1s": below the minimum threshold of "10s".
-					return fmt.Errorf(`%s has invalid value %q: %s`, parameterErrorPrefix(subagent, kind, id, componentType, p.Name), p.Value, err)
-				}
-			}
-		}
 	}
 	return nil
 }
@@ -818,31 +859,6 @@ var (
 		"iis":                     1,
 		"mssql":                   1,
 	}
-
-	supportedParameters = map[string][]string{
-		"exclude_metrics": []string{"metrics_pattern"},
-	}
-
-	additionalParameterValidation = map[string]map[string]func(interface{}) error{
-		"exclude_metrics": map[string]func(interface{}) error{
-			"metrics_pattern": func(v interface{}) error {
-				var errors []string
-				for _, prefix := range v.([]string) {
-					if !strings.HasSuffix(prefix, "/*") {
-						errors = append(errors, fmt.Sprintf(`%q must end with "/*"`, prefix))
-					}
-					// TODO: Relax the prefix check when we support metrics with other prefixes.
-					if !strings.HasPrefix(prefix, "agent.googleapis.com/") {
-						errors = append(errors, fmt.Sprintf(`%q must start with "agent.googleapis.com/"`, prefix))
-					}
-				}
-				if len(errors) > 0 {
-					return fmt.Errorf(strings.Join(errors, " | "))
-				}
-				return nil
-			},
-		},
-	}
 )
 
 // mapKeys returns keys from a map[string]Any as a map[string]bool.
@@ -885,7 +901,11 @@ func mapKeys(m interface{}) map[string]bool {
 		for k := range m {
 			keys[k] = true
 		}
-	case map[string]*MetricsProcessor:
+	case map[string]MetricsProcessor:
+		for k := range m {
+			keys[k] = true
+		}
+	case metricsProcessorMap:
 		for k := range m {
 			keys[k] = true
 		}
diff --git a/confgenerator/confmerger.go b/confgenerator/confmerger.go
index ae812ab7c3..f8a13c2dbc 100644
--- a/confgenerator/confmerger.go
+++ b/confgenerator/confmerger.go
@@ -50,8 +50,8 @@ var (
 						metricsReceiverShared: metricsReceiverShared{CollectionInterval: "60s"},
 					},
 				},
-				Processors: map[string]*MetricsProcessor{
-					"metrics_filter": &MetricsProcessor{
+				Processors: map[string]MetricsProcessor{
+					"metrics_filter": &MetricsProcessorExcludeMetrics{
 						configComponent: configComponent{ComponentType: "exclude_metrics"},
 					},
 				},
@@ -97,8 +97,8 @@ var (
 						metricsReceiverShared: metricsReceiverShared{CollectionInterval: "60s"},
 					},
 				},
-				Processors: map[string]*MetricsProcessor{
-					"metrics_filter": &MetricsProcessor{
+				Processors: map[string]MetricsProcessor{
+					"metrics_filter": &MetricsProcessorExcludeMetrics{
 						configComponent: configComponent{ComponentType: "exclude_metrics"},
 					},
 				},
diff --git a/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
index 72da139814..56a44eb816 100644
--- a/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
@@ -1 +1 @@
-metrics processor "processor_1" with type "unsupported_type" is not supported. Supported metrics processor types: [exclude_metrics].
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: Unknown type "unsupported_type"
\ No newline at end of file

From c123eaa01b1d0d54805a45d7e3c593f309e9196b Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Sat, 31 Jul 2021 16:17:10 -0400
Subject: [PATCH 17/41] Use a custom unmarshaler for the metrics exporter
 types.

---
 confgenerator/confgenerator.go |  6 +--
 confgenerator/config.go        | 70 +++++++++++++++++++++++++++++++---
 confgenerator/confmerger.go    |  2 +-
 3 files changed, 68 insertions(+), 10 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 6c2fda834e..25d9794064 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -52,8 +52,8 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 	if metrics != nil {
 		// Override any user-specified exporters
 		// TODO: Refactor remaining code to not consult these fields
-		metrics.Exporters = map[string]*MetricsExporter{
-			"google": &MetricsExporter{
+		metrics.Exporters = map[string]MetricsExporter{
+			"google": &MetricsExporterGoogleCloudMonitoring{
 				configComponent: configComponent{ComponentType: "google_cloud_monitoring"},
 			},
 		}
@@ -412,7 +412,7 @@ func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[s
 	return hostMetricsList, mssqlList, iisList, receiverNameMap, nil
 }
 
-func generateOtelExporters(exporters map[string]*MetricsExporter, pipelines map[string]*MetricsPipeline) ([]*otel.Stackdriver, map[string]string, error) {
+func generateOtelExporters(exporters map[string]MetricsExporter, pipelines map[string]*MetricsPipeline) ([]*otel.Stackdriver, map[string]string, error) {
 	stackdriverList := []*otel.Stackdriver{}
 	exportNameMap := make(map[string]string)
 	for _, pID := range sortedKeys(pipelines) {
diff --git a/confgenerator/config.go b/confgenerator/config.go
index 3ccf988be5..b85d0e8126 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -347,11 +347,12 @@ type LoggingPipeline struct {
 // Ops Agent metrics config.
 type metricsReceiverMap map[string]MetricsReceiver
 type metricsProcessorMap map[string]MetricsProcessor
+type metricsExporterMap map[string]MetricsExporter
 type Metrics struct {
-	Receivers  metricsReceiverMap          `yaml:"receivers"`
-	Processors metricsProcessorMap         `yaml:"processors"`
-	Exporters  map[string]*MetricsExporter `yaml:"exporters,omitempty"`
-	Service    *MetricsService             `yaml:"service"`
+	Receivers  metricsReceiverMap  `yaml:"receivers"`
+	Processors metricsProcessorMap `yaml:"processors"`
+	Exporters  metricsExporterMap  `yaml:"exporters,omitempty"`
+	Service    *MetricsService     `yaml:"service"`
 }
 
 type MetricsReceiver interface {
@@ -501,10 +502,59 @@ func (m *metricsProcessorMap) UnmarshalYAML(unmarshal func(interface{}) error) e
 	return nil
 }
 
-type MetricsExporter struct {
+type MetricsExporter interface {
+	component
+}
+
+type MetricsExporterGoogleCloudMonitoring struct {
 	configComponent `yaml:",inline"`
 }
 
+func (r MetricsExporterGoogleCloudMonitoring) Type() string {
+	return "google_cloud_monitoring"
+}
+
+func init() {
+	registerMetricsExporterType(func() component { return &MetricsExporterGoogleCloudMonitoring{} })
+}
+
+var metricsExporterTypes = &componentTypeRegistry{
+	Subagent: "metrics", Kind: "exporter",
+	TypeMap: map[string]func() component{},
+}
+
+func registerMetricsExporterType(constructor func() component) error {
+	name := constructor().(MetricsExporter).Type()
+	if _, ok := metricsExporterTypes.TypeMap[name]; ok {
+		return fmt.Errorf("Duplicate exporter type: %q", name)
+	}
+	metricsExporterTypes.TypeMap[name] = constructor
+	return nil
+}
+
+// Wrapper type to store the unmarshaled YAML value.
+type metricsExporterWrapper struct {
+	inner interface{}
+}
+
+func (l *metricsExporterWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	return unmarshalComponentYaml(metricsExporterTypes, &l.inner, unmarshal)
+}
+
+func (m *metricsExporterMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// Unmarshal into a temporary map to capture types.
+	tm := map[string]metricsExporterWrapper{}
+	if err := unmarshal(&tm); err != nil {
+		return err
+	}
+	// Unwrap the structs.
+	*m = metricsExporterMap{}
+	for k, r := range tm {
+		(*m)[k] = r.inner.(MetricsExporter)
+	}
+	return nil
+}
+
 type MetricsService struct {
 	Pipelines map[string]*MetricsPipeline `yaml:"pipelines"`
 }
@@ -765,6 +815,10 @@ func (p *MetricsProcessorExcludeMetrics) ValidateParameters(subagent string, kin
 	return nil
 }
 
+func (r *MetricsExporterGoogleCloudMonitoring) ValidateParameters(subagent string, kind string, id string) error {
+	panic("Should never be called")
+}
+
 type yamlField struct {
 	Name     string
 	Required bool
@@ -909,7 +963,11 @@ func mapKeys(m interface{}) map[string]bool {
 		for k := range m {
 			keys[k] = true
 		}
-	case map[string]*MetricsExporter:
+	case map[string]MetricsExporter:
+		for k := range m {
+			keys[k] = true
+		}
+	case metricsExporterMap:
 		for k := range m {
 			keys[k] = true
 		}
diff --git a/confgenerator/confmerger.go b/confgenerator/confmerger.go
index f8a13c2dbc..484cec7a51 100644
--- a/confgenerator/confmerger.go
+++ b/confgenerator/confmerger.go
@@ -223,7 +223,7 @@ func mergeConfigs(original, overrides *UnifiedConfig) {
 		}
 
 		// Overrides metrics.exporters.
-		original.Metrics.Exporters = map[string]*MetricsExporter{}
+		original.Metrics.Exporters = map[string]MetricsExporter{}
 		for k, v := range overrides.Metrics.Exporters {
 			original.Metrics.Exporters[k] = v
 		}

From 3440618eec494a2b3cce92a10391e870148b4c7f Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Sat, 31 Jul 2021 16:32:25 -0400
Subject: [PATCH 18/41] Factor out common type registration code.

---
 confgenerator/config.go                       | 103 +++++-------------
 .../golden_error                              |   2 +-
 .../golden_error                              |   2 +-
 .../golden_error                              |   2 +-
 .../golden_error                              |   2 +-
 .../golden_error                              |   2 +-
 6 files changed, 34 insertions(+), 79 deletions(-)

diff --git a/confgenerator/config.go b/confgenerator/config.go
index b85d0e8126..99f3e149d7 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -93,12 +93,21 @@ type componentTypeRegistry struct {
 	TypeMap  map[string]func() component
 }
 
-func unmarshalComponentYaml(registry *componentTypeRegistry, inner *interface{}, unmarshal func(interface{}) error) error {
+func (r *componentTypeRegistry) registerType(constructor func() component) error {
+	name := constructor().(component).Type()
+	if _, ok := r.TypeMap[name]; ok {
+		return fmt.Errorf("Duplicate %s %s type: %q", r.Subagent, r.Kind, name)
+	}
+	r.TypeMap[name] = constructor
+	return nil
+}
+
+func (r *componentTypeRegistry) unmarshalComponentYaml(inner *interface{}, unmarshal func(interface{}) error) error {
 	c := configComponent{}
 	unmarshal(&c) // Get the type; ignore the error
-	f := registry.TypeMap[c.Type()]
+	f := r.TypeMap[c.Type()]
 	if f == nil {
-		return fmt.Errorf("Unknown type %q", c.Type())
+		return fmt.Errorf("Unknown %s %s type %q", r.Subagent, r.Kind, c.Type())
 	}
 	*inner = f()
 	return unmarshal(*inner)
@@ -131,7 +140,7 @@ func (r LoggingReceiverFiles) Type() string {
 }
 
 func init() {
-	registerLoggingReceiverType(func() component { return &LoggingReceiverFiles{} })
+	loggingReceiverTypes.registerType(func() component { return &LoggingReceiverFiles{} })
 }
 
 type LoggingReceiverSyslog struct {
@@ -147,7 +156,7 @@ func (r LoggingReceiverSyslog) Type() string {
 }
 
 func init() {
-	registerLoggingReceiverType(func() component { return &LoggingReceiverSyslog{} })
+	loggingReceiverTypes.registerType(func() component { return &LoggingReceiverSyslog{} })
 }
 
 type LoggingReceiverWinevtlog struct {
@@ -161,7 +170,7 @@ func (r LoggingReceiverWinevtlog) Type() string {
 }
 
 func init() {
-	registerLoggingReceiverType(func() component { return &LoggingReceiverWinevtlog{} })
+	loggingReceiverTypes.registerType(func() component { return &LoggingReceiverWinevtlog{} })
 }
 
 var loggingReceiverTypes = &componentTypeRegistry{
@@ -169,22 +178,13 @@ var loggingReceiverTypes = &componentTypeRegistry{
 	TypeMap: map[string]func() component{},
 }
 
-func registerLoggingReceiverType(constructor func() component) error {
-	name := constructor().(LoggingReceiver).Type()
-	if _, ok := loggingReceiverTypes.TypeMap[name]; ok {
-		return fmt.Errorf("Duplicate receiver type: %q", name)
-	}
-	loggingReceiverTypes.TypeMap[name] = constructor
-	return nil
-}
-
 // Wrapper type to store the unmarshaled YAML value.
 type loggingReceiverWrapper struct {
 	inner interface{}
 }
 
 func (l *loggingReceiverWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	return unmarshalComponentYaml(loggingReceiverTypes, &l.inner, unmarshal)
+	return loggingReceiverTypes.unmarshalComponentYaml(&l.inner, unmarshal)
 }
 
 func (m *loggingReceiverMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -226,7 +226,7 @@ func (r LoggingProcessorParseJson) Type() string {
 }
 
 func init() {
-	registerLoggingProcessorType(func() component { return &LoggingProcessorParseJson{} })
+	loggingProcessorTypes.registerType(func() component { return &LoggingProcessorParseJson{} })
 }
 
 type LoggingProcessorParseRegex struct {
@@ -241,7 +241,7 @@ func (r LoggingProcessorParseRegex) Type() string {
 }
 
 func init() {
-	registerLoggingProcessorType(func() component { return &LoggingProcessorParseRegex{} })
+	loggingProcessorTypes.registerType(func() component { return &LoggingProcessorParseRegex{} })
 }
 
 var loggingProcessorTypes = &componentTypeRegistry{
@@ -249,22 +249,13 @@ var loggingProcessorTypes = &componentTypeRegistry{
 	TypeMap: map[string]func() component{},
 }
 
-func registerLoggingProcessorType(constructor func() component) error {
-	name := constructor().(LoggingProcessor).Type()
-	if _, ok := loggingProcessorTypes.TypeMap[name]; ok {
-		return fmt.Errorf("Duplicate processor type: %q", name)
-	}
-	loggingProcessorTypes.TypeMap[name] = constructor
-	return nil
-}
-
 // Wrapper type to store the unmarshaled YAML value.
 type loggingProcessorWrapper struct {
 	inner interface{}
 }
 
 func (l *loggingProcessorWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	return unmarshalComponentYaml(loggingProcessorTypes, &l.inner, unmarshal)
+	return loggingProcessorTypes.unmarshalComponentYaml(&l.inner, unmarshal)
 }
 
 func (m *loggingProcessorMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -294,7 +285,7 @@ func (r LoggingExporterGoogleCloudLogging) Type() string {
 }
 
 func init() {
-	registerLoggingExporterType(func() component { return &LoggingExporterGoogleCloudLogging{} })
+	loggingExporterTypes.registerType(func() component { return &LoggingExporterGoogleCloudLogging{} })
 }
 
 var loggingExporterTypes = &componentTypeRegistry{
@@ -302,22 +293,13 @@ var loggingExporterTypes = &componentTypeRegistry{
 	TypeMap: map[string]func() component{},
 }
 
-func registerLoggingExporterType(constructor func() component) error {
-	name := constructor().(LoggingExporter).Type()
-	if _, ok := loggingExporterTypes.TypeMap[name]; ok {
-		return fmt.Errorf("Duplicate exporter type: %q", name)
-	}
-	loggingExporterTypes.TypeMap[name] = constructor
-	return nil
-}
-
 // Wrapper type to store the unmarshaled YAML value.
 type loggingExporterWrapper struct {
 	inner interface{}
 }
 
 func (l *loggingExporterWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	return unmarshalComponentYaml(loggingExporterTypes, &l.inner, unmarshal)
+	return loggingExporterTypes.unmarshalComponentYaml(&l.inner, unmarshal)
 }
 
 func (m *loggingExporterMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -379,7 +361,7 @@ func (r MetricsReceiverHostmetrics) Type() string {
 }
 
 func init() {
-	registerMetricsReceiverType(func() component { return &MetricsReceiverHostmetrics{} })
+	metricsReceiverTypes.registerType(func() component { return &MetricsReceiverHostmetrics{} })
 }
 
 type MetricsReceiverIis struct {
@@ -393,7 +375,7 @@ func (r MetricsReceiverIis) Type() string {
 }
 
 func init() {
-	registerMetricsReceiverType(func() component { return &MetricsReceiverIis{} })
+	metricsReceiverTypes.registerType(func() component { return &MetricsReceiverIis{} })
 }
 
 type MetricsReceiverMssql struct {
@@ -407,7 +389,7 @@ func (r MetricsReceiverMssql) Type() string {
 }
 
 func init() {
-	registerMetricsReceiverType(func() component { return &MetricsReceiverMssql{} })
+	metricsReceiverTypes.registerType(func() component { return &MetricsReceiverMssql{} })
 }
 
 var metricsReceiverTypes = &componentTypeRegistry{
@@ -415,22 +397,13 @@ var metricsReceiverTypes = &componentTypeRegistry{
 	TypeMap: map[string]func() component{},
 }
 
-func registerMetricsReceiverType(constructor func() component) error {
-	name := constructor().(MetricsReceiver).Type()
-	if _, ok := metricsReceiverTypes.TypeMap[name]; ok {
-		return fmt.Errorf("Duplicate receiver type: %q", name)
-	}
-	metricsReceiverTypes.TypeMap[name] = constructor
-	return nil
-}
-
 // Wrapper type to store the unmarshaled YAML value.
 type metricsReceiverWrapper struct {
 	inner interface{}
 }
 
 func (m *metricsReceiverWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	return unmarshalComponentYaml(metricsReceiverTypes, &m.inner, unmarshal)
+	return metricsReceiverTypes.unmarshalComponentYaml(&m.inner, unmarshal)
 }
 
 func (m *metricsReceiverMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -462,7 +435,7 @@ func (r MetricsProcessorExcludeMetrics) Type() string {
 }
 
 func init() {
-	registerMetricsProcessorType(func() component { return &MetricsProcessorExcludeMetrics{} })
+	metricsProcessorTypes.registerType(func() component { return &MetricsProcessorExcludeMetrics{} })
 }
 
 var metricsProcessorTypes = &componentTypeRegistry{
@@ -470,22 +443,13 @@ var metricsProcessorTypes = &componentTypeRegistry{
 	TypeMap: map[string]func() component{},
 }
 
-func registerMetricsProcessorType(constructor func() component) error {
-	name := constructor().(MetricsProcessor).Type()
-	if _, ok := metricsProcessorTypes.TypeMap[name]; ok {
-		return fmt.Errorf("Duplicate processor type: %q", name)
-	}
-	metricsProcessorTypes.TypeMap[name] = constructor
-	return nil
-}
-
 // Wrapper type to store the unmarshaled YAML value.
 type metricsProcessorWrapper struct {
 	inner interface{}
 }
 
 func (m *metricsProcessorWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	return unmarshalComponentYaml(metricsProcessorTypes, &m.inner, unmarshal)
+	return metricsProcessorTypes.unmarshalComponentYaml(&m.inner, unmarshal)
 }
 
 func (m *metricsProcessorMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -515,7 +479,7 @@ func (r MetricsExporterGoogleCloudMonitoring) Type() string {
 }
 
 func init() {
-	registerMetricsExporterType(func() component { return &MetricsExporterGoogleCloudMonitoring{} })
+	metricsExporterTypes.registerType(func() component { return &MetricsExporterGoogleCloudMonitoring{} })
 }
 
 var metricsExporterTypes = &componentTypeRegistry{
@@ -523,22 +487,13 @@ var metricsExporterTypes = &componentTypeRegistry{
 	TypeMap: map[string]func() component{},
 }
 
-func registerMetricsExporterType(constructor func() component) error {
-	name := constructor().(MetricsExporter).Type()
-	if _, ok := metricsExporterTypes.TypeMap[name]; ok {
-		return fmt.Errorf("Duplicate exporter type: %q", name)
-	}
-	metricsExporterTypes.TypeMap[name] = constructor
-	return nil
-}
-
 // Wrapper type to store the unmarshaled YAML value.
 type metricsExporterWrapper struct {
 	inner interface{}
 }
 
 func (l *metricsExporterWrapper) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	return unmarshalComponentYaml(metricsExporterTypes, &l.inner, unmarshal)
+	return metricsExporterTypes.unmarshalComponentYaml(&l.inner, unmarshal)
 }
 
 func (m *metricsExporterMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
diff --git a/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
index 56a44eb816..a9b6d3d93a 100644
--- a/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: Unknown type "unsupported_type"
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: Unknown logging processor type "unsupported_type"
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
index 56a44eb816..c1239deb57 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: Unknown type "unsupported_type"
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: Unknown logging receiver type "unsupported_type"
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
index 56a44eb816..794ceaf2ee 100644
--- a/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: Unknown type "unsupported_type"
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: Unknown metrics processor type "unsupported_type"
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
index 56a44eb816..fc86952cee 100644
--- a/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: Unknown type "unsupported_type"
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: Unknown metrics receiver type "unsupported_type"
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
index 56a44eb816..fc86952cee 100644
--- a/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: Unknown type "unsupported_type"
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: Unknown metrics receiver type "unsupported_type"
\ No newline at end of file

From 12916864fabf0389c950d6babbd26e901147b382 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Sat, 31 Jul 2021 22:20:54 -0400
Subject: [PATCH 19/41] Reorder functions to group metrics generation and
 logging generation.

---
 confgenerator/confgenerator.go | 447 +++++++++++++++++----------------
 1 file changed, 224 insertions(+), 223 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 25d9794064..18a4682c66 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -60,6 +60,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 		for _, p := range metrics.Service.Pipelines {
 			p.ExporterIDs = []string{"google"}
 		}
+
 		var err error
 		hostMetricsList, mssqlList, iisList, receiverNameMap, err = generateOtelReceivers(metrics.Receivers, metrics.Service.Pipelines)
 		if err != nil {
@@ -96,6 +97,167 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 	return otelConfig, nil
 }
 
+type hostmetricsReceiverFactory struct {
+	CollectionInterval string
+}
+
+type mssqlReceiverFactory struct {
+	CollectionInterval string
+}
+
+type iisReceiverFactory struct {
+	CollectionInterval string
+}
+
+func extractOtelReceiverFactories(receivers map[string]MetricsReceiver) (map[string]*hostmetricsReceiverFactory, map[string]*mssqlReceiverFactory, map[string]*iisReceiverFactory, error) {
+	hostmetricsReceiverFactories := map[string]*hostmetricsReceiverFactory{}
+	mssqlReceiverFactories := map[string]*mssqlReceiverFactory{}
+	iisReceiverFactories := map[string]*iisReceiverFactory{}
+	for n, r := range receivers {
+		switch r.Type() {
+		case "hostmetrics":
+			r := r.(*MetricsReceiverHostmetrics)
+			hostmetricsReceiverFactories[n] = &hostmetricsReceiverFactory{
+				CollectionInterval: r.CollectionInterval,
+			}
+		case "mssql":
+			r := r.(*MetricsReceiverMssql)
+			mssqlReceiverFactories[n] = &mssqlReceiverFactory{
+				CollectionInterval: r.CollectionInterval,
+			}
+		case "iis":
+			r := r.(*MetricsReceiverIis)
+			iisReceiverFactories[n] = &iisReceiverFactory{
+				CollectionInterval: r.CollectionInterval,
+			}
+		}
+	}
+	return hostmetricsReceiverFactories, mssqlReceiverFactories, iisReceiverFactories, nil
+}
+
+func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]*otel.HostMetrics, []*otel.MSSQL, []*otel.IIS, map[string]string, error) {
+	hostMetricsList := []*otel.HostMetrics{}
+	mssqlList := []*otel.MSSQL{}
+	iisList := []*otel.IIS{}
+	receiverNameMap := make(map[string]string)
+	hostmetricsReceiverFactories, mssqlReceiverFactories, iisReceiverFactories, err := extractOtelReceiverFactories(receivers)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	for _, pID := range sortedKeys(pipelines) {
+		p := pipelines[pID]
+		for _, rID := range p.ReceiverIDs {
+			if _, ok := receiverNameMap[rID]; ok {
+				continue
+			}
+			if h, ok := hostmetricsReceiverFactories[rID]; ok {
+				hostMetrics := otel.HostMetrics{
+					HostMetricsID:      rID,
+					CollectionInterval: h.CollectionInterval,
+				}
+				hostMetricsList = append(hostMetricsList, &hostMetrics)
+				receiverNameMap[rID] = "hostmetrics/" + rID
+			} else if m, ok := mssqlReceiverFactories[rID]; ok {
+				mssql := otel.MSSQL{
+					MSSQLID:            rID,
+					CollectionInterval: m.CollectionInterval,
+				}
+				mssqlList = append(mssqlList, &mssql)
+				receiverNameMap[rID] = "windowsperfcounters/mssql_" + rID
+			} else if i, ok := iisReceiverFactories[rID]; ok {
+				iis := otel.IIS{
+					IISID:              rID,
+					CollectionInterval: i.CollectionInterval,
+				}
+				iisList = append(iisList, &iis)
+				receiverNameMap[rID] = "windowsperfcounters/iis_" + rID
+			}
+		}
+	}
+	return hostMetricsList, mssqlList, iisList, receiverNameMap, nil
+}
+
+func generateOtelExporters(exporters map[string]MetricsExporter, pipelines map[string]*MetricsPipeline) ([]*otel.Stackdriver, map[string]string, error) {
+	stackdriverList := []*otel.Stackdriver{}
+	exportNameMap := make(map[string]string)
+	for _, pID := range sortedKeys(pipelines) {
+		p := pipelines[pID]
+		for _, eID := range p.ExporterIDs {
+			exporter, ok := exporters[eID]
+			if !ok {
+				continue
+			}
+			switch exporter.Type() {
+			case "google_cloud_monitoring":
+				if _, ok := exportNameMap[eID]; !ok {
+					stackdriver := otel.Stackdriver{
+						StackdriverID: eID,
+						Prefix:        "agent.googleapis.com/",
+					}
+					stackdriverList = append(stackdriverList, &stackdriver)
+					exportNameMap[eID] = "googlecloud/" + eID
+				}
+			}
+		}
+	}
+	return stackdriverList, exportNameMap, nil
+}
+
+type excludemetricsProcessorFactory struct {
+	MetricsPattern []string
+}
+
+func extractOtelProcessorFactories(processors map[string]MetricsProcessor) (map[string]*excludemetricsProcessorFactory, error) {
+	excludemetricsProcessorFactories := map[string]*excludemetricsProcessorFactory{}
+	for n, p := range processors {
+		switch p.Type() {
+		case "exclude_metrics":
+			p := p.(*MetricsProcessorExcludeMetrics)
+			excludemetricsProcessorFactories[n] = &excludemetricsProcessorFactory{
+				MetricsPattern: p.MetricsPattern,
+			}
+		}
+	}
+	return excludemetricsProcessorFactories, nil
+}
+
+func generateOtelProcessors(processors map[string]MetricsProcessor, pipelines map[string]*MetricsPipeline) ([]*otel.ExcludeMetrics, map[string]string, error) {
+	excludeMetricsList := []*otel.ExcludeMetrics{}
+	processorNameMap := make(map[string]string)
+	excludemetricsProcessorFactories, err := extractOtelProcessorFactories(processors)
+	if err != nil {
+		return nil, nil, err
+	}
+	for _, pID := range sortedKeys(pipelines) {
+		p := pipelines[pID]
+		for _, processorID := range p.ProcessorIDs {
+			if _, ok := processorNameMap[processorID]; ok {
+				continue
+			}
+			if p, ok := excludemetricsProcessorFactories[processorID]; ok {
+				var metricNames []string
+				for _, glob := range p.MetricsPattern {
+					// TODO: Remove TrimPrefix when we support metrics with other prefixes.
+					glob = strings.TrimPrefix(glob, "agent.googleapis.com/")
+					// TODO: Move this glob to regexp into a template function inside otel/conf.go.
+					var literals []string
+					for _, g := range strings.Split(glob, "*") {
+						literals = append(literals, regexp.QuoteMeta(g))
+					}
+					metricNames = append(metricNames, fmt.Sprintf(`^%s$`, strings.Join(literals, `.*`)))
+				}
+				processorNameMap[processorID] = "filter/exclude_" + processorID
+				excludeMetrics := otel.ExcludeMetrics{
+					ExcludeMetricsID: processorNameMap[processorID],
+					MetricNames:      metricNames,
+				}
+				excludeMetricsList = append(excludeMetricsList, &excludeMetrics)
+			}
+		}
+	}
+	return excludeMetricsList, processorNameMap, nil
+}
+
 func generateOtelServices(receiverNameMap map[string]string, exporterNameMap map[string]string, processorNameMap map[string]string, pipelines map[string]*MetricsPipeline) ([]*otel.Service, error) {
 	serviceList := []*otel.Service{}
 	for _, pID := range sortedKeys(pipelines) {
@@ -136,6 +298,68 @@ func generateOtelServices(receiverNameMap map[string]string, exporterNameMap map
 	return serviceList, nil
 }
 
+// GenerateFluentBitConfigs generates FluentBit configuration from unified agents configuration
+// in yaml. GenerateFluentBitConfigs returns empty configurations without an error if `logging`
+// does not exist as a top-level field in the input yaml format.
+func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir string, hostInfo *host.InfoStat) (string, string, error) {
+	logging := uc.Logging
+	fbTails := defaultTails(logsDir, stateDir, hostInfo)
+	userAgent, _ := getUserAgent("Google-Cloud-Ops-Agent-Logging", hostInfo)
+	fbStackdrivers := defaultStackdriverOutputs(hostInfo)
+	fbSyslogs := []*fluentbit.Syslog{}
+	fbWinEventlogs := []*fluentbit.WindowsEventlog{}
+	fbFilterParserGroups := []fluentbit.FilterParserGroup{}
+	fbFilterAddLogNames := []*fluentbit.FilterModifyAddLogName{}
+	fbFilterRewriteTags := []*fluentbit.FilterRewriteTag{}
+	fbFilterRemoveLogNames := []*fluentbit.FilterModifyRemoveLogName{}
+	jsonParsers := []*fluentbit.ParserJSON{}
+	regexParsers := []*fluentbit.ParserRegex{}
+
+	if logging != nil && logging.Service != nil {
+		// Override any user-specified exporters
+		// TODO: Refactor remaining code to not consult these fields
+		logging.Exporters = map[string]LoggingExporter{
+			"google": &LoggingExporterGoogleCloudLogging{
+				configComponent: configComponent{ComponentType: "google_cloud_logging"},
+			},
+		}
+		for _, p := range logging.Service.Pipelines {
+			p.ExporterIDs = []string{"google"}
+		}
+
+		extractedTails := []*fluentbit.Tail{}
+		var err error
+		extractedTails, fbSyslogs, fbWinEventlogs, err = generateFluentBitInputs(logging.Receivers, logging.Service.Pipelines, stateDir, hostInfo)
+		if err != nil {
+			return "", "", err
+		}
+		fbTails = append(fbTails, extractedTails...)
+		fbFilterParserGroups, err = generateFluentBitFilters(logging.Processors, logging.Service.Pipelines)
+		if err != nil {
+			return "", "", err
+		}
+		extractedStackdrivers := []*fluentbit.Stackdriver{}
+		fbFilterAddLogNames, fbFilterRewriteTags, fbFilterRemoveLogNames, extractedStackdrivers, err = extractExporterPlugins(logging.Exporters, logging.Service.Pipelines, hostInfo)
+		if err != nil {
+			return "", "", err
+		}
+		fbStackdrivers = append(fbStackdrivers, extractedStackdrivers...)
+		jsonParsers, regexParsers, err = extractFluentBitParsers(logging.Processors)
+		if err != nil {
+			return "", "", err
+		}
+	}
+	mainConfig, err := fluentbit.GenerateFluentBitMainConfig(fbTails, fbSyslogs, fbWinEventlogs, fbFilterParserGroups, fbFilterAddLogNames, fbFilterRewriteTags, fbFilterRemoveLogNames, fbStackdrivers, userAgent)
+	if err != nil {
+		return "", "", err
+	}
+	parserConfig, err := fluentbit.GenerateFluentBitParserConfig(jsonParsers, regexParsers)
+	if err != nil {
+		return "", "", err
+	}
+	return mainConfig, parserConfig, nil
+}
+
 // defaultTails returns the default Tail sections for the agents' own logs.
 func defaultTails(logsDir string, stateDir string, hostInfo *host.InfoStat) (tails []*fluentbit.Tail) {
 	tails = []*fluentbit.Tail{}
@@ -208,68 +432,6 @@ func getWorkers(hostInfo *host.InfoStat) int {
 	}
 }
 
-// GenerateFluentBitConfigs generates FluentBit configuration from unified agents configuration
-// in yaml. GenerateFluentBitConfigs returns empty configurations without an error if `logging`
-// does not exist as a top-level field in the input yaml format.
-func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir string, hostInfo *host.InfoStat) (string, string, error) {
-	logging := uc.Logging
-	fbTails := defaultTails(logsDir, stateDir, hostInfo)
-	userAgent, _ := getUserAgent("Google-Cloud-Ops-Agent-Logging", hostInfo)
-	fbStackdrivers := defaultStackdriverOutputs(hostInfo)
-	fbSyslogs := []*fluentbit.Syslog{}
-	fbWinEventlogs := []*fluentbit.WindowsEventlog{}
-	fbFilterParserGroups := []fluentbit.FilterParserGroup{}
-	fbFilterAddLogNames := []*fluentbit.FilterModifyAddLogName{}
-	fbFilterRewriteTags := []*fluentbit.FilterRewriteTag{}
-	fbFilterRemoveLogNames := []*fluentbit.FilterModifyRemoveLogName{}
-	jsonParsers := []*fluentbit.ParserJSON{}
-	regexParsers := []*fluentbit.ParserRegex{}
-
-	if logging != nil && logging.Service != nil {
-		// Override any user-specified exporters
-		// TODO: Refactor remaining code to not consult these fields
-		logging.Exporters = map[string]LoggingExporter{
-			"google": &LoggingExporterGoogleCloudLogging{
-				configComponent: configComponent{ComponentType: "google_cloud_logging"},
-			},
-		}
-		for _, p := range logging.Service.Pipelines {
-			p.ExporterIDs = []string{"google"}
-		}
-
-		extractedTails := []*fluentbit.Tail{}
-		var err error
-		extractedTails, fbSyslogs, fbWinEventlogs, err = generateFluentBitInputs(logging.Receivers, logging.Service.Pipelines, stateDir, hostInfo)
-		if err != nil {
-			return "", "", err
-		}
-		fbTails = append(fbTails, extractedTails...)
-		fbFilterParserGroups, err = generateFluentBitFilters(logging.Processors, logging.Service.Pipelines)
-		if err != nil {
-			return "", "", err
-		}
-		extractedStackdrivers := []*fluentbit.Stackdriver{}
-		fbFilterAddLogNames, fbFilterRewriteTags, fbFilterRemoveLogNames, extractedStackdrivers, err = extractExporterPlugins(logging.Exporters, logging.Service.Pipelines, hostInfo)
-		if err != nil {
-			return "", "", err
-		}
-		fbStackdrivers = append(fbStackdrivers, extractedStackdrivers...)
-		jsonParsers, regexParsers, err = extractFluentBitParsers(logging.Processors)
-		if err != nil {
-			return "", "", err
-		}
-	}
-	mainConfig, err := fluentbit.GenerateFluentBitMainConfig(fbTails, fbSyslogs, fbWinEventlogs, fbFilterParserGroups, fbFilterAddLogNames, fbFilterRewriteTags, fbFilterRemoveLogNames, fbStackdrivers, userAgent)
-	if err != nil {
-		return "", "", err
-	}
-	parserConfig, err := fluentbit.GenerateFluentBitParserConfig(jsonParsers, regexParsers)
-	if err != nil {
-		return "", "", err
-	}
-	return mainConfig, parserConfig, nil
-}
-
 type syslogReceiverFactory struct {
 	TransportProtocol string
 	ListenHost        string
@@ -285,62 +447,6 @@ type wineventlogReceiverFactory struct {
 	Channels []string
 }
 
-type hostmetricsReceiverFactory struct {
-	CollectionInterval string
-}
-
-type mssqlReceiverFactory struct {
-	CollectionInterval string
-}
-
-type iisReceiverFactory struct {
-	CollectionInterval string
-}
-
-type excludemetricsProcessorFactory struct {
-	MetricsPattern []string
-}
-
-func extractOtelReceiverFactories(receivers map[string]MetricsReceiver) (map[string]*hostmetricsReceiverFactory, map[string]*mssqlReceiverFactory, map[string]*iisReceiverFactory, error) {
-	hostmetricsReceiverFactories := map[string]*hostmetricsReceiverFactory{}
-	mssqlReceiverFactories := map[string]*mssqlReceiverFactory{}
-	iisReceiverFactories := map[string]*iisReceiverFactory{}
-	for n, r := range receivers {
-		switch r.Type() {
-		case "hostmetrics":
-			r := r.(*MetricsReceiverHostmetrics)
-			hostmetricsReceiverFactories[n] = &hostmetricsReceiverFactory{
-				CollectionInterval: r.CollectionInterval,
-			}
-		case "mssql":
-			r := r.(*MetricsReceiverMssql)
-			mssqlReceiverFactories[n] = &mssqlReceiverFactory{
-				CollectionInterval: r.CollectionInterval,
-			}
-		case "iis":
-			r := r.(*MetricsReceiverIis)
-			iisReceiverFactories[n] = &iisReceiverFactory{
-				CollectionInterval: r.CollectionInterval,
-			}
-		}
-	}
-	return hostmetricsReceiverFactories, mssqlReceiverFactories, iisReceiverFactories, nil
-}
-
-func extractOtelProcessorFactories(processors map[string]MetricsProcessor) (map[string]*excludemetricsProcessorFactory, error) {
-	excludemetricsProcessorFactories := map[string]*excludemetricsProcessorFactory{}
-	for n, p := range processors {
-		switch p.Type() {
-		case "exclude_metrics":
-			p := p.(*MetricsProcessorExcludeMetrics)
-			excludemetricsProcessorFactories[n] = &excludemetricsProcessorFactory{
-				MetricsPattern: p.MetricsPattern,
-			}
-		}
-	}
-	return excludemetricsProcessorFactories, nil
-}
-
 func extractReceiverFactories(receivers map[string]LoggingReceiver) (map[string]*fileReceiverFactory, map[string]*syslogReceiverFactory, map[string]*wineventlogReceiverFactory, error) {
 	fileReceiverFactories := map[string]*fileReceiverFactory{}
 	syslogReceiverFactories := map[string]*syslogReceiverFactory{}
@@ -370,111 +476,6 @@ func extractReceiverFactories(receivers map[string]LoggingReceiver) (map[string]
 	return fileReceiverFactories, syslogReceiverFactories, wineventlogReceiverFactories, nil
 }
 
-func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]*otel.HostMetrics, []*otel.MSSQL, []*otel.IIS, map[string]string, error) {
-	hostMetricsList := []*otel.HostMetrics{}
-	mssqlList := []*otel.MSSQL{}
-	iisList := []*otel.IIS{}
-	receiverNameMap := make(map[string]string)
-	hostmetricsReceiverFactories, mssqlReceiverFactories, iisReceiverFactories, err := extractOtelReceiverFactories(receivers)
-	if err != nil {
-		return nil, nil, nil, nil, err
-	}
-	for _, pID := range sortedKeys(pipelines) {
-		p := pipelines[pID]
-		for _, rID := range p.ReceiverIDs {
-			if _, ok := receiverNameMap[rID]; ok {
-				continue
-			}
-			if h, ok := hostmetricsReceiverFactories[rID]; ok {
-				hostMetrics := otel.HostMetrics{
-					HostMetricsID:      rID,
-					CollectionInterval: h.CollectionInterval,
-				}
-				hostMetricsList = append(hostMetricsList, &hostMetrics)
-				receiverNameMap[rID] = "hostmetrics/" + rID
-			} else if m, ok := mssqlReceiverFactories[rID]; ok {
-				mssql := otel.MSSQL{
-					MSSQLID:            rID,
-					CollectionInterval: m.CollectionInterval,
-				}
-				mssqlList = append(mssqlList, &mssql)
-				receiverNameMap[rID] = "windowsperfcounters/mssql_" + rID
-			} else if i, ok := iisReceiverFactories[rID]; ok {
-				iis := otel.IIS{
-					IISID:              rID,
-					CollectionInterval: i.CollectionInterval,
-				}
-				iisList = append(iisList, &iis)
-				receiverNameMap[rID] = "windowsperfcounters/iis_" + rID
-			}
-		}
-	}
-	return hostMetricsList, mssqlList, iisList, receiverNameMap, nil
-}
-
-func generateOtelExporters(exporters map[string]MetricsExporter, pipelines map[string]*MetricsPipeline) ([]*otel.Stackdriver, map[string]string, error) {
-	stackdriverList := []*otel.Stackdriver{}
-	exportNameMap := make(map[string]string)
-	for _, pID := range sortedKeys(pipelines) {
-		p := pipelines[pID]
-		for _, eID := range p.ExporterIDs {
-			exporter, ok := exporters[eID]
-			if !ok {
-				continue
-			}
-			switch exporter.Type() {
-			case "google_cloud_monitoring":
-				if _, ok := exportNameMap[eID]; !ok {
-					stackdriver := otel.Stackdriver{
-						StackdriverID: eID,
-						Prefix:        "agent.googleapis.com/",
-					}
-					stackdriverList = append(stackdriverList, &stackdriver)
-					exportNameMap[eID] = "googlecloud/" + eID
-				}
-			}
-		}
-	}
-	return stackdriverList, exportNameMap, nil
-}
-
-func generateOtelProcessors(processors map[string]MetricsProcessor, pipelines map[string]*MetricsPipeline) ([]*otel.ExcludeMetrics, map[string]string, error) {
-	excludeMetricsList := []*otel.ExcludeMetrics{}
-	processorNameMap := make(map[string]string)
-	excludemetricsProcessorFactories, err := extractOtelProcessorFactories(processors)
-	if err != nil {
-		return nil, nil, err
-	}
-	for _, pID := range sortedKeys(pipelines) {
-		p := pipelines[pID]
-		for _, processorID := range p.ProcessorIDs {
-			if _, ok := processorNameMap[processorID]; ok {
-				continue
-			}
-			if p, ok := excludemetricsProcessorFactories[processorID]; ok {
-				var metricNames []string
-				for _, glob := range p.MetricsPattern {
-					// TODO: Remove TrimPrefix when we support metrics with other prefixes.
-					glob = strings.TrimPrefix(glob, "agent.googleapis.com/")
-					// TODO: Move this glob to regexp into a template function inside otel/conf.go.
-					var literals []string
-					for _, g := range strings.Split(glob, "*") {
-						literals = append(literals, regexp.QuoteMeta(g))
-					}
-					metricNames = append(metricNames, fmt.Sprintf(`^%s$`, strings.Join(literals, `.*`)))
-				}
-				processorNameMap[processorID] = "filter/exclude_" + processorID
-				excludeMetrics := otel.ExcludeMetrics{
-					ExcludeMetricsID: processorNameMap[processorID],
-					MetricNames:      metricNames,
-				}
-				excludeMetricsList = append(excludeMetricsList, &excludeMetrics)
-			}
-		}
-	}
-	return excludeMetricsList, processorNameMap, nil
-}
-
 func generateFluentBitInputs(receivers map[string]LoggingReceiver, pipelines map[string]*LoggingPipeline, stateDir string, hostInfo *host.InfoStat) ([]*fluentbit.Tail, []*fluentbit.Syslog, []*fluentbit.WindowsEventlog, error) {
 	fbTails := []*fluentbit.Tail{}
 	fbSyslogs := []*fluentbit.Syslog{}

From 26986573ca206c65c1d5f8fe344f7015513bce56 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Tue, 3 Aug 2021 20:38:59 -0400
Subject: [PATCH 20/41] Inline extract.*Factories functions.

---
 confgenerator/confgenerator.go | 48 +++++++++-------------------------
 1 file changed, 12 insertions(+), 36 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 18a4682c66..ca8f5a914f 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -109,7 +109,11 @@ type iisReceiverFactory struct {
 	CollectionInterval string
 }
 
-func extractOtelReceiverFactories(receivers map[string]MetricsReceiver) (map[string]*hostmetricsReceiverFactory, map[string]*mssqlReceiverFactory, map[string]*iisReceiverFactory, error) {
+func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]*otel.HostMetrics, []*otel.MSSQL, []*otel.IIS, map[string]string, error) {
+	hostMetricsList := []*otel.HostMetrics{}
+	mssqlList := []*otel.MSSQL{}
+	iisList := []*otel.IIS{}
+	receiverNameMap := make(map[string]string)
 	hostmetricsReceiverFactories := map[string]*hostmetricsReceiverFactory{}
 	mssqlReceiverFactories := map[string]*mssqlReceiverFactory{}
 	iisReceiverFactories := map[string]*iisReceiverFactory{}
@@ -132,18 +136,6 @@ func extractOtelReceiverFactories(receivers map[string]MetricsReceiver) (map[str
 			}
 		}
 	}
-	return hostmetricsReceiverFactories, mssqlReceiverFactories, iisReceiverFactories, nil
-}
-
-func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]*otel.HostMetrics, []*otel.MSSQL, []*otel.IIS, map[string]string, error) {
-	hostMetricsList := []*otel.HostMetrics{}
-	mssqlList := []*otel.MSSQL{}
-	iisList := []*otel.IIS{}
-	receiverNameMap := make(map[string]string)
-	hostmetricsReceiverFactories, mssqlReceiverFactories, iisReceiverFactories, err := extractOtelReceiverFactories(receivers)
-	if err != nil {
-		return nil, nil, nil, nil, err
-	}
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
 		for _, rID := range p.ReceiverIDs {
@@ -207,7 +199,9 @@ type excludemetricsProcessorFactory struct {
 	MetricsPattern []string
 }
 
-func extractOtelProcessorFactories(processors map[string]MetricsProcessor) (map[string]*excludemetricsProcessorFactory, error) {
+func generateOtelProcessors(processors map[string]MetricsProcessor, pipelines map[string]*MetricsPipeline) ([]*otel.ExcludeMetrics, map[string]string, error) {
+	excludeMetricsList := []*otel.ExcludeMetrics{}
+	processorNameMap := make(map[string]string)
 	excludemetricsProcessorFactories := map[string]*excludemetricsProcessorFactory{}
 	for n, p := range processors {
 		switch p.Type() {
@@ -218,16 +212,6 @@ func extractOtelProcessorFactories(processors map[string]MetricsProcessor) (map[
 			}
 		}
 	}
-	return excludemetricsProcessorFactories, nil
-}
-
-func generateOtelProcessors(processors map[string]MetricsProcessor, pipelines map[string]*MetricsPipeline) ([]*otel.ExcludeMetrics, map[string]string, error) {
-	excludeMetricsList := []*otel.ExcludeMetrics{}
-	processorNameMap := make(map[string]string)
-	excludemetricsProcessorFactories, err := extractOtelProcessorFactories(processors)
-	if err != nil {
-		return nil, nil, err
-	}
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
 		for _, processorID := range p.ProcessorIDs {
@@ -447,7 +431,10 @@ type wineventlogReceiverFactory struct {
 	Channels []string
 }
 
-func extractReceiverFactories(receivers map[string]LoggingReceiver) (map[string]*fileReceiverFactory, map[string]*syslogReceiverFactory, map[string]*wineventlogReceiverFactory, error) {
+func generateFluentBitInputs(receivers map[string]LoggingReceiver, pipelines map[string]*LoggingPipeline, stateDir string, hostInfo *host.InfoStat) ([]*fluentbit.Tail, []*fluentbit.Syslog, []*fluentbit.WindowsEventlog, error) {
+	fbTails := []*fluentbit.Tail{}
+	fbSyslogs := []*fluentbit.Syslog{}
+	fbWinEventlogs := []*fluentbit.WindowsEventlog{}
 	fileReceiverFactories := map[string]*fileReceiverFactory{}
 	syslogReceiverFactories := map[string]*syslogReceiverFactory{}
 	wineventlogReceiverFactories := map[string]*wineventlogReceiverFactory{}
@@ -473,17 +460,6 @@ func extractReceiverFactories(receivers map[string]LoggingReceiver) (map[string]
 			}
 		}
 	}
-	return fileReceiverFactories, syslogReceiverFactories, wineventlogReceiverFactories, nil
-}
-
-func generateFluentBitInputs(receivers map[string]LoggingReceiver, pipelines map[string]*LoggingPipeline, stateDir string, hostInfo *host.InfoStat) ([]*fluentbit.Tail, []*fluentbit.Syslog, []*fluentbit.WindowsEventlog, error) {
-	fbTails := []*fluentbit.Tail{}
-	fbSyslogs := []*fluentbit.Syslog{}
-	fbWinEventlogs := []*fluentbit.WindowsEventlog{}
-	fileReceiverFactories, syslogReceiverFactories, wineventlogReceiverFactories, err := extractReceiverFactories(receivers)
-	if err != nil {
-		return nil, nil, nil, err
-	}
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
 		for _, rID := range p.ReceiverIDs {

From c3870121c658f42ed33bc4684d67d67665ae3723 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Tue, 3 Aug 2021 23:48:11 -0400
Subject: [PATCH 21/41] Eliminate otel receiver factories.

---
 confgenerator/confgenerator.go | 80 +++++++++++-----------------------
 1 file changed, 26 insertions(+), 54 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index ca8f5a914f..6d231a3826 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -97,72 +97,44 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 	return otelConfig, nil
 }
 
-type hostmetricsReceiverFactory struct {
-	CollectionInterval string
-}
-
-type mssqlReceiverFactory struct {
-	CollectionInterval string
-}
-
-type iisReceiverFactory struct {
-	CollectionInterval string
-}
-
 func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]*otel.HostMetrics, []*otel.MSSQL, []*otel.IIS, map[string]string, error) {
 	hostMetricsList := []*otel.HostMetrics{}
 	mssqlList := []*otel.MSSQL{}
 	iisList := []*otel.IIS{}
 	receiverNameMap := make(map[string]string)
-	hostmetricsReceiverFactories := map[string]*hostmetricsReceiverFactory{}
-	mssqlReceiverFactories := map[string]*mssqlReceiverFactory{}
-	iisReceiverFactories := map[string]*iisReceiverFactory{}
-	for n, r := range receivers {
-		switch r.Type() {
-		case "hostmetrics":
-			r := r.(*MetricsReceiverHostmetrics)
-			hostmetricsReceiverFactories[n] = &hostmetricsReceiverFactory{
-				CollectionInterval: r.CollectionInterval,
-			}
-		case "mssql":
-			r := r.(*MetricsReceiverMssql)
-			mssqlReceiverFactories[n] = &mssqlReceiverFactory{
-				CollectionInterval: r.CollectionInterval,
-			}
-		case "iis":
-			r := r.(*MetricsReceiverIis)
-			iisReceiverFactories[n] = &iisReceiverFactory{
-				CollectionInterval: r.CollectionInterval,
-			}
-		}
-	}
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
 		for _, rID := range p.ReceiverIDs {
 			if _, ok := receiverNameMap[rID]; ok {
 				continue
 			}
-			if h, ok := hostmetricsReceiverFactories[rID]; ok {
-				hostMetrics := otel.HostMetrics{
-					HostMetricsID:      rID,
-					CollectionInterval: h.CollectionInterval,
-				}
-				hostMetricsList = append(hostMetricsList, &hostMetrics)
-				receiverNameMap[rID] = "hostmetrics/" + rID
-			} else if m, ok := mssqlReceiverFactories[rID]; ok {
-				mssql := otel.MSSQL{
-					MSSQLID:            rID,
-					CollectionInterval: m.CollectionInterval,
-				}
-				mssqlList = append(mssqlList, &mssql)
-				receiverNameMap[rID] = "windowsperfcounters/mssql_" + rID
-			} else if i, ok := iisReceiverFactories[rID]; ok {
-				iis := otel.IIS{
-					IISID:              rID,
-					CollectionInterval: i.CollectionInterval,
+			if r, ok := receivers[rID]; ok {
+				switch r.Type() {
+				case "hostmetrics":
+					r := r.(*MetricsReceiverHostmetrics)
+					hostMetrics := otel.HostMetrics{
+						HostMetricsID:      rID,
+						CollectionInterval: r.CollectionInterval,
+					}
+					hostMetricsList = append(hostMetricsList, &hostMetrics)
+					receiverNameMap[rID] = "hostmetrics/" + rID
+				case "mssql":
+					r := r.(*MetricsReceiverMssql)
+					mssql := otel.MSSQL{
+						MSSQLID:            rID,
+						CollectionInterval: r.CollectionInterval,
+					}
+					mssqlList = append(mssqlList, &mssql)
+					receiverNameMap[rID] = "windowsperfcounters/mssql_" + rID
+				case "iis":
+					r := r.(*MetricsReceiverIis)
+					iis := otel.IIS{
+						IISID:              rID,
+						CollectionInterval: r.CollectionInterval,
+					}
+					iisList = append(iisList, &iis)
+					receiverNameMap[rID] = "windowsperfcounters/iis_" + rID
 				}
-				iisList = append(iisList, &iis)
-				receiverNameMap[rID] = "windowsperfcounters/iis_" + rID
 			}
 		}
 	}

From c2cb23aeb9bb936a1f12328855152121d5284080 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Tue, 3 Aug 2021 23:52:37 -0400
Subject: [PATCH 22/41] Eliminate otel processor factories.

---
 confgenerator/confgenerator.go | 50 ++++++++++++++--------------------
 1 file changed, 20 insertions(+), 30 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 6d231a3826..3edcd6e9a5 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -167,47 +167,37 @@ func generateOtelExporters(exporters map[string]MetricsExporter, pipelines map[s
 	return stackdriverList, exportNameMap, nil
 }
 
-type excludemetricsProcessorFactory struct {
-	MetricsPattern []string
-}
-
 func generateOtelProcessors(processors map[string]MetricsProcessor, pipelines map[string]*MetricsPipeline) ([]*otel.ExcludeMetrics, map[string]string, error) {
 	excludeMetricsList := []*otel.ExcludeMetrics{}
 	processorNameMap := make(map[string]string)
-	excludemetricsProcessorFactories := map[string]*excludemetricsProcessorFactory{}
-	for n, p := range processors {
-		switch p.Type() {
-		case "exclude_metrics":
-			p := p.(*MetricsProcessorExcludeMetrics)
-			excludemetricsProcessorFactories[n] = &excludemetricsProcessorFactory{
-				MetricsPattern: p.MetricsPattern,
-			}
-		}
-	}
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
 		for _, processorID := range p.ProcessorIDs {
 			if _, ok := processorNameMap[processorID]; ok {
 				continue
 			}
-			if p, ok := excludemetricsProcessorFactories[processorID]; ok {
-				var metricNames []string
-				for _, glob := range p.MetricsPattern {
-					// TODO: Remove TrimPrefix when we support metrics with other prefixes.
-					glob = strings.TrimPrefix(glob, "agent.googleapis.com/")
-					// TODO: Move this glob to regexp into a template function inside otel/conf.go.
-					var literals []string
-					for _, g := range strings.Split(glob, "*") {
-						literals = append(literals, regexp.QuoteMeta(g))
+			if p, ok := processors[processorID]; ok {
+				switch p.Type() {
+				case "exclude_metrics":
+					p := p.(*MetricsProcessorExcludeMetrics)
+					var metricNames []string
+					for _, glob := range p.MetricsPattern {
+						// TODO: Remove TrimPrefix when we support metrics with other prefixes.
+						glob = strings.TrimPrefix(glob, "agent.googleapis.com/")
+						// TODO: Move this glob to regexp into a template function inside otel/conf.go.
+						var literals []string
+						for _, g := range strings.Split(glob, "*") {
+							literals = append(literals, regexp.QuoteMeta(g))
+						}
+						metricNames = append(metricNames, fmt.Sprintf(`^%s$`, strings.Join(literals, `.*`)))
 					}
-					metricNames = append(metricNames, fmt.Sprintf(`^%s$`, strings.Join(literals, `.*`)))
-				}
-				processorNameMap[processorID] = "filter/exclude_" + processorID
-				excludeMetrics := otel.ExcludeMetrics{
-					ExcludeMetricsID: processorNameMap[processorID],
-					MetricNames:      metricNames,
+					processorNameMap[processorID] = "filter/exclude_" + processorID
+					excludeMetrics := otel.ExcludeMetrics{
+						ExcludeMetricsID: processorNameMap[processorID],
+						MetricNames:      metricNames,
+					}
+					excludeMetricsList = append(excludeMetricsList, &excludeMetrics)
 				}
-				excludeMetricsList = append(excludeMetricsList, &excludeMetrics)
 			}
 		}
 	}

From 01fd6d33b5ccf8b62605e7ac612d890b1d634da1 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Tue, 3 Aug 2021 23:57:46 -0400
Subject: [PATCH 23/41] Eliminate fluentbit receiver factories.

---
 confgenerator/confgenerator.go | 101 ++++++++++-----------------------
 1 file changed, 31 insertions(+), 70 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 3edcd6e9a5..c9e7814389 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -378,84 +378,45 @@ func getWorkers(hostInfo *host.InfoStat) int {
 	}
 }
 
-type syslogReceiverFactory struct {
-	TransportProtocol string
-	ListenHost        string
-	ListenPort        uint16
-}
-
-type fileReceiverFactory struct {
-	IncludePaths []string
-	ExcludePaths []string
-}
-
-type wineventlogReceiverFactory struct {
-	Channels []string
-}
-
 func generateFluentBitInputs(receivers map[string]LoggingReceiver, pipelines map[string]*LoggingPipeline, stateDir string, hostInfo *host.InfoStat) ([]*fluentbit.Tail, []*fluentbit.Syslog, []*fluentbit.WindowsEventlog, error) {
 	fbTails := []*fluentbit.Tail{}
 	fbSyslogs := []*fluentbit.Syslog{}
 	fbWinEventlogs := []*fluentbit.WindowsEventlog{}
-	fileReceiverFactories := map[string]*fileReceiverFactory{}
-	syslogReceiverFactories := map[string]*syslogReceiverFactory{}
-	wineventlogReceiverFactories := map[string]*wineventlogReceiverFactory{}
-	for rID, r := range receivers {
-		switch r.Type() {
-		case "files":
-			r := r.(*LoggingReceiverFiles)
-			fileReceiverFactories[rID] = &fileReceiverFactory{
-				IncludePaths: r.IncludePaths,
-				ExcludePaths: r.ExcludePaths,
-			}
-		case "syslog":
-			r := r.(*LoggingReceiverSyslog)
-			syslogReceiverFactories[rID] = &syslogReceiverFactory{
-				TransportProtocol: r.TransportProtocol,
-				ListenHost:        r.ListenHost,
-				ListenPort:        r.ListenPort,
-			}
-		case "windows_event_log":
-			r := r.(*LoggingReceiverWinevtlog)
-			wineventlogReceiverFactories[rID] = &wineventlogReceiverFactory{
-				Channels: r.Channels,
-			}
-		}
-	}
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
 		for _, rID := range p.ReceiverIDs {
-			if f, ok := fileReceiverFactories[rID]; ok {
-				fbTail := fluentbit.Tail{
-					Tag:  fmt.Sprintf("%s.%s", pID, rID),
-					DB:   filepathJoin(hostInfo.OS, stateDir, "buffers", pID+"_"+rID),
-					Path: strings.Join(f.IncludePaths, ","),
-				}
-				if len(f.ExcludePaths) != 0 {
-					fbTail.ExcludePath = strings.Join(f.ExcludePaths, ",")
-				}
-				fbTails = append(fbTails, &fbTail)
-				continue
-			}
-			if f, ok := syslogReceiverFactories[rID]; ok {
-				fbSyslog := fluentbit.Syslog{
-					Tag:    fmt.Sprintf("%s.%s", pID, rID),
-					Listen: f.ListenHost,
-					Mode:   f.TransportProtocol,
-					Port:   f.ListenPort,
-				}
-				fbSyslogs = append(fbSyslogs, &fbSyslog)
-				continue
-			}
-			if f, ok := wineventlogReceiverFactories[rID]; ok {
-				fbWinlog := fluentbit.WindowsEventlog{
-					Tag:          fmt.Sprintf("%s.%s", pID, rID),
-					Channels:     strings.Join(f.Channels, ","),
-					Interval_Sec: "1",
-					DB:           filepathJoin(hostInfo.OS, stateDir, "buffers", pID+"_"+rID),
+			if r, ok := receivers[rID]; ok {
+				switch r.Type() {
+				case "files":
+					r := r.(*LoggingReceiverFiles)
+					fbTail := fluentbit.Tail{
+						Tag:  fmt.Sprintf("%s.%s", pID, rID),
+						DB:   filepathJoin(hostInfo.OS, stateDir, "buffers", pID+"_"+rID),
+						Path: strings.Join(r.IncludePaths, ","),
+					}
+					if len(r.ExcludePaths) != 0 {
+						fbTail.ExcludePath = strings.Join(r.ExcludePaths, ",")
+					}
+					fbTails = append(fbTails, &fbTail)
+				case "syslog":
+					r := r.(*LoggingReceiverSyslog)
+					fbSyslog := fluentbit.Syslog{
+						Tag:    fmt.Sprintf("%s.%s", pID, rID),
+						Listen: r.ListenHost,
+						Mode:   r.TransportProtocol,
+						Port:   r.ListenPort,
+					}
+					fbSyslogs = append(fbSyslogs, &fbSyslog)
+				case "windows_event_log":
+					r := r.(*LoggingReceiverWinevtlog)
+					fbWinlog := fluentbit.WindowsEventlog{
+						Tag:          fmt.Sprintf("%s.%s", pID, rID),
+						Channels:     strings.Join(r.Channels, ","),
+						Interval_Sec: "1",
+						DB:           filepathJoin(hostInfo.OS, stateDir, "buffers", pID+"_"+rID),
+					}
+					fbWinEventlogs = append(fbWinEventlogs, &fbWinlog)
 				}
-				fbWinEventlogs = append(fbWinEventlogs, &fbWinlog)
-				continue
 			}
 		}
 	}

From 294d2a822f7edcd0b2d6d0576c29a1d6e4f1b7db Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Wed, 4 Aug 2021 00:08:59 -0400
Subject: [PATCH 24/41] Don't mangle otel entity ids.

---
 confgenerator/confgenerator.go  |  8 ++++----
 confgenerator/otel/conf.go      | 10 +++++-----
 confgenerator/otel/conf_test.go | 16 ++++++++--------
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index c9e7814389..3b91010d04 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -113,7 +113,7 @@ func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[s
 				case "hostmetrics":
 					r := r.(*MetricsReceiverHostmetrics)
 					hostMetrics := otel.HostMetrics{
-						HostMetricsID:      rID,
+						HostMetricsID:      "hostmetrics/" + rID,
 						CollectionInterval: r.CollectionInterval,
 					}
 					hostMetricsList = append(hostMetricsList, &hostMetrics)
@@ -121,7 +121,7 @@ func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[s
 				case "mssql":
 					r := r.(*MetricsReceiverMssql)
 					mssql := otel.MSSQL{
-						MSSQLID:            rID,
+						MSSQLID:            "windowsperfcounters/mssql_" + rID,
 						CollectionInterval: r.CollectionInterval,
 					}
 					mssqlList = append(mssqlList, &mssql)
@@ -129,7 +129,7 @@ func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[s
 				case "iis":
 					r := r.(*MetricsReceiverIis)
 					iis := otel.IIS{
-						IISID:              rID,
+						IISID:              "windowsperfcounters/iis_" + rID,
 						CollectionInterval: r.CollectionInterval,
 					}
 					iisList = append(iisList, &iis)
@@ -155,7 +155,7 @@ func generateOtelExporters(exporters map[string]MetricsExporter, pipelines map[s
 			case "google_cloud_monitoring":
 				if _, ok := exportNameMap[eID]; !ok {
 					stackdriver := otel.Stackdriver{
-						StackdriverID: eID,
+						StackdriverID: "googlecloud/" + eID,
 						Prefix:        "agent.googleapis.com/",
 					}
 					stackdriverList = append(stackdriverList, &stackdriver)
diff --git a/confgenerator/otel/conf.go b/confgenerator/otel/conf.go
index 76f903bbb3..e210451b43 100644
--- a/confgenerator/otel/conf.go
+++ b/confgenerator/otel/conf.go
@@ -49,7 +49,7 @@ service:
     {{template "service" .}}
 {{- end}}
 {{define "hostmetrics" -}}
-  hostmetrics/{{.HostMetricsID}}:
+  {{.HostMetricsID}}:
     collection_interval: {{.CollectionInterval}}
     scrapers:
       cpu:
@@ -75,7 +75,7 @@ service:
 {{- end -}}
 
 {{define "iis" -}}
-windowsperfcounters/iis_{{.IISID}}:
+{{.IISID}}:
     collection_interval: {{.CollectionInterval}}
     perfcounters:
       - object: Web Service
@@ -95,7 +95,7 @@ windowsperfcounters/iis_{{.IISID}}:
 {{- end -}}
 
 {{define "mssql" -}}
-windowsperfcounters/mssql_{{.MSSQLID}}:
+{{.MSSQLID}}:
     collection_interval: {{.CollectionInterval}}
     perfcounters:
       - object: SQLServer:General Statistics
@@ -110,7 +110,7 @@ windowsperfcounters/mssql_{{.MSSQLID}}:
 {{- end -}}
 
 {{define "stackdriver" -}}
-  googlecloud/{{.StackdriverID}}:
+  {{.StackdriverID}}:
     user_agent: {{.UserAgent}}
     metric:
       prefix: {{.Prefix}}
@@ -634,7 +634,7 @@ type Config struct {
 
 func (c Config) Generate() (string, error) {
 	c.Stackdriver = append(c.Stackdriver, &Stackdriver{
-		StackdriverID: "agent",
+		StackdriverID: "googlecloud/agent",
 		Prefix:        "agent.googleapis.com/",
 		UserAgent:     c.UserAgent,
 	})
diff --git a/confgenerator/otel/conf_test.go b/confgenerator/otel/conf_test.go
index 8a5d960fa1..cf3f712339 100644
--- a/confgenerator/otel/conf_test.go
+++ b/confgenerator/otel/conf_test.go
@@ -30,7 +30,7 @@ func TestSection(t *testing.T) {
 	}{
 		{
 			section: HostMetrics{
-				HostMetricsID:      "hostmetrics",
+				HostMetricsID:      "hostmetrics/hostmetrics",
 				CollectionInterval: "60s",
 			},
 			want: `hostmetrics/hostmetrics:
@@ -48,7 +48,7 @@ func TestSection(t *testing.T) {
 		},
 		{
 			section: IIS{
-				IISID:              "iis",
+				IISID:              "windowsperfcounters/iis_iis",
 				CollectionInterval: "60s",
 			},
 			want: `windowsperfcounters/iis_iis:
@@ -71,7 +71,7 @@ func TestSection(t *testing.T) {
 		},
 		{
 			section: MSSQL{
-				MSSQLID:            "mssql",
+				MSSQLID:            "windowsperfcounters/mssql_mssql",
 				CollectionInterval: "60s",
 			},
 			want: `windowsperfcounters/mssql_mssql:
@@ -89,7 +89,7 @@ func TestSection(t *testing.T) {
 		},
 		{
 			section: Stackdriver{
-				StackdriverID: "agent",
+				StackdriverID: "googlecloud/agent",
 				UserAgent:     "$USERAGENT",
 				Prefix:        "agent.googleapis.com/",
 			},
@@ -151,19 +151,19 @@ func TestGenerateOtelConfig(t *testing.T) {
 		{
 			name: "default system metrics config",
 			hostMetricsList: []*HostMetrics{{
-				HostMetricsID:      "hostmetrics",
+				HostMetricsID:      "hostmetrics/hostmetrics",
 				CollectionInterval: "60s",
 			}},
 			mssqlList: []*MSSQL{{
-				MSSQLID:            "mssql",
+				MSSQLID:            "windowsperfcounters/mssql_mssql",
 				CollectionInterval: "60s",
 			}},
 			iisList: []*IIS{{
-				IISID:              "iis",
+				IISID:              "windowsperfcounters/iis_iis",
 				CollectionInterval: "60s",
 			}},
 			stackdriverList: []*Stackdriver{{
-				StackdriverID: "google",
+				StackdriverID: "googlecloud/google",
 				UserAgent:     "$IGNORED_VALUE",
 				Prefix:        "agent.googleapis.com/",
 			}},

From 24d757d34da7a744f9d129e5821dcd37b0dcf63c Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Wed, 4 Aug 2021 14:48:29 -0400
Subject: [PATCH 25/41] Cosmetic: reorder otel structs for consistency.

---
 confgenerator/confgenerator.go  |  2 +-
 confgenerator/otel/conf.go      | 24 ++++++++++++------------
 confgenerator/otel/conf_test.go | 15 ++++++++-------
 3 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 3b91010d04..dbd40510d2 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -81,8 +81,8 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 	}
 	otelConfig, err := otel.Config{
 		HostMetrics:    hostMetricsList,
-		MSSQL:          mssqlList,
 		IIS:            iisList,
+		MSSQL:          mssqlList,
 		ExcludeMetrics: excludeMetricsList,
 		Stackdriver:    stackdriverList,
 		Service:        serviceList,
diff --git a/confgenerator/otel/conf.go b/confgenerator/otel/conf.go
index e210451b43..72a818ccb7 100644
--- a/confgenerator/otel/conf.go
+++ b/confgenerator/otel/conf.go
@@ -587,8 +587,8 @@ service:
           - action: toggle_scalar_data_type
 {{- end -}}`))
 
-type MSSQL struct {
-	MSSQLID            string
+type HostMetrics struct {
+	HostMetricsID      string
 	CollectionInterval string
 }
 
@@ -597,8 +597,8 @@ type IIS struct {
 	CollectionInterval string
 }
 
-type HostMetrics struct {
-	HostMetricsID      string
+type MSSQL struct {
+	MSSQLID            string
 	CollectionInterval string
 }
 
@@ -607,6 +607,12 @@ type ExcludeMetrics struct {
 	MetricNames      []string
 }
 
+type Stackdriver struct {
+	StackdriverID string
+	UserAgent     string
+	Prefix        string
+}
+
 type Service struct {
 	ID         string
 	Processors string
@@ -614,19 +620,13 @@ type Service struct {
 	Exporters  string
 }
 
-type Stackdriver struct {
-	StackdriverID string
-	UserAgent     string
-	Prefix        string
-}
-
 type Config struct {
 	HostMetrics    []*HostMetrics
-	MSSQL          []*MSSQL
 	IIS            []*IIS
+	MSSQL          []*MSSQL
+	ExcludeMetrics []*ExcludeMetrics
 	Stackdriver    []*Stackdriver
 	Service        []*Service
-	ExcludeMetrics []*ExcludeMetrics
 	UserAgent      string
 	Version        string
 	Windows        bool
diff --git a/confgenerator/otel/conf_test.go b/confgenerator/otel/conf_test.go
index cf3f712339..0530336043 100644
--- a/confgenerator/otel/conf_test.go
+++ b/confgenerator/otel/conf_test.go
@@ -142,8 +142,8 @@ func TestGenerateOtelConfig(t *testing.T) {
 	tests := []struct {
 		name            string
 		hostMetricsList []*HostMetrics
-		mssqlList       []*MSSQL
 		iisList         []*IIS
+		mssqlList       []*MSSQL
 		stackdriverList []*Stackdriver
 		serviceList     []*Service
 		want            string
@@ -154,14 +154,14 @@ func TestGenerateOtelConfig(t *testing.T) {
 				HostMetricsID:      "hostmetrics/hostmetrics",
 				CollectionInterval: "60s",
 			}},
-			mssqlList: []*MSSQL{{
-				MSSQLID:            "windowsperfcounters/mssql_mssql",
-				CollectionInterval: "60s",
-			}},
 			iisList: []*IIS{{
 				IISID:              "windowsperfcounters/iis_iis",
 				CollectionInterval: "60s",
 			}},
+			mssqlList: []*MSSQL{{
+				MSSQLID:            "windowsperfcounters/mssql_mssql",
+				CollectionInterval: "60s",
+			}},
 			stackdriverList: []*Stackdriver{{
 				StackdriverID: "googlecloud/google",
 				UserAgent:     "$IGNORED_VALUE",
@@ -180,7 +180,8 @@ func TestGenerateOtelConfig(t *testing.T) {
 					Processors: "[metricstransform/mssql,resourcedetection]",
 					Exporters:  "[googlecloud/google]",
 				},
-				{ID: "iis",
+				{
+					ID:         "iis",
 					Receivers:  "[windowsperfcounters/iis_iis]",
 					Processors: "[metricstransform/iis,resourcedetection]",
 					Exporters:  "[googlecloud/google]",
@@ -715,8 +716,8 @@ service:
 		t.Run(tc.name, func(t *testing.T) {
 			got, err := Config{
 				HostMetrics: tc.hostMetricsList,
-				MSSQL:       tc.mssqlList,
 				IIS:         tc.iisList,
+				MSSQL:       tc.mssqlList,
 				Stackdriver: tc.stackdriverList,
 				Service:     tc.serviceList,
 

From 5d3a61aff71a68c20fdf9d52a9d752bb23f1638e Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Wed, 4 Aug 2021 15:52:18 -0400
Subject: [PATCH 26/41] Unify otel's HostMetrics, IIS, and MSSQL structs under
 the otel.Receiver interface.

---
 confgenerator/confgenerator.go  | 24 +++++------
 confgenerator/otel/conf.go      | 70 +++++++++++++++++++++++++++------
 confgenerator/otel/conf_test.go | 34 ++++++++--------
 3 files changed, 86 insertions(+), 42 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index dbd40510d2..f717539b4d 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -40,9 +40,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 	metrics := uc.Metrics
 	userAgent, _ := getUserAgent("Google-Cloud-Ops-Agent-Metrics", hostInfo)
 	versionLabel, _ := getVersionLabel("google-cloud-ops-agent-metrics")
-	hostMetricsList := []*otel.HostMetrics{}
-	mssqlList := []*otel.MSSQL{}
-	iisList := []*otel.IIS{}
+	receiverList := []otel.Receiver{}
 	stackdriverList := []*otel.Stackdriver{}
 	serviceList := []*otel.Service{}
 	excludeMetricsList := []*otel.ExcludeMetrics{}
@@ -62,7 +60,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 		}
 
 		var err error
-		hostMetricsList, mssqlList, iisList, receiverNameMap, err = generateOtelReceivers(metrics.Receivers, metrics.Service.Pipelines)
+		receiverList, receiverNameMap, err = generateOtelReceivers(metrics.Receivers, metrics.Service.Pipelines)
 		if err != nil {
 			return "", err
 		}
@@ -80,9 +78,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 		}
 	}
 	otelConfig, err := otel.Config{
-		HostMetrics:    hostMetricsList,
-		IIS:            iisList,
-		MSSQL:          mssqlList,
+		Receivers:      receiverList,
 		ExcludeMetrics: excludeMetricsList,
 		Stackdriver:    stackdriverList,
 		Service:        serviceList,
@@ -97,10 +93,10 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 	return otelConfig, nil
 }
 
-func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]*otel.HostMetrics, []*otel.MSSQL, []*otel.IIS, map[string]string, error) {
-	hostMetricsList := []*otel.HostMetrics{}
-	mssqlList := []*otel.MSSQL{}
-	iisList := []*otel.IIS{}
+func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]otel.Receiver, map[string]string, error) {
+	hostMetricsList := []otel.Receiver{}
+	mssqlList := []otel.Receiver{}
+	iisList := []otel.Receiver{}
 	receiverNameMap := make(map[string]string)
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
@@ -138,7 +134,11 @@ func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[s
 			}
 		}
 	}
-	return hostMetricsList, mssqlList, iisList, receiverNameMap, nil
+	receiverList := []otel.Receiver{}
+	receiverList = append(receiverList, hostMetricsList...)
+	receiverList = append(receiverList, mssqlList...)
+	receiverList = append(receiverList, iisList...)
+	return receiverList, receiverNameMap, nil
 }
 
 func generateOtelExporters(exporters map[string]MetricsExporter, pipelines map[string]*MetricsPipeline) ([]*otel.Stackdriver, map[string]string, error) {
diff --git a/confgenerator/otel/conf.go b/confgenerator/otel/conf.go
index 72a818ccb7..28660b2e84 100644
--- a/confgenerator/otel/conf.go
+++ b/confgenerator/otel/conf.go
@@ -23,15 +23,9 @@ import (
 var confTemplate = template.Must(template.New("conf").Parse(
 	`receivers:
   {{template "agentreceiver" .}}
-{{- range .HostMetrics}}
-  {{template "hostmetrics" .}}
-{{- end}}
-{{- range .MSSQL}}
-  {{template "mssql" .}}
-{{- end}}
-{{- range .IIS}}
-  {{template "iis" .}}
-{{- end}}
+{{- range .Receivers}}
+  {{.Generate -}}
+{{end}}
 processors:
   {{template "defaultprocessor" .}}
 {{- range .ExcludeMetrics}}
@@ -587,21 +581,75 @@ service:
           - action: toggle_scalar_data_type
 {{- end -}}`))
 
+type Receiver interface {
+	Generate() (string, error)
+	pipelineID() string
+	defaultProcessors() []string
+}
+
 type HostMetrics struct {
 	HostMetricsID      string
 	CollectionInterval string
 }
 
+func (r *HostMetrics) Generate() (string, error) {
+	var builder strings.Builder
+	if err := confTemplate.ExecuteTemplate(&builder, "hostmetrics", r); err != nil {
+		return "", err
+	}
+	return builder.String(), nil
+}
+
+func (r *HostMetrics) defaultProcessors() []string {
+	return []string{"agentmetrics/system", "filter/system", "metricstransform/system", "resourcedetection"}
+}
+
+func (r *HostMetrics) pipelineID() string {
+	return "system"
+}
+
 type IIS struct {
 	IISID              string
 	CollectionInterval string
 }
 
+func (r *IIS) Generate() (string, error) {
+	var builder strings.Builder
+	if err := confTemplate.ExecuteTemplate(&builder, "iis", r); err != nil {
+		return "", err
+	}
+	return builder.String(), nil
+}
+
+func (r *IIS) defaultProcessors() []string {
+	return []string{"metricstransform/iis", "resourcedetection"}
+}
+
+func (r *IIS) pipelineID() string {
+	return "iis"
+}
+
 type MSSQL struct {
 	MSSQLID            string
 	CollectionInterval string
 }
 
+func (r *MSSQL) Generate() (string, error) {
+	var builder strings.Builder
+	if err := confTemplate.ExecuteTemplate(&builder, "mssql", r); err != nil {
+		return "", err
+	}
+	return builder.String(), nil
+}
+
+func (r *MSSQL) defaultProcessors() []string {
+	return []string{"metricstransform/mssql", "resourcedetection"}
+}
+
+func (r *MSSQL) pipelineID() string {
+	return "mssql"
+}
+
 type ExcludeMetrics struct {
 	ExcludeMetricsID string
 	MetricNames      []string
@@ -621,9 +669,7 @@ type Service struct {
 }
 
 type Config struct {
-	HostMetrics    []*HostMetrics
-	IIS            []*IIS
-	MSSQL          []*MSSQL
+	Receivers      []Receiver
 	ExcludeMetrics []*ExcludeMetrics
 	Stackdriver    []*Stackdriver
 	Service        []*Service
diff --git a/confgenerator/otel/conf_test.go b/confgenerator/otel/conf_test.go
index 0530336043..19e84f2ce2 100644
--- a/confgenerator/otel/conf_test.go
+++ b/confgenerator/otel/conf_test.go
@@ -141,27 +141,27 @@ func TestSection(t *testing.T) {
 func TestGenerateOtelConfig(t *testing.T) {
 	tests := []struct {
 		name            string
-		hostMetricsList []*HostMetrics
-		iisList         []*IIS
-		mssqlList       []*MSSQL
+		receiverList    []Receiver
 		stackdriverList []*Stackdriver
 		serviceList     []*Service
 		want            string
 	}{
 		{
 			name: "default system metrics config",
-			hostMetricsList: []*HostMetrics{{
-				HostMetricsID:      "hostmetrics/hostmetrics",
-				CollectionInterval: "60s",
-			}},
-			iisList: []*IIS{{
-				IISID:              "windowsperfcounters/iis_iis",
-				CollectionInterval: "60s",
-			}},
-			mssqlList: []*MSSQL{{
-				MSSQLID:            "windowsperfcounters/mssql_mssql",
-				CollectionInterval: "60s",
-			}},
+			receiverList: []Receiver{
+				&HostMetrics{
+					HostMetricsID:      "hostmetrics/hostmetrics",
+					CollectionInterval: "60s",
+				},
+				&MSSQL{
+					MSSQLID:            "windowsperfcounters/mssql_mssql",
+					CollectionInterval: "60s",
+				},
+				&IIS{
+					IISID:              "windowsperfcounters/iis_iis",
+					CollectionInterval: "60s",
+				},
+			},
 			stackdriverList: []*Stackdriver{{
 				StackdriverID: "googlecloud/google",
 				UserAgent:     "$IGNORED_VALUE",
@@ -715,9 +715,7 @@ service:
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			got, err := Config{
-				HostMetrics: tc.hostMetricsList,
-				IIS:         tc.iisList,
-				MSSQL:       tc.mssqlList,
+				Receivers:   tc.receiverList,
 				Stackdriver: tc.stackdriverList,
 				Service:     tc.serviceList,
 

From 29c4b963104403dd3c67731ab7a6410a951482ce Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Wed, 4 Aug 2021 17:44:59 -0400
Subject: [PATCH 27/41] Use the otel.Receiver interface instead of the receiver
 name map.

---
 confgenerator/confgenerator.go | 42 ++++++++++++++--------------------
 confgenerator/otel/conf.go     | 29 ++++++++++++++++-------
 2 files changed, 38 insertions(+), 33 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index f717539b4d..9f01fba89b 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -44,7 +44,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 	stackdriverList := []*otel.Stackdriver{}
 	serviceList := []*otel.Service{}
 	excludeMetricsList := []*otel.ExcludeMetrics{}
-	receiverNameMap := make(map[string]string)
+	receiverMap := make(map[string]otel.Receiver)
 	exporterNameMap := make(map[string]string)
 	processorNameMap := make(map[string]string)
 	if metrics != nil {
@@ -60,7 +60,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 		}
 
 		var err error
-		receiverList, receiverNameMap, err = generateOtelReceivers(metrics.Receivers, metrics.Service.Pipelines)
+		receiverList, receiverMap, err = generateOtelReceivers(metrics.Receivers, metrics.Service.Pipelines)
 		if err != nil {
 			return "", err
 		}
@@ -72,7 +72,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 		if err != nil {
 			return "", err
 		}
-		serviceList, err = generateOtelServices(receiverNameMap, exporterNameMap, processorNameMap, metrics.Service.Pipelines)
+		serviceList, err = generateOtelServices(receiverMap, exporterNameMap, processorNameMap, metrics.Service.Pipelines)
 		if err != nil {
 			return "", err
 		}
@@ -93,15 +93,15 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 	return otelConfig, nil
 }
 
-func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]otel.Receiver, map[string]string, error) {
+func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[string]*MetricsPipeline) ([]otel.Receiver, map[string]otel.Receiver, error) {
 	hostMetricsList := []otel.Receiver{}
 	mssqlList := []otel.Receiver{}
 	iisList := []otel.Receiver{}
-	receiverNameMap := make(map[string]string)
+	receiverMap := make(map[string]otel.Receiver)
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
 		for _, rID := range p.ReceiverIDs {
-			if _, ok := receiverNameMap[rID]; ok {
+			if _, ok := receiverMap[rID]; ok {
 				continue
 			}
 			if r, ok := receivers[rID]; ok {
@@ -113,7 +113,7 @@ func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[s
 						CollectionInterval: r.CollectionInterval,
 					}
 					hostMetricsList = append(hostMetricsList, &hostMetrics)
-					receiverNameMap[rID] = "hostmetrics/" + rID
+					receiverMap[rID] = &hostMetrics
 				case "mssql":
 					r := r.(*MetricsReceiverMssql)
 					mssql := otel.MSSQL{
@@ -121,7 +121,7 @@ func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[s
 						CollectionInterval: r.CollectionInterval,
 					}
 					mssqlList = append(mssqlList, &mssql)
-					receiverNameMap[rID] = "windowsperfcounters/mssql_" + rID
+					receiverMap[rID] = &mssql
 				case "iis":
 					r := r.(*MetricsReceiverIis)
 					iis := otel.IIS{
@@ -129,7 +129,7 @@ func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[s
 						CollectionInterval: r.CollectionInterval,
 					}
 					iisList = append(iisList, &iis)
-					receiverNameMap[rID] = "windowsperfcounters/iis_" + rID
+					receiverMap[rID] = &iis
 				}
 			}
 		}
@@ -138,7 +138,7 @@ func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[s
 	receiverList = append(receiverList, hostMetricsList...)
 	receiverList = append(receiverList, mssqlList...)
 	receiverList = append(receiverList, iisList...)
-	return receiverList, receiverNameMap, nil
+	return receiverList, receiverMap, nil
 }
 
 func generateOtelExporters(exporters map[string]MetricsExporter, pipelines map[string]*MetricsPipeline) ([]*otel.Stackdriver, map[string]string, error) {
@@ -204,26 +204,18 @@ func generateOtelProcessors(processors map[string]MetricsProcessor, pipelines ma
 	return excludeMetricsList, processorNameMap, nil
 }
 
-func generateOtelServices(receiverNameMap map[string]string, exporterNameMap map[string]string, processorNameMap map[string]string, pipelines map[string]*MetricsPipeline) ([]*otel.Service, error) {
+func generateOtelServices(receiverMap map[string]otel.Receiver, exporterNameMap map[string]string, processorNameMap map[string]string, pipelines map[string]*MetricsPipeline) ([]*otel.Service, error) {
 	serviceList := []*otel.Service{}
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
 		for _, rID := range p.ReceiverIDs {
-			var pipelineID string
-			var defaultProcessors []string
-			if strings.HasPrefix(receiverNameMap[rID], "hostmetrics/") {
-				defaultProcessors = []string{"agentmetrics/system", "filter/system", "metricstransform/system", "resourcedetection"}
-				pipelineID = "system"
-			} else if strings.HasPrefix(receiverNameMap[rID], "windowsperfcounters/mssql") {
-				defaultProcessors = []string{"metricstransform/mssql", "resourcedetection"}
-				pipelineID = "mssql"
-			} else if strings.HasPrefix(receiverNameMap[rID], "windowsperfcounters/iis") {
-				defaultProcessors = []string{"metricstransform/iis", "resourcedetection"}
-				pipelineID = "iis"
+			r, ok := receiverMap[rID]
+			if !ok {
+				panic(fmt.Sprintf("Internal error: receiver %q not found", rID))
 			}
 
 			var processorIDs []string
-			processorIDs = append(processorIDs, defaultProcessors...)
+			processorIDs = append(processorIDs, r.DefaultProcessors()...)
 			for _, processorID := range p.ProcessorIDs {
 				processorIDs = append(processorIDs, processorNameMap[processorID])
 			}
@@ -233,8 +225,8 @@ func generateOtelServices(receiverNameMap map[string]string, exporterNameMap map
 				pExportIDs = append(pExportIDs, exporterNameMap[eID])
 			}
 			service := otel.Service{
-				ID:         pipelineID,
-				Receivers:  fmt.Sprintf("[%s]", receiverNameMap[rID]),
+				ID:         r.DefaultPipelineID(),
+				Receivers:  fmt.Sprintf("[%s]", r.GetID()),
 				Processors: fmt.Sprintf("[%s]", strings.Join(processorIDs, ",")),
 				Exporters:  fmt.Sprintf("[%s]", strings.Join(pExportIDs, ",")),
 			}
diff --git a/confgenerator/otel/conf.go b/confgenerator/otel/conf.go
index 28660b2e84..a8e92d19d2 100644
--- a/confgenerator/otel/conf.go
+++ b/confgenerator/otel/conf.go
@@ -583,8 +583,9 @@ service:
 
 type Receiver interface {
 	Generate() (string, error)
-	pipelineID() string
-	defaultProcessors() []string
+	GetID() string
+	DefaultPipelineID() string
+	DefaultProcessors() []string
 }
 
 type HostMetrics struct {
@@ -600,11 +601,15 @@ func (r *HostMetrics) Generate() (string, error) {
 	return builder.String(), nil
 }
 
-func (r *HostMetrics) defaultProcessors() []string {
+func (r *HostMetrics) GetID() string {
+	return r.HostMetricsID
+}
+
+func (r *HostMetrics) DefaultProcessors() []string {
 	return []string{"agentmetrics/system", "filter/system", "metricstransform/system", "resourcedetection"}
 }
 
-func (r *HostMetrics) pipelineID() string {
+func (r *HostMetrics) DefaultPipelineID() string {
 	return "system"
 }
 
@@ -621,11 +626,15 @@ func (r *IIS) Generate() (string, error) {
 	return builder.String(), nil
 }
 
-func (r *IIS) defaultProcessors() []string {
+func (r *IIS) GetID() string {
+	return r.IISID
+}
+
+func (r *IIS) DefaultProcessors() []string {
 	return []string{"metricstransform/iis", "resourcedetection"}
 }
 
-func (r *IIS) pipelineID() string {
+func (r *IIS) DefaultPipelineID() string {
 	return "iis"
 }
 
@@ -642,11 +651,15 @@ func (r *MSSQL) Generate() (string, error) {
 	return builder.String(), nil
 }
 
-func (r *MSSQL) defaultProcessors() []string {
+func (r *MSSQL) GetID() string {
+	return r.MSSQLID
+}
+
+func (r *MSSQL) DefaultProcessors() []string {
 	return []string{"metricstransform/mssql", "resourcedetection"}
 }
 
-func (r *MSSQL) pipelineID() string {
+func (r *MSSQL) DefaultPipelineID() string {
 	return "mssql"
 }
 

From eca881bbf9dc1a05e5f6b2d9f74f4f3e5c783877 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Wed, 4 Aug 2021 21:48:59 -0400
Subject: [PATCH 28/41] Wrap otel's ExcludeMetrics into the otel.Processor
 interface, and otel's Stackdriver into the otel.Exporter interface. Use the
 otel.Processor and otel.Exporter interfaces instead of the processor/exporter
 name maps.

---
 confgenerator/confgenerator.go  | 65 ++++++++++++++++-----------------
 confgenerator/otel/conf.go      | 61 ++++++++++++++++++++++++-------
 confgenerator/otel/conf_test.go | 28 +++++++-------
 3 files changed, 95 insertions(+), 59 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 9f01fba89b..f0576fd5a0 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -41,12 +41,12 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 	userAgent, _ := getUserAgent("Google-Cloud-Ops-Agent-Metrics", hostInfo)
 	versionLabel, _ := getVersionLabel("google-cloud-ops-agent-metrics")
 	receiverList := []otel.Receiver{}
-	stackdriverList := []*otel.Stackdriver{}
+	exporterList := []otel.Exporter{}
 	serviceList := []*otel.Service{}
-	excludeMetricsList := []*otel.ExcludeMetrics{}
+	processorList := []otel.Processor{}
 	receiverMap := make(map[string]otel.Receiver)
-	exporterNameMap := make(map[string]string)
-	processorNameMap := make(map[string]string)
+	exporterMap := make(map[string]otel.Exporter)
+	processorMap := make(map[string]otel.Processor)
 	if metrics != nil {
 		// Override any user-specified exporters
 		// TODO: Refactor remaining code to not consult these fields
@@ -64,24 +64,24 @@ func (uc *UnifiedConfig) GenerateOtelConfig(hostInfo *host.InfoStat) (string, er
 		if err != nil {
 			return "", err
 		}
-		stackdriverList, exporterNameMap, err = generateOtelExporters(metrics.Exporters, metrics.Service.Pipelines)
+		exporterList, exporterMap, err = generateOtelExporters(metrics.Exporters, metrics.Service.Pipelines)
 		if err != nil {
 			return "", err
 		}
-		excludeMetricsList, processorNameMap, err = generateOtelProcessors(metrics.Processors, metrics.Service.Pipelines)
+		processorList, processorMap, err = generateOtelProcessors(metrics.Processors, metrics.Service.Pipelines)
 		if err != nil {
 			return "", err
 		}
-		serviceList, err = generateOtelServices(receiverMap, exporterNameMap, processorNameMap, metrics.Service.Pipelines)
+		serviceList, err = generateOtelServices(receiverMap, exporterMap, processorMap, metrics.Service.Pipelines)
 		if err != nil {
 			return "", err
 		}
 	}
 	otelConfig, err := otel.Config{
-		Receivers:      receiverList,
-		ExcludeMetrics: excludeMetricsList,
-		Stackdriver:    stackdriverList,
-		Service:        serviceList,
+		Receivers:  receiverList,
+		Processors: processorList,
+		Exporters:  exporterList,
+		Service:    serviceList,
 
 		UserAgent: userAgent,
 		Version:   versionLabel,
@@ -141,39 +141,38 @@ func generateOtelReceivers(receivers map[string]MetricsReceiver, pipelines map[s
 	return receiverList, receiverMap, nil
 }
 
-func generateOtelExporters(exporters map[string]MetricsExporter, pipelines map[string]*MetricsPipeline) ([]*otel.Stackdriver, map[string]string, error) {
-	stackdriverList := []*otel.Stackdriver{}
-	exportNameMap := make(map[string]string)
+func generateOtelExporters(exporters map[string]MetricsExporter, pipelines map[string]*MetricsPipeline) ([]otel.Exporter, map[string]otel.Exporter, error) {
+	exporterList := []otel.Exporter{}
+	exporterMap := make(map[string]otel.Exporter)
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
 		for _, eID := range p.ExporterIDs {
-			exporter, ok := exporters[eID]
-			if !ok {
+			if _, ok := exporterMap[eID]; ok {
 				continue
 			}
-			switch exporter.Type() {
-			case "google_cloud_monitoring":
-				if _, ok := exportNameMap[eID]; !ok {
+			if exporter, ok := exporters[eID]; ok {
+				switch exporter.Type() {
+				case "google_cloud_monitoring":
 					stackdriver := otel.Stackdriver{
 						StackdriverID: "googlecloud/" + eID,
 						Prefix:        "agent.googleapis.com/",
 					}
-					stackdriverList = append(stackdriverList, &stackdriver)
-					exportNameMap[eID] = "googlecloud/" + eID
+					exporterList = append(exporterList, &stackdriver)
+					exporterMap[eID] = &stackdriver
 				}
 			}
 		}
 	}
-	return stackdriverList, exportNameMap, nil
+	return exporterList, exporterMap, nil
 }
 
-func generateOtelProcessors(processors map[string]MetricsProcessor, pipelines map[string]*MetricsPipeline) ([]*otel.ExcludeMetrics, map[string]string, error) {
-	excludeMetricsList := []*otel.ExcludeMetrics{}
-	processorNameMap := make(map[string]string)
+func generateOtelProcessors(processors map[string]MetricsProcessor, pipelines map[string]*MetricsPipeline) ([]otel.Processor, map[string]otel.Processor, error) {
+	processorList := []otel.Processor{}
+	processorMap := make(map[string]otel.Processor)
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
 		for _, processorID := range p.ProcessorIDs {
-			if _, ok := processorNameMap[processorID]; ok {
+			if _, ok := processorMap[processorID]; ok {
 				continue
 			}
 			if p, ok := processors[processorID]; ok {
@@ -191,20 +190,20 @@ func generateOtelProcessors(processors map[string]MetricsProcessor, pipelines ma
 						}
 						metricNames = append(metricNames, fmt.Sprintf(`^%s$`, strings.Join(literals, `.*`)))
 					}
-					processorNameMap[processorID] = "filter/exclude_" + processorID
 					excludeMetrics := otel.ExcludeMetrics{
-						ExcludeMetricsID: processorNameMap[processorID],
+						ExcludeMetricsID: "filter/exclude_" + processorID,
 						MetricNames:      metricNames,
 					}
-					excludeMetricsList = append(excludeMetricsList, &excludeMetrics)
+					processorList = append(processorList, &excludeMetrics)
+					processorMap[processorID] = &excludeMetrics
 				}
 			}
 		}
 	}
-	return excludeMetricsList, processorNameMap, nil
+	return processorList, processorMap, nil
 }
 
-func generateOtelServices(receiverMap map[string]otel.Receiver, exporterNameMap map[string]string, processorNameMap map[string]string, pipelines map[string]*MetricsPipeline) ([]*otel.Service, error) {
+func generateOtelServices(receiverMap map[string]otel.Receiver, exporterMap map[string]otel.Exporter, processorMap map[string]otel.Processor, pipelines map[string]*MetricsPipeline) ([]*otel.Service, error) {
 	serviceList := []*otel.Service{}
 	for _, pID := range sortedKeys(pipelines) {
 		p := pipelines[pID]
@@ -217,12 +216,12 @@ func generateOtelServices(receiverMap map[string]otel.Receiver, exporterNameMap
 			var processorIDs []string
 			processorIDs = append(processorIDs, r.DefaultProcessors()...)
 			for _, processorID := range p.ProcessorIDs {
-				processorIDs = append(processorIDs, processorNameMap[processorID])
+				processorIDs = append(processorIDs, processorMap[processorID].GetID())
 			}
 
 			var pExportIDs []string
 			for _, eID := range p.ExporterIDs {
-				pExportIDs = append(pExportIDs, exporterNameMap[eID])
+				pExportIDs = append(pExportIDs, exporterMap[eID].GetID())
 			}
 			service := otel.Service{
 				ID:         r.DefaultPipelineID(),
diff --git a/confgenerator/otel/conf.go b/confgenerator/otel/conf.go
index a8e92d19d2..ecc8094821 100644
--- a/confgenerator/otel/conf.go
+++ b/confgenerator/otel/conf.go
@@ -28,12 +28,12 @@ var confTemplate = template.Must(template.New("conf").Parse(
 {{end}}
 processors:
   {{template "defaultprocessor" .}}
-{{- range .ExcludeMetrics}}
-  {{template "excludemetrics" .}}
+{{- range .Processors}}
+  {{.Generate -}}
 {{- end}}
 exporters:
-{{- range .Stackdriver}}
-  {{template "stackdriver" .}}
+{{- range .Exporters}}
+  {{.Generate -}}
 {{- end}}
 extensions:
 service:
@@ -663,17 +663,51 @@ func (r *MSSQL) DefaultPipelineID() string {
 	return "mssql"
 }
 
+type Processor interface {
+	Generate() (string, error)
+	GetID() string
+}
+
 type ExcludeMetrics struct {
 	ExcludeMetricsID string
 	MetricNames      []string
 }
 
+func (p *ExcludeMetrics) Generate() (string, error) {
+	var builder strings.Builder
+	if err := confTemplate.ExecuteTemplate(&builder, "excludemetrics", p); err != nil {
+		return "", err
+	}
+	return builder.String(), nil
+}
+
+func (p *ExcludeMetrics) GetID() string {
+	return p.ExcludeMetricsID
+}
+
+type Exporter interface {
+	Generate() (string, error)
+	GetID() string
+}
+
 type Stackdriver struct {
 	StackdriverID string
 	UserAgent     string
 	Prefix        string
 }
 
+func (e *Stackdriver) Generate() (string, error) {
+	var builder strings.Builder
+	if err := confTemplate.ExecuteTemplate(&builder, "stackdriver", e); err != nil {
+		return "", err
+	}
+	return builder.String(), nil
+}
+
+func (e *Stackdriver) GetID() string {
+	return e.StackdriverID
+}
+
 type Service struct {
 	ID         string
 	Processors string
@@ -682,23 +716,24 @@ type Service struct {
 }
 
 type Config struct {
-	Receivers      []Receiver
-	ExcludeMetrics []*ExcludeMetrics
-	Stackdriver    []*Stackdriver
-	Service        []*Service
-	UserAgent      string
-	Version        string
-	Windows        bool
+	Receivers  []Receiver
+	Processors []Processor
+	Exporters  []Exporter
+	Service    []*Service
+	UserAgent  string
+	Version    string
+	Windows    bool
 }
 
 func (c Config) Generate() (string, error) {
-	c.Stackdriver = append(c.Stackdriver, &Stackdriver{
+	c.Exporters = append(c.Exporters, &Stackdriver{
 		StackdriverID: "googlecloud/agent",
 		Prefix:        "agent.googleapis.com/",
 		UserAgent:     c.UserAgent,
 	})
 
-	for _, s := range c.Stackdriver {
+	for _, s := range c.Exporters {
+		s := s.(*Stackdriver)
 		s.UserAgent = c.UserAgent
 	}
 
diff --git a/confgenerator/otel/conf_test.go b/confgenerator/otel/conf_test.go
index 19e84f2ce2..64bab8efd0 100644
--- a/confgenerator/otel/conf_test.go
+++ b/confgenerator/otel/conf_test.go
@@ -140,11 +140,11 @@ func TestSection(t *testing.T) {
 
 func TestGenerateOtelConfig(t *testing.T) {
 	tests := []struct {
-		name            string
-		receiverList    []Receiver
-		stackdriverList []*Stackdriver
-		serviceList     []*Service
-		want            string
+		name         string
+		receiverList []Receiver
+		exporterList []Exporter
+		serviceList  []*Service
+		want         string
 	}{
 		{
 			name: "default system metrics config",
@@ -162,11 +162,13 @@ func TestGenerateOtelConfig(t *testing.T) {
 					CollectionInterval: "60s",
 				},
 			},
-			stackdriverList: []*Stackdriver{{
-				StackdriverID: "googlecloud/google",
-				UserAgent:     "$IGNORED_VALUE",
-				Prefix:        "agent.googleapis.com/",
-			}},
+			exporterList: []Exporter{
+				&Stackdriver{
+					StackdriverID: "googlecloud/google",
+					UserAgent:     "$IGNORED_VALUE",
+					Prefix:        "agent.googleapis.com/",
+				},
+			},
 			serviceList: []*Service{
 				{
 					ID:         "system",
@@ -715,9 +717,9 @@ service:
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			got, err := Config{
-				Receivers:   tc.receiverList,
-				Stackdriver: tc.stackdriverList,
-				Service:     tc.serviceList,
+				Receivers: tc.receiverList,
+				Exporters: tc.exporterList,
+				Service:   tc.serviceList,
 
 				UserAgent: "Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)",
 				Version:   "google-cloud-ops-agent-metrics/latest-build_distro",

From ccd3a5de3cd096f9f424f1df73493af98d179a82 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 5 Aug 2021 13:02:59 -0400
Subject: [PATCH 29/41] Use lists instead of strings in otel.Service
 representation.

---
 confgenerator/confgenerator.go  |  6 +++---
 confgenerator/otel/conf.go      | 17 +++++++++++------
 confgenerator/otel/conf_test.go | 24 ++++++++++++------------
 3 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index f0576fd5a0..82a079cc62 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -225,9 +225,9 @@ func generateOtelServices(receiverMap map[string]otel.Receiver, exporterMap map[
 			}
 			service := otel.Service{
 				ID:         r.DefaultPipelineID(),
-				Receivers:  fmt.Sprintf("[%s]", r.GetID()),
-				Processors: fmt.Sprintf("[%s]", strings.Join(processorIDs, ",")),
-				Exporters:  fmt.Sprintf("[%s]", strings.Join(pExportIDs, ",")),
+				Receivers:  []string{r.GetID()},
+				Processors: processorIDs,
+				Exporters:  pExportIDs,
 			}
 			serviceList = append(serviceList, &service)
 		}
diff --git a/confgenerator/otel/conf.go b/confgenerator/otel/conf.go
index ecc8094821..626af21d67 100644
--- a/confgenerator/otel/conf.go
+++ b/confgenerator/otel/conf.go
@@ -16,6 +16,7 @@
 package otel
 
 import (
+	"fmt"
 	"strings"
 	"text/template"
 )
@@ -135,9 +136,9 @@ service:
 
 {{define "service" -}}
     metrics/{{.ID}}:
-      receivers:  {{.Receivers}}
-      processors: {{.Processors}}
-      exporters: {{.Exporters}}
+      receivers:  {{.Join .Receivers}}
+      processors: {{.Join .Processors}}
+      exporters: {{.Join .Exporters}}
 {{- end -}}
 
 {{define "defaultprocessor" -}}
@@ -710,9 +711,13 @@ func (e *Stackdriver) GetID() string {
 
 type Service struct {
 	ID         string
-	Processors string
-	Receivers  string
-	Exporters  string
+	Processors []string
+	Receivers  []string
+	Exporters  []string
+}
+
+func (s Service) Join(l []string) string {
+	return fmt.Sprintf("[%s]", strings.Join(l, ","))
 }
 
 type Config struct {
diff --git a/confgenerator/otel/conf_test.go b/confgenerator/otel/conf_test.go
index 64bab8efd0..c81bca6f8d 100644
--- a/confgenerator/otel/conf_test.go
+++ b/confgenerator/otel/conf_test.go
@@ -101,9 +101,9 @@ func TestSection(t *testing.T) {
 		{
 			section: Service{
 				ID:         "system",
-				Processors: "[agentmetrics/system,filter/system,metricstransform/system,resourcedetection]",
-				Receivers:  "[hostmetrics/hostmetrics]",
-				Exporters:  "[googlecloud/google]",
+				Processors: []string{"agentmetrics/system", "filter/system", "metricstransform/system", "resourcedetection"},
+				Receivers:  []string{"hostmetrics/hostmetrics"},
+				Exporters:  []string{"googlecloud/google"},
 			},
 			want: `metrics/system:
       receivers:  [hostmetrics/hostmetrics]
@@ -172,21 +172,21 @@ func TestGenerateOtelConfig(t *testing.T) {
 			serviceList: []*Service{
 				{
 					ID:         "system",
-					Receivers:  "[hostmetrics/hostmetrics]",
-					Processors: "[agentmetrics/system,filter/system,metricstransform/system,resourcedetection]",
-					Exporters:  "[googlecloud/google]",
+					Receivers:  []string{"hostmetrics/hostmetrics"},
+					Processors: []string{"agentmetrics/system", "filter/system", "metricstransform/system", "resourcedetection"},
+					Exporters:  []string{"googlecloud/google"},
 				},
 				{
 					ID:         "mssql",
-					Receivers:  "[windowsperfcounters/mssql_mssql]",
-					Processors: "[metricstransform/mssql,resourcedetection]",
-					Exporters:  "[googlecloud/google]",
+					Receivers:  []string{"windowsperfcounters/mssql_mssql"},
+					Processors: []string{"metricstransform/mssql", "resourcedetection"},
+					Exporters:  []string{"googlecloud/google"},
 				},
 				{
 					ID:         "iis",
-					Receivers:  "[windowsperfcounters/iis_iis]",
-					Processors: "[metricstransform/iis,resourcedetection]",
-					Exporters:  "[googlecloud/google]",
+					Receivers:  []string{"windowsperfcounters/iis_iis"},
+					Processors: []string{"metricstransform/iis", "resourcedetection"},
+					Exporters:  []string{"googlecloud/google"},
 				},
 			},
 			want: `receivers:

From 2a23768a8c38240137e046f294b703c7e5342932 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 5 Aug 2021 16:25:13 -0400
Subject: [PATCH 30/41] Reorder fluentbit templates to match the corresponding
 config type.

---
 confgenerator/fluentbit/conf.go | 164 ++++++++++++++++----------------
 1 file changed, 82 insertions(+), 82 deletions(-)

diff --git a/confgenerator/fluentbit/conf.go b/confgenerator/fluentbit/conf.go
index f1a955fa31..f47624a512 100644
--- a/confgenerator/fluentbit/conf.go
+++ b/confgenerator/fluentbit/conf.go
@@ -84,67 +84,6 @@ const (
 {{- range .StackdriverConfigSections -}}
 {{.}}
 
-{{end}}`
-
-	parserConfTemplate = `[PARSER]
-    Name        lib:default_message_parser
-    Format      regex
-    Regex       ^(?<message>.*)$
-
-[PARSER]
-    Name        lib:apache
-    Format      regex
-    Regex       ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
-    Time_Key    time
-    Time_Format %d/%b/%Y:%H:%M:%S %z
-
-[PARSER]
-    Name        lib:apache2
-    Format      regex
-    Regex       ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>.*)")?$
-    Time_Key    time
-    Time_Format %d/%b/%Y:%H:%M:%S %z
-
-[PARSER]
-    Name   lib:apache_error
-    Format regex
-    Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
-
-[PARSER]
-    Name        lib:mongodb
-    Format      regex
-    Regex       ^(?<time>[^ ]*)\s+(?<severity>\w)\s+(?<component>[^ ]+)\s+\[(?<context>[^\]]+)]\s+(?<message>.*?) *(?<ms>(\d+))?(:?ms)?$
-    Time_Key    time
-    Time_Format %Y-%m-%dT%H:%M:%S.%L
-
-[PARSER]
-    Name        lib:nginx
-    Format      regex
-    Regex       ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
-    Time_Key    time
-    Time_Format %d/%b/%Y:%H:%M:%S %z
-
-[PARSER]
-    Name        lib:syslog-rfc5424
-    Format      regex
-    Regex       ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*?)\]|-)) (?<message>.+)$
-    Time_Key    time
-    Time_Format %Y-%m-%dT%H:%M:%S.%L%Z
-
-[PARSER]
-    Name        lib:syslog-rfc3164
-    Format      regex
-    Regex       /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-    Time_Key    time
-    Time_Format %b %d %H:%M:%S
-
-{{range .JSONParserConfigSections -}}
-{{.}}
-
-{{end}}
-{{- range .RegexParserConfigSections -}}
-{{.}}
-
 {{end}}`
 
 	filterModifyAddLogNameConf = `[FILTER]
@@ -170,27 +109,6 @@ const (
     Emitter_Storage.type  filesystem
     Emitter_Mem_Buf_Limit 10M`
 
-	parserJSONConf = `[PARSER]
-    Name        {{.Name}}
-    Format      json
-{{- if (ne .TimeKey "")}}
-    Time_Key    {{.TimeKey}}
-{{- end}}
-{{- if (ne .TimeFormat "")}}
-    Time_Format {{.TimeFormat}}
-{{- end}}`
-
-	parserRegexConf = `[PARSER]
-    Name        {{.Name}}
-    Format      regex
-    Regex       {{.Regex}}
-{{- if (ne .TimeKey "")}}
-    Time_Key    {{.TimeKey}}
-{{- end}}
-{{- if (ne .TimeFormat "")}}
-    Time_Format {{.TimeFormat}}
-{{- end}}`
-
 	tailConf = `[INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
@@ -271,6 +189,88 @@ const (
     tls         On
     # Do not force certificate validation.
     tls.verify  Off`
+
+	parserConfTemplate = `[PARSER]
+    Name        lib:default_message_parser
+    Format      regex
+    Regex       ^(?<message>.*)$
+
+[PARSER]
+    Name        lib:apache
+    Format      regex
+    Regex       ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+    Time_Key    time
+    Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+    Name        lib:apache2
+    Format      regex
+    Regex       ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>.*)")?$
+    Time_Key    time
+    Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+    Name   lib:apache_error
+    Format regex
+    Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
+
+[PARSER]
+    Name        lib:mongodb
+    Format      regex
+    Regex       ^(?<time>[^ ]*)\s+(?<severity>\w)\s+(?<component>[^ ]+)\s+\[(?<context>[^\]]+)]\s+(?<message>.*?) *(?<ms>(\d+))?(:?ms)?$
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L
+
+[PARSER]
+    Name        lib:nginx
+    Format      regex
+    Regex       ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
+    Time_Key    time
+    Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+    Name        lib:syslog-rfc5424
+    Format      regex
+    Regex       ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*?)\]|-)) (?<message>.+)$
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L%Z
+
+[PARSER]
+    Name        lib:syslog-rfc3164
+    Format      regex
+    Regex       /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+    Time_Key    time
+    Time_Format %b %d %H:%M:%S
+
+{{range .JSONParserConfigSections -}}
+{{.}}
+
+{{end}}
+{{- range .RegexParserConfigSections -}}
+{{.}}
+
+{{end}}`
+
+	parserJSONConf = `[PARSER]
+    Name        {{.Name}}
+    Format      json
+{{- if (ne .TimeKey "")}}
+    Time_Key    {{.TimeKey}}
+{{- end}}
+{{- if (ne .TimeFormat "")}}
+    Time_Format {{.TimeFormat}}
+{{- end}}`
+
+	parserRegexConf = `[PARSER]
+    Name        {{.Name}}
+    Format      regex
+    Regex       {{.Regex}}
+{{- if (ne .TimeKey "")}}
+    Time_Key    {{.TimeKey}}
+{{- end}}
+{{- if (ne .TimeFormat "")}}
+    Time_Format {{.TimeFormat}}
+{{- end}}`
 )
 
 type mainConfigSections struct {

From 211673a55e51e67a9a720b4c3aa4376c4db14cb1 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 5 Aug 2021 16:54:21 -0400
Subject: [PATCH 31/41] Render fluentbit config from a single set of templates.

---
 confgenerator/fluentbit/conf.go | 137 ++++++++++++++------------------
 1 file changed, 60 insertions(+), 77 deletions(-)

diff --git a/confgenerator/fluentbit/conf.go b/confgenerator/fluentbit/conf.go
index f47624a512..4f773eb1a2 100644
--- a/confgenerator/fluentbit/conf.go
+++ b/confgenerator/fluentbit/conf.go
@@ -23,8 +23,7 @@ import (
 	"text/template"
 )
 
-const (
-	mainConfTemplate = `[SERVICE]
+var mainConfTemplate = template.Must(template.New("fluentBitMainConf").Parse(`[SERVICE]
     # https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file#config_section
     # Flush logs every 1 second, even if the buffer is not full to minimize log entry arrival delay.
     Flush      1
@@ -84,32 +83,36 @@ const (
 {{- range .StackdriverConfigSections -}}
 {{.}}
 
-{{end}}`
-
-	filterModifyAddLogNameConf = `[FILTER]
+{{end}}
+{{- define "filter_modify_add_log_name" -}}
+[FILTER]
     Name  modify
     Match {{.Match}}
-    Add   logName {{.LogName}}`
-
-	filterModifyRemoveLogNameConf = `[FILTER]
+    Add   logName {{.LogName}}
+{{- end -}}
+{{- define "filter_modify_remove_log_name" -}}
+[FILTER]
     Name   modify
     Match  {{.Match}}
-    Remove logName`
-
-	filterParserConf = `[FILTER]
+    Remove logName
+{{- end -}}
+{{- define "filter_parser" -}}
+[FILTER]
     Name     parser
     Match    {{.Match}}
     Key_Name {{.KeyName}}
-    Parser   {{.Parser}}`
-
-	filterRewriteTagConf = `[FILTER]
+    Parser   {{.Parser}}
+{{- end -}}
+{{- define "filter_rewrite_tag" -}}
+[FILTER]
     Name                  rewrite_tag
     Match                 {{.Match}}
     Rule                  $logName .* $logName false
     Emitter_Storage.type  filesystem
-    Emitter_Mem_Buf_Limit 10M`
-
-	tailConf = `[INPUT]
+    Emitter_Mem_Buf_Limit 10M
+{{- end -}}
+{{- define "tail" -}}
+[INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
     DB                 {{.DB}}
@@ -140,9 +143,10 @@ const (
     # This is used to deal with backpressure scenarios (e.g: cannot flush data for some reason).
     # When the input plugin hits "mem_buf_limit", because we have enabled filesystem storage type, mem_buf_limit acts
     # as a hint to set "how much data can be up in memory", once the limit is reached it continues writing to disk.
-    Mem_Buf_Limit      10M`
-
-	syslogConf = `[INPUT]
+    Mem_Buf_Limit      10M
+{{- end -}}
+{{- define "syslog" -}}
+[INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
     Mode           {{.Mode}}
@@ -160,17 +164,19 @@ const (
     # This is used to deal with backpressure scenarios (e.g: cannot flush data for some reason).
     # When the input plugin hits "mem_buf_limit", because we have enabled filesystem storage type, mem_buf_limit acts
     # as a hint to set "how much data can be up in memory", once the limit is reached it continues writing to disk.
-    Mem_Buf_Limit  10M`
-
-	wineventlogConf = `[INPUT]
+    Mem_Buf_Limit  10M
+{{- end -}}
+{{- define "wineventlog" -}}
+[INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
     Tag            {{.Tag}}
     Name           winlog
     Channels       {{.Channels}}
     Interval_Sec   1
-    DB             {{.DB}}`
-
-	stackdriverConf = `[OUTPUT]
+    DB             {{.DB}}
+{{- end -}}
+{{- define "stackdriver" -}}
+[OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
     resource          gce_instance
@@ -188,9 +194,11 @@ const (
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off`
+    tls.verify  Off
+{{- end -}}
+`))
 
-	parserConfTemplate = `[PARSER]
+var parserConfTemplate = template.Must(template.New("fluentBitParserConf").Parse(`[PARSER]
     Name        lib:default_message_parser
     Format      regex
     Regex       ^(?<message>.*)$
@@ -249,9 +257,9 @@ const (
 {{- range .RegexParserConfigSections -}}
 {{.}}
 
-{{end}}`
-
-	parserJSONConf = `[PARSER]
+{{end}}
+{{- define "parserJSON" -}}
+[PARSER]
     Name        {{.Name}}
     Format      json
 {{- if (ne .TimeKey "")}}
@@ -259,9 +267,10 @@ const (
 {{- end}}
 {{- if (ne .TimeFormat "")}}
     Time_Format {{.TimeFormat}}
-{{- end}}`
-
-	parserRegexConf = `[PARSER]
+{{- end}}
+{{- end -}}
+{{- define "parserRegex" -}}
+[PARSER]
     Name        {{.Name}}
     Format      regex
     Regex       {{.Regex}}
@@ -270,8 +279,9 @@ const (
 {{- end}}
 {{- if (ne .TimeFormat "")}}
     Time_Format {{.TimeFormat}}
-{{- end}}`
-)
+{{- end}}
+{{- end -}}
+`))
 
 type mainConfigSections struct {
 	TailConfigSections                      []string
@@ -385,13 +395,9 @@ func GenerateFluentBitMainConfig(tails []*Tail, syslogs []*Syslog, wineventlogs
 		FilterModifyRemoveLogNameConfigSections: filterModifyRemoveLogNameConfigSections,
 		StackdriverConfigSections:               stackdriverConfigSections,
 	}
-	mt, err := template.New("fluentBitMainConf").Parse(mainConfTemplate)
-	if err != nil {
-		return "", err
-	}
 
 	var mainConfigBuilder strings.Builder
-	if err := mt.Execute(&mainConfigBuilder, configSections); err != nil {
+	if err := mainConfTemplate.Execute(&mainConfigBuilder, configSections); err != nil {
 		return "", err
 	}
 	return mainConfigBuilder.String(), nil
@@ -417,16 +423,13 @@ func GenerateFluentBitParserConfig(jsonParsers []*ParserJSON, regexParsers []*Pa
 		regexParserConfigSections = append(regexParserConfigSections, configSection)
 	}
 
-	var parserConfigBuilder strings.Builder
-	parserTemplate, err := template.New("fluentBitParserConf").Parse(parserConfTemplate)
-	if err != nil {
-		return "", err
-	}
 	parsers := parserConfigSections{
 		JSONParserConfigSections:  jsonParserConfigSections,
 		RegexParserConfigSections: regexParserConfigSections,
 	}
-	if err := parserTemplate.Execute(&parserConfigBuilder, parsers); err != nil {
+
+	var parserConfigBuilder strings.Builder
+	if err := parserConfTemplate.Execute(&parserConfigBuilder, parsers); err != nil {
 		return "", err
 	}
 	return parserConfigBuilder.String(), nil
@@ -451,8 +454,6 @@ type FilterParser struct {
 	Parser  string
 }
 
-var filterParserTemplate = template.Must(template.New("filter_parser").Parse(filterParserConf))
-
 func (f FilterParser) renderConfig() (string, error) {
 	if f.Match == "" {
 		return "", emptyFieldErr{
@@ -473,7 +474,7 @@ func (f FilterParser) renderConfig() (string, error) {
 		}
 	}
 	var renderedFilterParserConfig strings.Builder
-	if err := filterParserTemplate.Execute(&renderedFilterParserConfig, f); err != nil {
+	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterParserConfig, "filter_parser", f); err != nil {
 		return "", err
 	}
 	return renderedFilterParserConfig.String(), nil
@@ -486,8 +487,6 @@ type FilterModifyAddLogName struct {
 	LogName string
 }
 
-var filterModifyAddLogNameTemplate = template.Must(template.New("filter_modify_add_log_name").Parse(filterModifyAddLogNameConf))
-
 func (f FilterModifyAddLogName) renderConfig() (string, error) {
 	if f.Match == "" {
 		return "", emptyFieldErr{
@@ -502,7 +501,7 @@ func (f FilterModifyAddLogName) renderConfig() (string, error) {
 		}
 	}
 	var renderedFilterModifyAddLogNameConfig strings.Builder
-	if err := filterModifyAddLogNameTemplate.Execute(&renderedFilterModifyAddLogNameConfig, f); err != nil {
+	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterModifyAddLogNameConfig, "filter_modify_add_log_name", f); err != nil {
 		return "", err
 	}
 	return renderedFilterModifyAddLogNameConfig.String(), nil
@@ -514,8 +513,6 @@ type FilterModifyRemoveLogName struct {
 	Match string
 }
 
-var filterModifyRemoveLogNameTemplate = template.Must(template.New("filter_modify_remove_log_name").Parse(filterModifyRemoveLogNameConf))
-
 func (f FilterModifyRemoveLogName) renderConfig() (string, error) {
 	if f.Match == "" {
 		return "", emptyFieldErr{
@@ -524,7 +521,7 @@ func (f FilterModifyRemoveLogName) renderConfig() (string, error) {
 		}
 	}
 	var renderedFilterModifyRemoveLogNameConfig strings.Builder
-	if err := filterModifyRemoveLogNameTemplate.Execute(&renderedFilterModifyRemoveLogNameConfig, f); err != nil {
+	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterModifyRemoveLogNameConfig, "filter_modify_remove_log_name", f); err != nil {
 		return "", err
 	}
 	return renderedFilterModifyRemoveLogNameConfig.String(), nil
@@ -535,8 +532,6 @@ type FilterRewriteTag struct {
 	Match string
 }
 
-var filterRewriteTagTemplate = template.Must(template.New("filter_rewrite_tag").Parse(filterRewriteTagConf))
-
 func (f FilterRewriteTag) renderConfig() (string, error) {
 	if f.Match == "" {
 		return "", emptyFieldErr{
@@ -545,7 +540,7 @@ func (f FilterRewriteTag) renderConfig() (string, error) {
 		}
 	}
 	var renderedFilterRewriteTagConfig strings.Builder
-	if err := filterRewriteTagTemplate.Execute(&renderedFilterRewriteTagConfig, f); err != nil {
+	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterRewriteTagConfig, "filter_rewrite_tag", f); err != nil {
 		return "", err
 	}
 	return renderedFilterRewriteTagConfig.String(), nil
@@ -558,8 +553,6 @@ type ParserJSON struct {
 	TimeFormat string
 }
 
-var parserJSONTemplate = template.Must(template.New("parserJSON").Parse(parserJSONConf))
-
 // renderConfig generates a section for configure fluentBit JSON parser.
 func (p ParserJSON) renderConfig() (string, error) {
 	if p.Name == "" {
@@ -569,7 +562,7 @@ func (p ParserJSON) renderConfig() (string, error) {
 		}
 	}
 	var b strings.Builder
-	if err := parserJSONTemplate.Execute(&b, p); err != nil {
+	if err := parserConfTemplate.ExecuteTemplate(&b, "parserJSON", p); err != nil {
 		return "", err
 	}
 	return b.String(), nil
@@ -583,8 +576,6 @@ type ParserRegex struct {
 	TimeFormat string
 }
 
-var parserRegexTemplate = template.Must(template.New("parserRegex").Parse(parserRegexConf))
-
 // renderConfig generates a section for configure fluentBit Regex parser.
 func (p ParserRegex) renderConfig() (string, error) {
 	if p.Name == "" {
@@ -600,7 +591,7 @@ func (p ParserRegex) renderConfig() (string, error) {
 		}
 	}
 	var b strings.Builder
-	if err := parserRegexTemplate.Execute(&b, p); err != nil {
+	if err := parserConfTemplate.ExecuteTemplate(&b, "parserRegex", p); err != nil {
 		return "", err
 	}
 	return b.String(), nil
@@ -636,14 +627,12 @@ func (t Tail) renderConfig() (string, error) {
 		}
 	}
 	var renderedTailConfig strings.Builder
-	if err := tailTemplate.Execute(&renderedTailConfig, t); err != nil {
+	if err := mainConfTemplate.ExecuteTemplate(&renderedTailConfig, "tail", t); err != nil {
 		return "", err
 	}
 	return renderedTailConfig.String(), nil
 }
 
-var tailTemplate = template.Must(template.New("tail").Parse(tailConf))
-
 type invalidValueErr struct {
 	plugin                string
 	field                 string
@@ -662,8 +651,6 @@ type Syslog struct {
 	Tag    string
 }
 
-var syslogTemplate = template.Must(template.New("syslog").Parse(syslogConf))
-
 // renderConfig generates a section for configure fluentBit syslog input plugin.
 func (s Syslog) renderConfig() (string, error) {
 	switch m := s.Mode; m {
@@ -699,7 +686,7 @@ func (s Syslog) renderConfig() (string, error) {
 	}
 
 	var renderedSyslogConfig strings.Builder
-	if err := syslogTemplate.Execute(&renderedSyslogConfig, s); err != nil {
+	if err := mainConfTemplate.ExecuteTemplate(&renderedSyslogConfig, "syslog", s); err != nil {
 		return "", err
 	}
 	return renderedSyslogConfig.String(), nil
@@ -713,8 +700,6 @@ type WindowsEventlog struct {
 	DB           string
 }
 
-var wineventlogTemplate = template.Must(template.New("wineventlog").Parse(wineventlogConf))
-
 // renderConfig generates a section for configure fluentBit wineventlog input plugin.
 func (w WindowsEventlog) renderConfig() (string, error) {
 	//TODO: Add check that Channels is a comma separated list.
@@ -733,7 +718,7 @@ func (w WindowsEventlog) renderConfig() (string, error) {
 	}
 
 	var renderedWineventlogConfig strings.Builder
-	if err := wineventlogTemplate.Execute(&renderedWineventlogConfig, w); err != nil {
+	if err := mainConfTemplate.ExecuteTemplate(&renderedWineventlogConfig, "wineventlog", w); err != nil {
 		return "", err
 	}
 	return renderedWineventlogConfig.String(), nil
@@ -746,8 +731,6 @@ type Stackdriver struct {
 	Workers   int
 }
 
-var stackdriverTemplate = template.Must(template.New("stackdriver").Parse(stackdriverConf))
-
 // renderConfig generates a section for configure fluentBit stackdriver output plugin.
 func (s Stackdriver) renderConfig() (string, error) {
 	if s.Match == "" {
@@ -764,7 +747,7 @@ func (s Stackdriver) renderConfig() (string, error) {
 	}
 
 	var renderedStackdriverConfig strings.Builder
-	if err := stackdriverTemplate.Execute(&renderedStackdriverConfig, s); err != nil {
+	if err := mainConfTemplate.ExecuteTemplate(&renderedStackdriverConfig, "stackdriver", s); err != nil {
 		return "", err
 	}
 	return renderedStackdriverConfig.String(), nil

From 084affa2c6f0f7fb5f4db5bdd250326d2171adbe Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 5 Aug 2021 17:54:07 -0400
Subject: [PATCH 32/41] Use fluentbit.Config structs to generate configuration.

---
 confgenerator/confgenerator.go       | 19 ++++++---
 confgenerator/fluentbit/conf.go      | 60 +++++++++++++++++++---------
 confgenerator/fluentbit/conf_test.go | 27 ++++++++++---
 3 files changed, 77 insertions(+), 29 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 82a079cc62..688f5fd031 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -286,11 +286,20 @@ func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir strin
 			return "", "", err
 		}
 	}
-	mainConfig, err := fluentbit.GenerateFluentBitMainConfig(fbTails, fbSyslogs, fbWinEventlogs, fbFilterParserGroups, fbFilterAddLogNames, fbFilterRewriteTags, fbFilterRemoveLogNames, fbStackdrivers, userAgent)
-	if err != nil {
-		return "", "", err
-	}
-	parserConfig, err := fluentbit.GenerateFluentBitParserConfig(jsonParsers, regexParsers)
+	mainConfig, parserConfig, err := fluentbit.Config{
+		Tails:                      fbTails,
+		Syslogs:                    fbSyslogs,
+		Wineventlogs:               fbWinEventlogs,
+		FilterParserGroups:         fbFilterParserGroups,
+		FilterModifyAddLogNames:    fbFilterAddLogNames,
+		FilterModifyRemoveLogNames: fbFilterRemoveLogNames,
+		FilterRewriteTags:          fbFilterRewriteTags,
+		Stackdrivers:               fbStackdrivers,
+		JsonParsers:                jsonParsers,
+		RegexParsers:               regexParsers,
+
+		UserAgent: userAgent,
+	}.Generate()
 	if err != nil {
 		return "", "", err
 	}
diff --git a/confgenerator/fluentbit/conf.go b/confgenerator/fluentbit/conf.go
index 4f773eb1a2..bc0d5fa5eb 100644
--- a/confgenerator/fluentbit/conf.go
+++ b/confgenerator/fluentbit/conf.go
@@ -299,12 +299,21 @@ type parserConfigSections struct {
 	RegexParserConfigSections []string
 }
 
-// GenerateFluentBitMainConfig generates a FluentBit main configuration.
-func GenerateFluentBitMainConfig(tails []*Tail, syslogs []*Syslog, wineventlogs []*WindowsEventlog, filterParserGroups []FilterParserGroup,
-	filterModifyAddLogNames []*FilterModifyAddLogName,
-	filterRewriteTags []*FilterRewriteTag,
-	filterModifyRemoveLogNames []*FilterModifyRemoveLogName,
-	stackdrivers []*Stackdriver, userAgent string) (string, error) {
+type Config struct {
+	Tails                      []*Tail
+	Syslogs                    []*Syslog
+	Wineventlogs               []*WindowsEventlog
+	FilterParserGroups         []FilterParserGroup
+	FilterModifyAddLogNames    []*FilterModifyAddLogName
+	FilterRewriteTags          []*FilterRewriteTag
+	FilterModifyRemoveLogNames []*FilterModifyRemoveLogName
+	Stackdrivers               []*Stackdriver
+	JsonParsers                []*ParserJSON
+	RegexParsers               []*ParserRegex
+	UserAgent                  string
+}
+
+func (c Config) generateMain() (string, error) {
 	tailConfigSections := []string{}
 	syslogConfigSections := []string{}
 	wineventlogConfigSections := []string{}
@@ -313,28 +322,28 @@ func GenerateFluentBitMainConfig(tails []*Tail, syslogs []*Syslog, wineventlogs
 	filterRewriteTagSections := []string{}
 	filterModifyRemoveLogNameConfigSections := []string{}
 	stackdriverConfigSections := []string{}
-	for _, t := range tails {
+	for _, t := range c.Tails {
 		configSection, err := t.renderConfig()
 		if err != nil {
 			return "", err
 		}
 		tailConfigSections = append(tailConfigSections, configSection)
 	}
-	for _, s := range syslogs {
+	for _, s := range c.Syslogs {
 		configSection, err := s.renderConfig()
 		if err != nil {
 			return "", err
 		}
 		syslogConfigSections = append(syslogConfigSections, configSection)
 	}
-	for _, w := range wineventlogs {
+	for _, w := range c.Wineventlogs {
 		configSection, err := w.renderConfig()
 		if err != nil {
 			return "", err
 		}
 		wineventlogConfigSections = append(wineventlogConfigSections, configSection)
 	}
-	for _, filterParsers := range filterParserGroups {
+	for _, filterParsers := range c.FilterParserGroups {
 		var filters []string
 		for _, f := range filterParsers {
 			configSection, err := f.renderConfig()
@@ -345,29 +354,29 @@ func GenerateFluentBitMainConfig(tails []*Tail, syslogs []*Syslog, wineventlogs
 		}
 		filterParserConfigSections = append(filterParserConfigSections, strings.Join(filters, "\n\n"))
 	}
-	for _, f := range filterModifyAddLogNames {
+	for _, f := range c.FilterModifyAddLogNames {
 		configSection, err := f.renderConfig()
 		if err != nil {
 			return "", err
 		}
 		filterModifyAddLogNameConfigSections = append(filterModifyAddLogNameConfigSections, configSection)
 	}
-	for _, f := range filterRewriteTags {
+	for _, f := range c.FilterRewriteTags {
 		configSection, err := f.renderConfig()
 		if err != nil {
 			return "", err
 		}
 		filterRewriteTagSections = append(filterRewriteTagSections, configSection)
 	}
-	for _, f := range filterModifyRemoveLogNames {
+	for _, f := range c.FilterModifyRemoveLogNames {
 		configSection, err := f.renderConfig()
 		if err != nil {
 			return "", err
 		}
 		filterModifyRemoveLogNameConfigSections = append(filterModifyRemoveLogNameConfigSections, configSection)
 	}
-	for _, s := range stackdrivers {
-		s.UserAgent = userAgent
+	for _, s := range c.Stackdrivers {
+		s.UserAgent = c.UserAgent
 		configSection, err := s.renderConfig()
 		if err != nil {
 			return "", err
@@ -403,10 +412,9 @@ func GenerateFluentBitMainConfig(tails []*Tail, syslogs []*Syslog, wineventlogs
 	return mainConfigBuilder.String(), nil
 }
 
-// GenerateFluentBitParserConfig generates a FluentBit parser configuration.
-func GenerateFluentBitParserConfig(jsonParsers []*ParserJSON, regexParsers []*ParserRegex) (string, error) {
+func (c Config) generateParser() (string, error) {
 	jsonParserConfigSections := []string{}
-	for _, j := range jsonParsers {
+	for _, j := range c.JsonParsers {
 		configSection, err := j.renderConfig()
 		if err != nil {
 			return "", err
@@ -415,7 +423,7 @@ func GenerateFluentBitParserConfig(jsonParsers []*ParserJSON, regexParsers []*Pa
 	}
 
 	regexParserConfigSections := []string{}
-	for _, r := range regexParsers {
+	for _, r := range c.RegexParsers {
 		configSection, err := r.renderConfig()
 		if err != nil {
 			return "", err
@@ -435,6 +443,20 @@ func GenerateFluentBitParserConfig(jsonParsers []*ParserJSON, regexParsers []*Pa
 	return parserConfigBuilder.String(), nil
 }
 
+func (c Config) Generate() (mainConfig string, parserConfig string, err error) {
+	mainConfig, err = c.generateMain()
+	if err != nil {
+		return "", "", err
+	}
+
+	parserConfig, err = c.generateParser()
+	if err != nil {
+		return "", "", err
+	}
+
+	return mainConfig, parserConfig, nil
+}
+
 type emptyFieldErr struct {
 	plugin string
 	field  string
diff --git a/confgenerator/fluentbit/conf_test.go b/confgenerator/fluentbit/conf_test.go
index fc7a9166d4..3952e6e8d2 100644
--- a/confgenerator/fluentbit/conf_test.go
+++ b/confgenerator/fluentbit/conf_test.go
@@ -830,7 +830,10 @@ func TestGenerateFluentBitMainConfig(t *testing.T) {
 		},
 	}
 	for _, tc := range tests {
-		got, err := GenerateFluentBitMainConfig(tc.tails, tc.syslogs, nil, nil, nil, nil, nil, nil, "")
+		got, _, err := Config{
+			Tails:   tc.tails,
+			Syslogs: tc.syslogs,
+		}.Generate()
 		if err != nil {
 			t.Errorf("got error: %v, want no error", err)
 			return
@@ -867,7 +870,11 @@ func TestGenerateFluentBitMainConfigErrors(t *testing.T) {
 		},
 	}
 	for _, tc := range tests {
-		if _, err := GenerateFluentBitMainConfig(tc.tails, tc.syslogs, nil, nil, nil, nil, nil, nil, ""); err == nil {
+		_, _, err := Config{
+			Tails:   tc.tails,
+			Syslogs: tc.syslogs,
+		}.Generate()
+		if err == nil {
 			t.Errorf("test %q: GenerateFluentBitMainConfig succeeded, want error", tc.name)
 		}
 	}
@@ -1009,7 +1016,10 @@ func TestGenerateFluentBitMainConfigWindows(t *testing.T) {
 		},
 	}
 	for _, tc := range tests {
-		got, err := GenerateFluentBitMainConfig(tc.tails, nil, tc.wineventlogs, nil, nil, nil, nil, nil, "")
+		got, _, err := Config{
+			Tails:        tc.tails,
+			Wineventlogs: tc.wineventlogs,
+		}.Generate()
 		if err != nil {
 			t.Errorf("got error: %v, want no error", err)
 			return
@@ -1181,7 +1191,10 @@ func TestGenerateFluentBitParserConfig(t *testing.T) {
 		},
 	}
 	for _, tc := range tests {
-		got, err := GenerateFluentBitParserConfig(tc.jsonParsers, tc.regexParsers)
+		_, got, err := Config{
+			JsonParsers:  tc.jsonParsers,
+			RegexParsers: tc.regexParsers,
+		}.Generate()
 		if err != nil {
 			t.Errorf("test %q got error: %v, want no error", tc.name, err)
 			return
@@ -1208,7 +1221,11 @@ func TestGenerateFluentBitParserConfigErrors(t *testing.T) {
 		},
 	}
 	for _, tc := range tests {
-		if _, err := GenerateFluentBitParserConfig(tc.jsonParsers, tc.regexParsers); err == nil {
+		_, _, err := Config{
+			JsonParsers:  tc.jsonParsers,
+			RegexParsers: tc.regexParsers,
+		}.Generate()
+		if err == nil {
 			t.Errorf("test %q: GenerateFluentBitParserConfig succeeded, want error", tc.name)
 		}
 	}

From cfe8e4514128cca3a6c0aef6d006b72795bcbdad Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 5 Aug 2021 18:23:26 -0400
Subject: [PATCH 33/41] Remove internal validation from the fluentbit config
 generator, as it replicates what we have in the ops agent config.

---
 confgenerator/fluentbit/conf.go      | 160 ---------------
 confgenerator/fluentbit/conf_test.go | 281 ---------------------------
 2 files changed, 441 deletions(-)

diff --git a/confgenerator/fluentbit/conf.go b/confgenerator/fluentbit/conf.go
index bc0d5fa5eb..aa37a0bf9c 100644
--- a/confgenerator/fluentbit/conf.go
+++ b/confgenerator/fluentbit/conf.go
@@ -16,8 +16,6 @@
 package fluentbit
 
 import (
-	"fmt"
-	"net"
 	"sort"
 	"strings"
 	"text/template"
@@ -457,15 +455,6 @@ func (c Config) Generate() (mainConfig string, parserConfig string, err error) {
 	return mainConfig, parserConfig, nil
 }
 
-type emptyFieldErr struct {
-	plugin string
-	field  string
-}
-
-func (e emptyFieldErr) Error() string {
-	return fmt.Sprintf("%q plugin should not have empty field: %q", e.plugin, e.field)
-}
-
 // A FilterParserGroup represents a list of filters to be applied in order.
 type FilterParserGroup []*FilterParser
 
@@ -477,24 +466,6 @@ type FilterParser struct {
 }
 
 func (f FilterParser) renderConfig() (string, error) {
-	if f.Match == "" {
-		return "", emptyFieldErr{
-			plugin: "filter parser",
-			field:  "Match",
-		}
-	}
-	if f.KeyName == "" {
-		return "", emptyFieldErr{
-			plugin: "filter parser",
-			field:  "KeyName",
-		}
-	}
-	if f.Parser == "" {
-		return "", emptyFieldErr{
-			plugin: "filter parser",
-			field:  "Parser",
-		}
-	}
 	var renderedFilterParserConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterParserConfig, "filter_parser", f); err != nil {
 		return "", err
@@ -510,18 +481,6 @@ type FilterModifyAddLogName struct {
 }
 
 func (f FilterModifyAddLogName) renderConfig() (string, error) {
-	if f.Match == "" {
-		return "", emptyFieldErr{
-			plugin: "filter modify - add logName",
-			field:  "Match",
-		}
-	}
-	if f.LogName == "" {
-		return "", emptyFieldErr{
-			plugin: "filter modify - add logName",
-			field:  "LogName",
-		}
-	}
 	var renderedFilterModifyAddLogNameConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterModifyAddLogNameConfig, "filter_modify_add_log_name", f); err != nil {
 		return "", err
@@ -536,12 +495,6 @@ type FilterModifyRemoveLogName struct {
 }
 
 func (f FilterModifyRemoveLogName) renderConfig() (string, error) {
-	if f.Match == "" {
-		return "", emptyFieldErr{
-			plugin: "filter modify - remove logName",
-			field:  "Match",
-		}
-	}
 	var renderedFilterModifyRemoveLogNameConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterModifyRemoveLogNameConfig, "filter_modify_remove_log_name", f); err != nil {
 		return "", err
@@ -555,12 +508,6 @@ type FilterRewriteTag struct {
 }
 
 func (f FilterRewriteTag) renderConfig() (string, error) {
-	if f.Match == "" {
-		return "", emptyFieldErr{
-			plugin: "filter parser",
-			field:  "Match",
-		}
-	}
 	var renderedFilterRewriteTagConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterRewriteTagConfig, "filter_rewrite_tag", f); err != nil {
 		return "", err
@@ -577,12 +524,6 @@ type ParserJSON struct {
 
 // renderConfig generates a section for configure fluentBit JSON parser.
 func (p ParserJSON) renderConfig() (string, error) {
-	if p.Name == "" {
-		return "", emptyFieldErr{
-			plugin: "json parser",
-			field:  "name",
-		}
-	}
 	var b strings.Builder
 	if err := parserConfTemplate.ExecuteTemplate(&b, "parserJSON", p); err != nil {
 		return "", err
@@ -600,18 +541,6 @@ type ParserRegex struct {
 
 // renderConfig generates a section for configure fluentBit Regex parser.
 func (p ParserRegex) renderConfig() (string, error) {
-	if p.Name == "" {
-		return "", emptyFieldErr{
-			plugin: "regex parser",
-			field:  "name",
-		}
-	}
-	if p.Regex == "" {
-		return "", emptyFieldErr{
-			plugin: "regex parser",
-			field:  "regex",
-		}
-	}
 	var b strings.Builder
 	if err := parserConfTemplate.ExecuteTemplate(&b, "parserRegex", p); err != nil {
 		return "", err
@@ -629,25 +558,6 @@ type Tail struct {
 
 // renderConfig generates a section for configure fluentBit tail parser.
 func (t Tail) renderConfig() (string, error) {
-	if t.Tag == "" {
-		return "", emptyFieldErr{
-			plugin: "tail",
-			field:  "Tag",
-		}
-	}
-	//TODO: Add check that Path is a comma separated list.
-	if t.Path == "" {
-		return "", emptyFieldErr{
-			plugin: "tail",
-			field:  "Path",
-		}
-	}
-	if t.DB == "" {
-		return "", emptyFieldErr{
-			plugin: "tail",
-			field:  "DB",
-		}
-	}
 	var renderedTailConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedTailConfig, "tail", t); err != nil {
 		return "", err
@@ -655,16 +565,6 @@ func (t Tail) renderConfig() (string, error) {
 	return renderedTailConfig.String(), nil
 }
 
-type invalidValueErr struct {
-	plugin                string
-	field                 string
-	validValueExplanation string
-}
-
-func (e invalidValueErr) Error() string {
-	return fmt.Sprintf("got invalid value for %q plugin's field %q, should be %s", e.plugin, e.field, e.validValueExplanation)
-}
-
 // A Syslog represents the configuration data for fluentBit's syslog input plugin.
 type Syslog struct {
 	Mode   string
@@ -675,38 +575,6 @@ type Syslog struct {
 
 // renderConfig generates a section for configure fluentBit syslog input plugin.
 func (s Syslog) renderConfig() (string, error) {
-	switch m := s.Mode; m {
-	case "tcp", "udp":
-		if net.ParseIP(s.Listen) == nil {
-			return "", invalidValueErr{
-				plugin:                "syslog",
-				field:                 "Listen",
-				validValueExplanation: "a valid IP",
-			}
-		}
-		if s.Port == 0 {
-			return "", emptyFieldErr{
-				plugin: "syslog",
-				field:  "Port",
-			}
-		}
-	case "unix_tcp", "unix_udp":
-		// TODO: pending decision on setting up unix_tcp, unix_udp
-		fallthrough
-	default:
-		return "", invalidValueErr{
-			plugin:                "syslog",
-			field:                 "Mode",
-			validValueExplanation: "one of \"tcp\", \"udp\"",
-		}
-	}
-	if s.Tag == "" {
-		return "", emptyFieldErr{
-			plugin: "syslog",
-			field:  "Tag",
-		}
-	}
-
 	var renderedSyslogConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedSyslogConfig, "syslog", s); err != nil {
 		return "", err
@@ -724,21 +592,6 @@ type WindowsEventlog struct {
 
 // renderConfig generates a section for configure fluentBit wineventlog input plugin.
 func (w WindowsEventlog) renderConfig() (string, error) {
-	//TODO: Add check that Channels is a comma separated list.
-	if w.Channels == "" {
-		return "", emptyFieldErr{
-			plugin: "windows_event_log",
-			field:  "Channels",
-		}
-	}
-
-	if w.Tag == "" {
-		return "", emptyFieldErr{
-			plugin: "windows_event_log",
-			field:  "Tag",
-		}
-	}
-
 	var renderedWineventlogConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedWineventlogConfig, "wineventlog", w); err != nil {
 		return "", err
@@ -755,19 +608,6 @@ type Stackdriver struct {
 
 // renderConfig generates a section for configure fluentBit stackdriver output plugin.
 func (s Stackdriver) renderConfig() (string, error) {
-	if s.Match == "" {
-		return "", emptyFieldErr{
-			plugin: "stackdriver",
-			field:  "Match",
-		}
-	}
-	if s.UserAgent == "" {
-		return "", emptyFieldErr{
-			plugin: "stackdriver",
-			field:  "stackdriver_agent",
-		}
-	}
-
 	var renderedStackdriverConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedStackdriverConfig, "stackdriver", s); err != nil {
 		return "", err
diff --git a/confgenerator/fluentbit/conf_test.go b/confgenerator/fluentbit/conf_test.go
index 3952e6e8d2..9ac3153bfd 100644
--- a/confgenerator/fluentbit/conf_test.go
+++ b/confgenerator/fluentbit/conf_test.go
@@ -41,39 +41,6 @@ func TestFilterParser(t *testing.T) {
 	}
 }
 
-func TestFilterParserErrors(t *testing.T) {
-	tests := []struct {
-		filterParser FilterParser
-	}{
-		{
-			filterParser: FilterParser{},
-		},
-		{
-			filterParser: FilterParser{
-				Match:   "test_match",
-				KeyName: "test_key_name",
-			},
-		},
-		{
-			filterParser: FilterParser{
-				Match:  "test_match",
-				Parser: "test_parser",
-			},
-		},
-		{
-			filterParser: FilterParser{
-				KeyName: "test_key_name",
-				Parser:  "test_parser",
-			},
-		},
-	}
-	for _, tc := range tests {
-		if _, err := tc.filterParser.renderConfig(); err == nil {
-			t.Errorf("FilterParser %v: FilterParser.renderConfig() succeeded, want error", tc.filterParser)
-		}
-	}
-}
-
 func TestFilterRewriteTag(t *testing.T) {
 	f := FilterRewriteTag{
 		Match: "test_match",
@@ -94,13 +61,6 @@ func TestFilterRewriteTag(t *testing.T) {
 	}
 }
 
-func TestFilterRewriteTagError(t *testing.T) {
-	f := FilterRewriteTag{}
-	if _, err := f.renderConfig(); err == nil {
-		t.Errorf("FilterRewriteTag %v: FilterRewriteTag.renderConfig() succeeded, want error", f)
-	}
-}
-
 func TestParserJSON(t *testing.T) {
 	tests := []struct {
 		parserJSON         ParserJSON
@@ -151,37 +111,6 @@ func TestParserJSON(t *testing.T) {
 	}
 }
 
-func TestParserJSONErrors(t *testing.T) {
-	tests := []struct {
-		parserJSON ParserJSON
-	}{
-		{
-			parserJSON: ParserJSON{},
-		},
-		{
-			parserJSON: ParserJSON{
-				TimeKey:    "test_time_key",
-				TimeFormat: "test_time_format",
-			},
-		},
-		{
-			parserJSON: ParserJSON{
-				TimeKey: "test_time_key",
-			},
-		},
-		{
-			parserJSON: ParserJSON{
-				TimeFormat: "test_time_format",
-			},
-		},
-	}
-	for _, tc := range tests {
-		if _, err := tc.parserJSON.renderConfig(); err == nil {
-			t.Errorf("ParserJSON %v: ParserJSON.renderConfig() succeeded, want error", tc.parserJSON)
-		}
-	}
-}
-
 func TestParserRegex(t *testing.T) {
 	tests := []struct {
 		parserRegex        ParserRegex
@@ -249,39 +178,6 @@ func TestParserRegex(t *testing.T) {
 	}
 }
 
-func TestParserRegexErrors(t *testing.T) {
-	tests := []struct {
-		parserRegex ParserRegex
-	}{
-		{},
-		{
-			parserRegex: ParserRegex{
-				TimeKey:    "test_time_key",
-				TimeFormat: "test_time_format",
-			},
-		},
-		{
-			parserRegex: ParserRegex{
-				Name:       "test_name",
-				TimeKey:    "test_time_key",
-				TimeFormat: "test_time_format",
-			},
-		},
-		{
-			parserRegex: ParserRegex{
-				Regex:      "test_regex",
-				TimeKey:    "test_time_key",
-				TimeFormat: "test_time_format",
-			},
-		},
-	}
-	for _, tc := range tests {
-		if _, err := tc.parserRegex.renderConfig(); err == nil {
-			t.Errorf("ParserRegex %v: ParserRegex.renderConfig() succeeded, want error", tc.parserRegex)
-		}
-	}
-}
-
 func TestTail(t *testing.T) {
 	tests := []struct {
 		tail               Tail
@@ -411,36 +307,6 @@ func TestTail(t *testing.T) {
 	}
 }
 
-func TestTailErrors(t *testing.T) {
-	tests := []struct {
-		tail Tail
-	}{
-		{
-			tail: Tail{
-				DB:   "test_db",
-				Path: "test_path",
-			},
-		},
-		{
-			tail: Tail{
-				Tag:  "test_tag",
-				Path: "test_path",
-			},
-		},
-		{
-			tail: Tail{
-				Tag: "test_tag",
-				DB:  "test_db",
-			},
-		},
-	}
-	for _, tc := range tests {
-		if _, err := tc.tail.renderConfig(); err == nil {
-			t.Errorf("Tail %v: Tail.renderConfig() succeeded, want error", tc.tail)
-		}
-	}
-}
-
 func TestSyslog(t *testing.T) {
 	tests := []struct {
 		syslog               Syslog
@@ -486,53 +352,6 @@ func TestSyslog(t *testing.T) {
 	}
 }
 
-func TestSyslogErrors(t *testing.T) {
-	tests := []struct {
-		name   string
-		syslog Syslog
-	}{
-		{
-			name: "invalide mode",
-			syslog: Syslog{
-				Mode:   "invalid_mode",
-				Listen: "0.0.0.0",
-				Port:   1234,
-				Tag:    "test_tag",
-			},
-		},
-		{
-			name: "invalid listen",
-			syslog: Syslog{
-				Mode:   "tcp",
-				Listen: "non-IP",
-				Port:   1234,
-				Tag:    "test_tag",
-			},
-		},
-		{
-			name: "invalid port",
-			syslog: Syslog{
-				Mode:   "tcp",
-				Listen: "0.0.0.0",
-				Tag:    "test_tag",
-			},
-		},
-		{
-			name: "tag not provided",
-			syslog: Syslog{
-				Mode:   "tcp",
-				Listen: "0.0.0.0",
-				Port:   1234,
-			},
-		},
-	}
-	for _, tc := range tests {
-		if _, err := tc.syslog.renderConfig(); err == nil {
-			t.Errorf("test %q: syslog.renderConfig() succeeded, want error.", tc.name)
-		}
-	}
-}
-
 func TestWinlog(t *testing.T) {
 	tests := []struct {
 		wineventlog          WindowsEventlog
@@ -566,37 +385,6 @@ func TestWinlog(t *testing.T) {
 	}
 }
 
-func TestWinlogErrors(t *testing.T) {
-	tests := []struct {
-		name        string
-		wineventlog WindowsEventlog
-	}{
-		{
-			name: "empty channels",
-			wineventlog: WindowsEventlog{
-				Tag:          "test_tag",
-				Channels:     "",
-				Interval_Sec: "1",
-				DB:           "test_DB",
-			},
-		},
-		{
-			name: "empty tag",
-			wineventlog: WindowsEventlog{
-				Tag:          "",
-				Channels:     "test_chl",
-				Interval_Sec: "1",
-				DB:           "test_DB",
-			},
-		},
-	}
-	for _, tc := range tests {
-		if _, err := tc.wineventlog.renderConfig(); err == nil {
-			t.Errorf("test %q: wineventlog.renderConfig() succeeded, want error.", tc.name)
-		}
-	}
-}
-
 func TestStackdriver(t *testing.T) {
 	s := Stackdriver{
 		Match:     "test_match",
@@ -630,13 +418,6 @@ func TestStackdriver(t *testing.T) {
 	}
 }
 
-func TestStackdriverError(t *testing.T) {
-	s := Stackdriver{}
-	if _, err := s.renderConfig(); err == nil {
-		t.Errorf("Stackdriver %v: Stackdriver.renderConfig() succeeded, want error", s)
-	}
-}
-
 func TestGenerateFluentBitMainConfig(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -844,42 +625,6 @@ func TestGenerateFluentBitMainConfig(t *testing.T) {
 	}
 }
 
-func TestGenerateFluentBitMainConfigErrors(t *testing.T) {
-	tests := []struct {
-		name    string
-		tails   []*Tail
-		syslogs []*Syslog
-	}{
-		{
-			name: "an invalid Tail exists",
-			tails: []*Tail{{
-				DB:   "test_db",
-				Path: "test_path",
-			},
-			},
-		},
-		{
-			name: "an invalid Syslog exists",
-			syslogs: []*Syslog{{
-				Mode:   "not_syslog",
-				Listen: "",
-				Port:   0,
-				Tag:    "",
-			},
-			},
-		},
-	}
-	for _, tc := range tests {
-		_, _, err := Config{
-			Tails:   tc.tails,
-			Syslogs: tc.syslogs,
-		}.Generate()
-		if err == nil {
-			t.Errorf("test %q: GenerateFluentBitMainConfig succeeded, want error", tc.name)
-		}
-	}
-}
-
 func TestGenerateFluentBitMainConfigWindows(t *testing.T) {
 	tests := []struct {
 		name         string
@@ -1204,29 +949,3 @@ func TestGenerateFluentBitParserConfig(t *testing.T) {
 		}
 	}
 }
-
-func TestGenerateFluentBitParserConfigErrors(t *testing.T) {
-	tests := []struct {
-		name         string
-		jsonParsers  []*ParserJSON
-		regexParsers []*ParserRegex
-	}{
-		{
-			name:        "an invalid json parser exists",
-			jsonParsers: []*ParserJSON{{}},
-		},
-		{
-			name:         "an invalid regex parser exists",
-			regexParsers: []*ParserRegex{{}},
-		},
-	}
-	for _, tc := range tests {
-		_, _, err := Config{
-			JsonParsers:  tc.jsonParsers,
-			RegexParsers: tc.regexParsers,
-		}.Generate()
-		if err == nil {
-			t.Errorf("test %q: GenerateFluentBitParserConfig succeeded, want error", tc.name)
-		}
-	}
-}

From e34f779488313eea3edcf28ae21d5d3c0131e15b Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 5 Aug 2021 19:35:53 -0400
Subject: [PATCH 34/41] Use consistent ordering of options in fluent-bit
 configs. This is a pure reordering, with no functional changes.

---
 confgenerator/fluentbit/conf.go               | 10 ++---
 confgenerator/fluentbit/conf_test.go          | 42 +++++++++----------
 .../golden_fluent_bit_main.conf               | 16 +++----
 .../golden_fluent_bit_main.conf               | 16 +++----
 .../golden_fluent_bit_main.conf               | 16 +++----
 .../golden_fluent_bit_main.conf               | 10 ++---
 .../golden_fluent_bit_main.conf               | 24 +++++------
 .../golden_fluent_bit_main.conf               | 24 +++++------
 .../golden_fluent_bit_main.conf               | 24 +++++------
 .../golden_fluent_bit_main.conf               | 16 +++----
 .../golden_fluent_bit_main.conf               | 32 +++++++-------
 .../golden_fluent_bit_main.conf               | 16 +++----
 .../golden_fluent_bit_main.conf               | 16 +++----
 .../golden_fluent_bit_main.conf               | 16 +++----
 .../golden_fluent_bit_main.conf               | 16 +++----
 .../golden_fluent_bit_main.conf               | 16 +++----
 .../golden_fluent_bit_main.conf               | 10 ++---
 .../golden_fluent_bit_main.conf               | 10 ++---
 .../golden_fluent_bit_main.conf               | 10 ++---
 .../golden_fluent_bit_main.conf               |  6 +--
 .../golden_fluent_bit_main.conf               | 18 ++++----
 .../golden_fluent_bit_main.conf               | 10 ++---
 .../golden_fluent_bit_main.conf               | 10 ++---
 .../golden_fluent_bit_main.conf               | 10 ++---
 .../golden_fluent_bit_main.conf               | 10 ++---
 .../golden_fluent_bit_main.conf               | 10 ++---
 .../golden_fluent_bit_main.conf               | 10 ++---
 .../golden_fluent_bit_main.conf               | 10 ++---
 28 files changed, 217 insertions(+), 217 deletions(-)

diff --git a/confgenerator/fluentbit/conf.go b/confgenerator/fluentbit/conf.go
index aa37a0bf9c..677eecb2b0 100644
--- a/confgenerator/fluentbit/conf.go
+++ b/confgenerator/fluentbit/conf.go
@@ -113,9 +113,9 @@ var mainConfTemplate = template.Must(template.New("fluentBitMainConf").Parse(`[S
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 {{.DB}}
-    Path               {{.Path}}
     Tag                {{.Tag}}
+    Path               {{.Path}}
+    DB                 {{.DB}}
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -147,9 +147,9 @@ var mainConfTemplate = template.Must(template.New("fluentBitMainConf").Parse(`[S
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
+    Tag            {{.Tag}}
     Mode           {{.Mode}}
     Listen         {{.Listen}}
-    Tag            {{.Tag}}
     Port           {{.Port}}
     Parser         lib:default_message_parser
 
@@ -167,8 +167,8 @@ var mainConfTemplate = template.Must(template.New("fluentBitMainConf").Parse(`[S
 {{- define "wineventlog" -}}
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            {{.Tag}}
     Name           winlog
+    Tag            {{.Tag}}
     Channels       {{.Channels}}
     Interval_Sec   1
     DB             {{.DB}}
@@ -177,12 +177,12 @@ var mainConfTemplate = template.Must(template.New("fluentBitMainConf").Parse(`[S
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^({{.Match}})$
     resource          gce_instance
     stackdriver_agent {{.UserAgent}}
     {{- if .Workers}}
     workers           {{.Workers}}
     {{- end}}
-    Match_Regex       ^({{.Match}})$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/fluentbit/conf_test.go b/confgenerator/fluentbit/conf_test.go
index 9ac3153bfd..31d68b1141 100644
--- a/confgenerator/fluentbit/conf_test.go
+++ b/confgenerator/fluentbit/conf_test.go
@@ -192,9 +192,9 @@ func TestTail(t *testing.T) {
 			expectedTailConfig: `[INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 test_db
-    Path               test_path
     Tag                test_tag
+    Path               test_path
+    DB                 test_db
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -228,9 +228,9 @@ func TestTail(t *testing.T) {
 			expectedTailConfig: `[INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 test_db
-    Path               test_path
     Tag                test_tag
+    Path               test_path
+    DB                 test_db
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -266,9 +266,9 @@ func TestTail(t *testing.T) {
 			expectedTailConfig: `[INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 test_db
-    Path               test_path
     Tag                test_tag
+    Path               test_path
+    DB                 test_db
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -322,9 +322,9 @@ func TestSyslog(t *testing.T) {
 			expectedSyslogConfig: `[INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
+    Tag            test_tag
     Mode           tcp
     Listen         0.0.0.0
-    Tag            test_tag
     Port           1234
     Parser         lib:default_message_parser
 
@@ -366,8 +366,8 @@ func TestWinlog(t *testing.T) {
 			},
 			expectedWinlogConfig: `[INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            windows_event_log
     Name           winlog
+    Tag            windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             test_DB`,
@@ -394,10 +394,10 @@ func TestStackdriver(t *testing.T) {
 	want := `[OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(test_match)$
     resource          gce_instance
     stackdriver_agent user_agent
     workers           8
-    Match_Regex       ^(test_match)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -512,9 +512,9 @@ func TestGenerateFluentBitMainConfig(t *testing.T) {
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 test_db1
-    Path               test_path1
     Tag                test_tag1
+    Path               test_path1
+    DB                 test_db1
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -541,9 +541,9 @@ func TestGenerateFluentBitMainConfig(t *testing.T) {
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 test_db2
-    Path               test_path2
     Tag                test_tag2
+    Path               test_path2
+    DB                 test_db2
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -570,9 +570,9 @@ func TestGenerateFluentBitMainConfig(t *testing.T) {
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
+    Tag            test_tag1
     Mode           tcp
     Listen         0.0.0.0
-    Tag            test_tag1
     Port           1234
     Parser         lib:default_message_parser
 
@@ -590,9 +590,9 @@ func TestGenerateFluentBitMainConfig(t *testing.T) {
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
+    Tag            test_tag2
     Mode           udp
     Listen         0.0.0.0
-    Tag            test_tag2
     Port           5678
     Parser         lib:default_message_parser
 
@@ -686,9 +686,9 @@ func TestGenerateFluentBitMainConfigWindows(t *testing.T) {
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 test_db1
-    Path               test_path1
     Tag                test_tag1
+    Path               test_path1
+    DB                 test_db1
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -715,9 +715,9 @@ func TestGenerateFluentBitMainConfigWindows(t *testing.T) {
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 test_db2
-    Path               test_path2
     Tag                test_tag2
+    Path               test_path2
+    DB                 test_db2
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -743,16 +743,16 @@ func TestGenerateFluentBitMainConfigWindows(t *testing.T) {
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            win_tag1
     Name           winlog
+    Tag            win_tag1
     Channels       chl1
     Interval_Sec   1
     DB             test_DB1
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            win_tag2
     Name           winlog
+    Tag            win_tag2
     Channels       chl2
     Interval_Sec   1
     DB             test_DB2
diff --git a/confgenerator/testdata/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
index 3ab99ae576..075a5896ce 100644
--- a/confgenerator/testdata/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
-    Path               /var/log/messages,/var/log/syslog
     Tag                default_pipeline.syslog
+    Path               /var/log/messages,/var/log/syslog
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -134,10 +134,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -152,10 +152,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(syslog)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(syslog)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf
index 3ab99ae576..075a5896ce 100644
--- a/confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
-    Path               /var/log/messages,/var/log/syslog
     Tag                default_pipeline.syslog
+    Path               /var/log/messages,/var/log/syslog
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -134,10 +134,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -152,10 +152,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(syslog)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(syslog)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf
index 3ab99ae576..075a5896ce 100644
--- a/confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
-    Path               /var/log/messages,/var/log/syslog
     Tag                default_pipeline.syslog
+    Path               /var/log/messages,/var/log/syslog
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -134,10 +134,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -152,10 +152,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(syslog)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(syslog)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/logging-default_overrides_disable_all/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
index 3b42d46e3b..f42d5f08a5 100644
--- a/confgenerator/testdata/valid/linux/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,10 +88,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
index d657d80f0c..1a9957e268 100644
--- a/confgenerator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline1_log_source_id1
-    Path               /path/to/log/1/a/*,/path/to/log/1/b/*
     Tag                pipeline1.log_source_id1
+    Path               /path/to/log/1/a/*,/path/to/log/1/b/*
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline1_log_source_id1
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -119,9 +119,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline2_log_source_id2
-    Path               /path/to/log/2/a/*,/path/to/log/2/b/*
     Tag                pipeline2.log_source_id2
+    Path               /path/to/log/2/a/*,/path/to/log/2/b/*
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline2_log_source_id2
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -150,9 +150,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
+    Tag            pipeline3.test_syslog_source_id_tcp
     Mode           tcp
     Listen         1.1.1.1
-    Tag            pipeline3.test_syslog_source_id_tcp
     Port           1111
     Parser         lib:default_message_parser
 
@@ -170,9 +170,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
+    Tag            pipeline4.test_syslog_source_id_udp
     Mode           udp
     Listen         2.2.2.2
-    Tag            pipeline4.test_syslog_source_id_udp
     Port           2222
     Parser         lib:default_message_parser
 
@@ -270,10 +270,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(log_source_id1|log_source_id2|test_syslog_source_id_tcp|test_syslog_source_id_udp)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(log_source_id1|log_source_id2|test_syslog_source_id_tcp|test_syslog_source_id_udp)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -288,10 +288,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf
index ba9749cf5a..e91df3347a 100644
--- a/confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
-    Path               /var/log/messages,/var/log/syslog
     Tag                default_pipeline.syslog
+    Path               /var/log/messages,/var/log/syslog
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -117,9 +117,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline1_sample_logs
-    Path               /tmp/*.log
     Tag                pipeline1.sample_logs
+    Path               /tmp/*.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline1_sample_logs
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -146,9 +146,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline2_sample_logs
-    Path               /tmp/*.log
     Tag                pipeline2.sample_logs
+    Path               /tmp/*.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline2_sample_logs
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -250,10 +250,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -268,10 +268,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(syslog|sample_logs|sample_logs)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(syslog|sample_logs|sample_logs)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/logging-processor_parse_json_and_parse_regex_types/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-processor_parse_json_and_parse_regex_types/golden_fluent_bit_main.conf
index a2e8ac7e86..b4fa6b080c 100644
--- a/confgenerator/testdata/valid/linux/logging-processor_parse_json_and_parse_regex_types/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-processor_parse_json_and_parse_regex_types/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline1_log_source_id1
-    Path               /path/to/log/1/a/*,/path/to/log/1/b/*
     Tag                pipeline1.log_source_id1
+    Path               /path/to/log/1/a/*,/path/to/log/1/b/*
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline1_log_source_id1
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -119,9 +119,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline2_log_source_id2
-    Path               /path/to/log/2/a/*,/path/to/log/2/b/*
     Tag                pipeline2.log_source_id2
+    Path               /path/to/log/2/a/*,/path/to/log/2/b/*
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline2_log_source_id2
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -150,9 +150,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
+    Tag            pipeline3.test_syslog_source_id_tcp
     Mode           tcp
     Listen         1.1.1.1
-    Tag            pipeline3.test_syslog_source_id_tcp
     Port           1111
     Parser         lib:default_message_parser
 
@@ -170,9 +170,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
+    Tag            pipeline4.test_syslog_source_id_udp
     Mode           udp
     Listen         2.2.2.2
-    Tag            pipeline4.test_syslog_source_id_udp
     Port           2222
     Parser         lib:default_message_parser
 
@@ -282,10 +282,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(log_source_id1|log_source_id2|test_syslog_source_id_tcp|test_syslog_source_id_udp)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(log_source_id1|log_source_id2|test_syslog_source_id_tcp|test_syslog_source_id_udp)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -300,10 +300,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/logging-processor_parse_regex_type_on_default_pipeline/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-processor_parse_regex_type_on_default_pipeline/golden_fluent_bit_main.conf
index 190366b1df..e8a9b8c736 100644
--- a/confgenerator/testdata/valid/linux/logging-processor_parse_regex_type_on_default_pipeline/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-processor_parse_regex_type_on_default_pipeline/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
-    Path               /var/log/messages,/var/log/syslog
     Tag                default_pipeline.syslog
+    Path               /var/log/messages,/var/log/syslog
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -140,10 +140,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -158,10 +158,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(syslog)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(syslog)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
index 762c5c341f..c7a831dc04 100644
--- a/confgenerator/testdata/valid/linux/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline1_log_source_id1
-    Path               /path/to/log/1/*
     Tag                pipeline1.log_source_id1
+    Path               /path/to/log/1/*
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline1_log_source_id1
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -117,9 +117,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline2_log_source_id2
-    Path               /path/to/log/2/a,/path/to/log/2/b
     Tag                pipeline2.log_source_id2
+    Path               /path/to/log/2/a,/path/to/log/2/b
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline2_log_source_id2
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -146,9 +146,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline3_log_source_id3
-    Path               /path/to/log/3/*
     Tag                pipeline3.log_source_id3
+    Path               /path/to/log/3/*
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline3_log_source_id3
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -177,9 +177,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline4_log_source_id4
-    Path               /path/to/log/4/*
     Tag                pipeline4.log_source_id4
+    Path               /path/to/log/4/*
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline4_log_source_id4
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -208,9 +208,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline5_log_source_id5
-    Path               /path/to/log/5/*
     Tag                pipeline5.log_source_id5
+    Path               /path/to/log/5/*
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/pipeline5_log_source_id5
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -328,10 +328,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(log_source_id1|log_source_id2|log_source_id3|log_source_id4|log_source_id5)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(log_source_id1|log_source_id2|log_source_id3|log_source_id4|log_source_id5)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -346,10 +346,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/logging-receiver_syslog_type_multiple_receivers/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_syslog_type_multiple_receivers/golden_fluent_bit_main.conf
index 64df717202..1c6163a41d 100644
--- a/confgenerator/testdata/valid/linux/logging-receiver_syslog_type_multiple_receivers/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-receiver_syslog_type_multiple_receivers/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
+    Tag            pipeline1.test_syslog_source_id_tcp
     Mode           tcp
     Listen         1.1.1.1
-    Tag            pipeline1.test_syslog_source_id_tcp
     Port           1111
     Parser         lib:default_message_parser
 
@@ -108,9 +108,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/syslog
     Name           syslog
+    Tag            pipeline2.test_syslog_source_id_udp
     Mode           udp
     Listen         2.2.2.2
-    Tag            pipeline2.test_syslog_source_id_udp
     Port           2222
     Parser         lib:default_message_parser
 
@@ -174,10 +174,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -192,10 +192,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(test_syslog_source_id_tcp|test_syslog_source_id_udp)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(test_syslog_source_id_tcp|test_syslog_source_id_udp)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
index 3ab99ae576..075a5896ce 100644
--- a/confgenerator/testdata/valid/linux/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
-    Path               /var/log/messages,/var/log/syslog
     Tag                default_pipeline.syslog
+    Path               /var/log/messages,/var/log/syslog
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -134,10 +134,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -152,10 +152,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(syslog)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(syslog)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
index 3ab99ae576..075a5896ce 100644
--- a/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
-    Path               /var/log/messages,/var/log/syslog
     Tag                default_pipeline.syslog
+    Path               /var/log/messages,/var/log/syslog
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -134,10 +134,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -152,10 +152,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(syslog)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(syslog)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
index 3ab99ae576..075a5896ce 100644
--- a/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
-    Path               /var/log/messages,/var/log/syslog
     Tag                default_pipeline.syslog
+    Path               /var/log/messages,/var/log/syslog
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -134,10 +134,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -152,10 +152,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(syslog)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(syslog)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/linux/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
index 3ab99ae576..075a5896ce 100644
--- a/confgenerator/testdata/valid/linux/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
-    Path               /var/log/messages,/var/log/syslog
     Tag                default_pipeline.syslog
+    Path               /var/log/messages,/var/log/syslog
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/default_pipeline_syslog
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
-    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
     Tag                ops-agent-collectd
+    Path               /var/log/google-cloud-ops-agent/subagents/metrics-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-collectd
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
-    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               /var/log/google-cloud-ops-agent/subagents/logging-module.log
+    DB                 /var/lib/google-cloud-ops-agent/fluent-bit/buffers/ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -134,10 +134,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -152,10 +152,10 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(syslog)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
     workers           8
-    Match_Regex       ^(syslog)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
index eda4a3586a..2584aaaa3d 100644
--- a/confgenerator/testdata/valid/windows/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -58,8 +58,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -84,9 +84,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -101,9 +101,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/all-built_in_config/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/all-built_in_config/golden_fluent_bit_main.conf
index eda4a3586a..2584aaaa3d 100644
--- a/confgenerator/testdata/valid/windows/all-built_in_config/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/all-built_in_config/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -58,8 +58,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -84,9 +84,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -101,9 +101,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/all-user_config_file_deleted/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/all-user_config_file_deleted/golden_fluent_bit_main.conf
index eda4a3586a..2584aaaa3d 100644
--- a/confgenerator/testdata/valid/windows/all-user_config_file_deleted/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/all-user_config_file_deleted/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -58,8 +58,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -84,9 +84,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -101,9 +101,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/logging-default_overrides_disable_all/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
index 3488f777ad..358efb79df 100644
--- a/confgenerator/testdata/valid/windows/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
index d0a9ca1740..d55e5b2dce 100644
--- a/confgenerator/testdata/valid/windows/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -59,9 +59,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\pipeline1_log_source_id1
-    Path               /path/to/log/1/a
     Tag                pipeline1.log_source_id1
+    Path               /path/to/log/1/a
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\pipeline1_log_source_id1
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -88,9 +88,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\pipeline2_log_source_id2
-    Path               /path/to/log/2/a,/path/to/log/2/b
     Tag                pipeline2.log_source_id2
+    Path               /path/to/log/2/a,/path/to/log/2/b
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\pipeline2_log_source_id2
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -116,8 +116,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -182,9 +182,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -199,9 +199,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log|log_source_id1|log_source_id2)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log|log_source_id1|log_source_id2)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
index eda4a3586a..2584aaaa3d 100644
--- a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -58,8 +58,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -84,9 +84,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -101,9 +101,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_iis/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_iis/golden_fluent_bit_main.conf
index eda4a3586a..2584aaaa3d 100644
--- a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_iis/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_iis/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -58,8 +58,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -84,9 +84,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -101,9 +101,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_mssql/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_mssql/golden_fluent_bit_main.conf
index eda4a3586a..2584aaaa3d 100644
--- a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_mssql/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_mssql/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -58,8 +58,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -84,9 +84,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -101,9 +101,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/metrics-pipeline_multiple_pipelines/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
index eda4a3586a..2584aaaa3d 100644
--- a/confgenerator/testdata/valid/windows/metrics-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -58,8 +58,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -84,9 +84,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -101,9 +101,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
index eda4a3586a..2584aaaa3d 100644
--- a/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -58,8 +58,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -84,9 +84,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -101,9 +101,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
index eda4a3586a..2584aaaa3d 100644
--- a/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -58,8 +58,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -84,9 +84,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -101,9 +101,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
diff --git a/confgenerator/testdata/valid/windows/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
index eda4a3586a..2584aaaa3d 100644
--- a/confgenerator/testdata/valid/windows/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
@@ -30,9 +30,9 @@
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/tail#config
     Name               tail
-    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
-    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
     Tag                ops-agent-fluent-bit
+    Path               C:\ProgramData\Google\Cloud Operations\Ops Agent\log\logging-module.log
+    DB                 C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\ops-agent-fluent-bit
     Read_from_Head     True
     # Set the chunk limit conservatively to avoid exceeding the recommended chunk size of 5MB per write request.
     Buffer_Chunk_Size  512k
@@ -58,8 +58,8 @@
 
 [INPUT]
     # https://docs.fluentbit.io/manual/pipeline/inputs/windows-event-log
-    Tag            default_pipeline.windows_event_log
     Name           winlog
+    Tag            default_pipeline.windows_event_log
     Channels       System,Application,Security
     Interval_Sec   1
     DB             C:\ProgramData\Google\Cloud Operations\Ops Agent\run\buffers\default_pipeline_windows_event_log
@@ -84,9 +84,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(ops-agent-fluent-bit|ops-agent-collectd)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.
@@ -101,9 +101,9 @@
 [OUTPUT]
     # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
     Name              stackdriver
+    Match_Regex       ^(windows_event_log)$
     resource          gce_instance
     stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
-    Match_Regex       ^(windows_event_log)$
 
     # https://docs.fluentbit.io/manual/administration/scheduling-and-retries
     # After 3 retries, a given chunk will be discarded. So bad entries don't accidentally stay around forever.

From a8e6f6d279b24439d14add9d72960b4fa1edb414 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 5 Aug 2021 19:42:18 -0400
Subject: [PATCH 35/41] Sort input fluentbit config entities instead of the
 generated text.

---
 confgenerator/confgenerator.go  | 12 ++++++++++++
 confgenerator/fluentbit/conf.go | 11 -----------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 688f5fd031..942d3427e9 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 	"text/template"
 
@@ -286,6 +287,17 @@ func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir strin
 			return "", "", err
 		}
 	}
+
+	// make sure all collections are sorted so that generated configs are consistently generated
+	sort.Slice(fbTails, func(i, j int) bool { return fbTails[i].Tag < fbTails[j].Tag })
+	sort.Slice(fbSyslogs, func(i, j int) bool { return fbSyslogs[i].Tag < fbSyslogs[j].Tag })
+	sort.Slice(fbWinEventlogs, func(i, j int) bool { return fbWinEventlogs[i].Tag < fbWinEventlogs[j].Tag })
+	sort.Slice(fbFilterParserGroups, func(i, j int) bool { return fbFilterParserGroups[i][0].Match < fbFilterParserGroups[j][0].Match })
+	sort.Slice(fbFilterAddLogNames, func(i, j int) bool { return fbFilterAddLogNames[i].Match < fbFilterAddLogNames[j].Match })
+	sort.Slice(fbFilterRemoveLogNames, func(i, j int) bool { return fbFilterRemoveLogNames[i].Match < fbFilterRemoveLogNames[j].Match })
+	sort.Slice(fbFilterRewriteTags, func(i, j int) bool { return fbFilterRewriteTags[i].Match < fbFilterRewriteTags[j].Match })
+	sort.Slice(fbStackdrivers, func(i, j int) bool { return fbStackdrivers[i].Match < fbStackdrivers[j].Match })
+
 	mainConfig, parserConfig, err := fluentbit.Config{
 		Tails:                      fbTails,
 		Syslogs:                    fbSyslogs,
diff --git a/confgenerator/fluentbit/conf.go b/confgenerator/fluentbit/conf.go
index 677eecb2b0..3efa178fd0 100644
--- a/confgenerator/fluentbit/conf.go
+++ b/confgenerator/fluentbit/conf.go
@@ -16,7 +16,6 @@
 package fluentbit
 
 import (
-	"sort"
 	"strings"
 	"text/template"
 )
@@ -382,16 +381,6 @@ func (c Config) generateMain() (string, error) {
 		stackdriverConfigSections = append(stackdriverConfigSections, configSection)
 	}
 
-	// make sure all collections are sorted so that generated configs are consistently generated
-	sort.Strings(tailConfigSections)
-	sort.Strings(syslogConfigSections)
-	sort.Strings(wineventlogConfigSections)
-	sort.Strings(filterParserConfigSections)
-	sort.Strings(filterModifyAddLogNameConfigSections)
-	sort.Strings(filterRewriteTagSections)
-	sort.Strings(filterModifyRemoveLogNameConfigSections)
-	sort.Strings(stackdriverConfigSections)
-
 	configSections := mainConfigSections{
 		TailConfigSections:                      tailConfigSections,
 		SyslogConfigSections:                    syslogConfigSections,

From e6cda0704ff92bf53067d174693677fa335f0ce5 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 5 Aug 2021 22:06:06 -0400
Subject: [PATCH 36/41] Switch to explicit template rendering for fluentbit
 config components.

---
 confgenerator/fluentbit/conf.go | 164 +++++---------------------------
 1 file changed, 26 insertions(+), 138 deletions(-)

diff --git a/confgenerator/fluentbit/conf.go b/confgenerator/fluentbit/conf.go
index 3efa178fd0..64899305ca 100644
--- a/confgenerator/fluentbit/conf.go
+++ b/confgenerator/fluentbit/conf.go
@@ -49,36 +49,40 @@ var mainConfTemplate = template.Must(template.New("fluentBitMainConf").Parse(`[S
     # we don't want them to all be loaded into the memory.
     storage.max_chunks_up      128
 
-{{range .TailConfigSections -}}
-{{.}}
+{{range .Tails -}}
+{{template "tail" .}}
 
 {{end}}
-{{- range .SyslogConfigSections -}}
-{{.}}
+{{- range .Syslogs -}}
+{{template "syslog" .}}
 
 {{end}}
-{{- range .WineventlogConfigSections -}}
-{{.}}
+{{- range .Wineventlogs -}}
+{{template "wineventlog" .}}
 
 {{end}}
-{{- range .FilterParserConfigSections -}}
-{{.}}
+{{- range .FilterParserGroups -}}
+{{- range . -}}
+{{template "filter_parser" .}}
+
+{{end -}}
 
 {{end}}
-{{- range .FilterModifyAddLogNameConfigSections -}}
-{{.}}
+
+{{- range .FilterModifyAddLogNames -}}
+{{template "filter_modify_add_log_name" .}}
 
 {{end}}
-{{- range .FilterRewriteTagSections -}}
-{{.}}
+{{- range .FilterRewriteTags -}}
+{{template "filter_rewrite_tag" .}}
 
 {{end}}
-{{- range .FilterModifyRemoveLogNameConfigSections -}}
-{{.}}
+{{- range .FilterModifyRemoveLogNames -}}
+{{template "filter_modify_remove_log_name" .}}
 
 {{end}}
-{{- range .StackdriverConfigSections -}}
-{{.}}
+{{- range .Stackdrivers -}}
+{{template "stackdriver" .}}
 
 {{end}}
 {{- define "filter_modify_add_log_name" -}}
@@ -247,12 +251,12 @@ var parserConfTemplate = template.Must(template.New("fluentBitParserConf").Parse
     Time_Key    time
     Time_Format %b %d %H:%M:%S
 
-{{range .JSONParserConfigSections -}}
-{{.}}
+{{range .JsonParsers -}}
+{{template "parserJSON" .}}
 
 {{end}}
-{{- range .RegexParserConfigSections -}}
-{{.}}
+{{- range .RegexParsers -}}
+{{template "parserRegex" .}}
 
 {{end}}
 {{- define "parserJSON" -}}
@@ -280,22 +284,6 @@ var parserConfTemplate = template.Must(template.New("fluentBitParserConf").Parse
 {{- end -}}
 `))
 
-type mainConfigSections struct {
-	TailConfigSections                      []string
-	SyslogConfigSections                    []string
-	WineventlogConfigSections               []string
-	FilterParserConfigSections              []string
-	FilterModifyAddLogNameConfigSections    []string
-	FilterRewriteTagSections                []string
-	FilterModifyRemoveLogNameConfigSections []string
-	StackdriverConfigSections               []string
-}
-
-type parserConfigSections struct {
-	JSONParserConfigSections  []string
-	RegexParserConfigSections []string
-}
-
 type Config struct {
 	Tails                      []*Tail
 	Syslogs                    []*Syslog
@@ -311,120 +299,20 @@ type Config struct {
 }
 
 func (c Config) generateMain() (string, error) {
-	tailConfigSections := []string{}
-	syslogConfigSections := []string{}
-	wineventlogConfigSections := []string{}
-	filterParserConfigSections := []string{}
-	filterModifyAddLogNameConfigSections := []string{}
-	filterRewriteTagSections := []string{}
-	filterModifyRemoveLogNameConfigSections := []string{}
-	stackdriverConfigSections := []string{}
-	for _, t := range c.Tails {
-		configSection, err := t.renderConfig()
-		if err != nil {
-			return "", err
-		}
-		tailConfigSections = append(tailConfigSections, configSection)
-	}
-	for _, s := range c.Syslogs {
-		configSection, err := s.renderConfig()
-		if err != nil {
-			return "", err
-		}
-		syslogConfigSections = append(syslogConfigSections, configSection)
-	}
-	for _, w := range c.Wineventlogs {
-		configSection, err := w.renderConfig()
-		if err != nil {
-			return "", err
-		}
-		wineventlogConfigSections = append(wineventlogConfigSections, configSection)
-	}
-	for _, filterParsers := range c.FilterParserGroups {
-		var filters []string
-		for _, f := range filterParsers {
-			configSection, err := f.renderConfig()
-			if err != nil {
-				return "", err
-			}
-			filters = append(filters, configSection)
-		}
-		filterParserConfigSections = append(filterParserConfigSections, strings.Join(filters, "\n\n"))
-	}
-	for _, f := range c.FilterModifyAddLogNames {
-		configSection, err := f.renderConfig()
-		if err != nil {
-			return "", err
-		}
-		filterModifyAddLogNameConfigSections = append(filterModifyAddLogNameConfigSections, configSection)
-	}
-	for _, f := range c.FilterRewriteTags {
-		configSection, err := f.renderConfig()
-		if err != nil {
-			return "", err
-		}
-		filterRewriteTagSections = append(filterRewriteTagSections, configSection)
-	}
-	for _, f := range c.FilterModifyRemoveLogNames {
-		configSection, err := f.renderConfig()
-		if err != nil {
-			return "", err
-		}
-		filterModifyRemoveLogNameConfigSections = append(filterModifyRemoveLogNameConfigSections, configSection)
-	}
 	for _, s := range c.Stackdrivers {
 		s.UserAgent = c.UserAgent
-		configSection, err := s.renderConfig()
-		if err != nil {
-			return "", err
-		}
-		stackdriverConfigSections = append(stackdriverConfigSections, configSection)
-	}
-
-	configSections := mainConfigSections{
-		TailConfigSections:                      tailConfigSections,
-		SyslogConfigSections:                    syslogConfigSections,
-		WineventlogConfigSections:               wineventlogConfigSections,
-		FilterParserConfigSections:              filterParserConfigSections,
-		FilterModifyAddLogNameConfigSections:    filterModifyAddLogNameConfigSections,
-		FilterRewriteTagSections:                filterRewriteTagSections,
-		FilterModifyRemoveLogNameConfigSections: filterModifyRemoveLogNameConfigSections,
-		StackdriverConfigSections:               stackdriverConfigSections,
 	}
 
 	var mainConfigBuilder strings.Builder
-	if err := mainConfTemplate.Execute(&mainConfigBuilder, configSections); err != nil {
+	if err := mainConfTemplate.Execute(&mainConfigBuilder, c); err != nil {
 		return "", err
 	}
 	return mainConfigBuilder.String(), nil
 }
 
 func (c Config) generateParser() (string, error) {
-	jsonParserConfigSections := []string{}
-	for _, j := range c.JsonParsers {
-		configSection, err := j.renderConfig()
-		if err != nil {
-			return "", err
-		}
-		jsonParserConfigSections = append(jsonParserConfigSections, configSection)
-	}
-
-	regexParserConfigSections := []string{}
-	for _, r := range c.RegexParsers {
-		configSection, err := r.renderConfig()
-		if err != nil {
-			return "", err
-		}
-		regexParserConfigSections = append(regexParserConfigSections, configSection)
-	}
-
-	parsers := parserConfigSections{
-		JSONParserConfigSections:  jsonParserConfigSections,
-		RegexParserConfigSections: regexParserConfigSections,
-	}
-
 	var parserConfigBuilder strings.Builder
-	if err := parserConfTemplate.Execute(&parserConfigBuilder, parsers); err != nil {
+	if err := parserConfTemplate.Execute(&parserConfigBuilder, c); err != nil {
 		return "", err
 	}
 	return parserConfigBuilder.String(), nil

From c08fcf2882eed30b9f6f41319b00ddd845020c2d Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 5 Aug 2021 23:42:22 -0400
Subject: [PATCH 37/41] Standardize on the Diff library and the order of diffs.

---
 confgenerator/confgenerator_test.go  |  6 +++---
 confgenerator/fluentbit/conf_test.go | 24 ++++++++++++------------
 confgenerator/otel/conf_test.go      |  8 ++++----
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/confgenerator/confgenerator_test.go b/confgenerator/confgenerator_test.go
index 65765385f1..d14b164fb1 100644
--- a/confgenerator/confgenerator_test.go
+++ b/confgenerator/confgenerator_test.go
@@ -202,15 +202,15 @@ func updateOrCompareGolden(t *testing.T, testName string, goos string, expectedB
 	expected := strings.ReplaceAll(string(expectedBytes), "\r\n", "\n")
 	actual = strings.ReplaceAll(actual, "\r\n", "\n")
 	goldenPath := fmt.Sprintf(path, goos, testName)
-	if diff := cmp.Diff(actual, expected); diff != "" {
+	if diff := cmp.Diff(expected, actual); diff != "" {
 		if *updateGolden {
 			// Update the expected to match the actual.
-			t.Logf("Detected -update_golden flag. Rewriting the %q golden file to apply the following diff\n%s.", goldenPath, diff)
+			t.Logf("Detected -update_golden flag. Rewriting the %q golden file to apply the following diff\n%s.", goldenPath, cmp.Diff(actual, expected))
 			if err := ioutil.WriteFile(goldenPath, []byte(actual), 0644); err != nil {
 				t.Fatalf("error updating golden file at %q : %s", goldenPath, err)
 			}
 		} else {
-			t.Errorf("test %q: golden file at %s mismatch (-got +want):\n%s", testName, goldenPath, diff)
+			t.Errorf("test %q: golden file at %s mismatch (-want +got):\n%s", testName, goldenPath, diff)
 		}
 	}
 }
diff --git a/confgenerator/fluentbit/conf_test.go b/confgenerator/fluentbit/conf_test.go
index 31d68b1141..f7cd99b882 100644
--- a/confgenerator/fluentbit/conf_test.go
+++ b/confgenerator/fluentbit/conf_test.go
@@ -17,7 +17,7 @@ package fluentbit
 import (
 	"testing"
 
-	"github.com/kylelemons/godebug/diff"
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestFilterParser(t *testing.T) {
@@ -36,7 +36,7 @@ func TestFilterParser(t *testing.T) {
 		t.Errorf("got error: %v, want no error", err)
 		return
 	}
-	if diff := diff.Diff(want, got); diff != "" {
+	if diff := cmp.Diff(want, got); diff != "" {
 		t.Errorf("FilterParser %v: FilterParser.renderConfig() returned unexpected diff (-want +got):\n%s", want, diff)
 	}
 }
@@ -56,7 +56,7 @@ func TestFilterRewriteTag(t *testing.T) {
 		t.Errorf("got error: %v, want no error", err)
 		return
 	}
-	if diff := diff.Diff(want, got); diff != "" {
+	if diff := cmp.Diff(want, got); diff != "" {
 		t.Errorf("FilterRewriteTag %v: FilterRewriteTag.renderConfig() returned unexpected diff (-want +got):\n%s", want, diff)
 	}
 }
@@ -105,7 +105,7 @@ func TestParserJSON(t *testing.T) {
 			t.Errorf("got error: %v, want no error", err)
 			return
 		}
-		if diff := diff.Diff(tc.expectedTailConfig, got); diff != "" {
+		if diff := cmp.Diff(tc.expectedTailConfig, got); diff != "" {
 			t.Errorf("ParserJSON %v: ParserJSON.renderConfig() returned unexpected diff (-want +got):\n%s", tc.parserJSON, diff)
 		}
 	}
@@ -172,7 +172,7 @@ func TestParserRegex(t *testing.T) {
 			return
 
 		}
-		if diff := diff.Diff(tc.expectedTailConfig, got); diff != "" {
+		if diff := cmp.Diff(tc.expectedTailConfig, got); diff != "" {
 			t.Errorf("ParserRegex %v: ParserRegex.renderConfig() returned unexpected diff (-want +got):\n%s", tc.parserRegex, diff)
 		}
 	}
@@ -301,7 +301,7 @@ func TestTail(t *testing.T) {
 			t.Errorf("got error: %v, want no error", err)
 			return
 		}
-		if diff := diff.Diff(tc.expectedTailConfig, got); diff != "" {
+		if diff := cmp.Diff(tc.expectedTailConfig, got); diff != "" {
 			t.Errorf("Tail %v: ran Tail.renderConfig() returned unexpected diff (-want +got):\n%s", tc.tail, diff)
 		}
 	}
@@ -346,7 +346,7 @@ func TestSyslog(t *testing.T) {
 			t.Errorf("got error: %v, want no error", err)
 			return
 		}
-		if diff := diff.Diff(tc.expectedSyslogConfig, got); diff != "" {
+		if diff := cmp.Diff(tc.expectedSyslogConfig, got); diff != "" {
 			t.Errorf("Tail %v: ran syslog.renderConfig() returned unexpected diff (-want +got):\n%s", tc.syslog, diff)
 		}
 	}
@@ -379,7 +379,7 @@ func TestWinlog(t *testing.T) {
 			t.Errorf("got error: %v, want no error", err)
 			return
 		}
-		if diff := diff.Diff(tc.expectedWinlogConfig, got); diff != "" {
+		if diff := cmp.Diff(tc.expectedWinlogConfig, got); diff != "" {
 			t.Errorf("Tail %v: ran wineventlog.renderConfig() returned unexpected diff (-want +got):\n%s", tc.wineventlog, diff)
 		}
 	}
@@ -413,7 +413,7 @@ func TestStackdriver(t *testing.T) {
 		t.Errorf("got error: %v, want no error", err)
 		return
 	}
-	if diff := diff.Diff(want, got); diff != "" {
+	if diff := cmp.Diff(want, got); diff != "" {
 		t.Errorf("Stackdriver %v: Stackdriver.renderConfig() returned unexpected diff (-want +got):\n%s", want, diff)
 	}
 }
@@ -619,7 +619,7 @@ func TestGenerateFluentBitMainConfig(t *testing.T) {
 			t.Errorf("got error: %v, want no error", err)
 			return
 		}
-		if diff := diff.Diff(tc.want, got); diff != "" {
+		if diff := cmp.Diff(tc.want, got); diff != "" {
 			t.Errorf("test %q: ran GenerateFluentBitMainConfig returned unexpected diff (-want +got):\n%s", tc.name, diff)
 		}
 	}
@@ -769,7 +769,7 @@ func TestGenerateFluentBitMainConfigWindows(t *testing.T) {
 			t.Errorf("got error: %v, want no error", err)
 			return
 		}
-		if diff := diff.Diff(tc.want, got); diff != "" {
+		if diff := cmp.Diff(tc.want, got); diff != "" {
 			t.Errorf("test %q: ran GenerateFluentBitMainConfig returned unexpected diff (-want +got):\n%s", tc.name, diff)
 		}
 	}
@@ -944,7 +944,7 @@ func TestGenerateFluentBitParserConfig(t *testing.T) {
 			t.Errorf("test %q got error: %v, want no error", tc.name, err)
 			return
 		}
-		if diff := diff.Diff(tc.want, got); diff != "" {
+		if diff := cmp.Diff(tc.want, got); diff != "" {
 			t.Errorf("test %q: ran GenerateFluentBitParserConfig returned unexpected diff (-want +got):\n%s", tc.name, diff)
 		}
 	}
diff --git a/confgenerator/otel/conf_test.go b/confgenerator/otel/conf_test.go
index c81bca6f8d..6952f89040 100644
--- a/confgenerator/otel/conf_test.go
+++ b/confgenerator/otel/conf_test.go
@@ -19,7 +19,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/kylelemons/godebug/diff"
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestSection(t *testing.T) {
@@ -126,7 +126,7 @@ func TestSection(t *testing.T) {
 					t.Errorf("got error: %v, want no error", err)
 					return
 				}
-				if diff := diff.Diff(tc.want, got); diff != "" {
+				if diff := cmp.Diff(tc.want, got); diff != "" {
 					t.Errorf("service.renderConfig() returned unexpected diff (-want +got):\n%s", diff)
 				}
 			} else {
@@ -729,8 +729,8 @@ service:
 				t.Errorf("got error: %v, want no error", err)
 				return
 			}
-			if diff := diff.Diff(got, tc.want); diff != "" {
-				t.Errorf("test %q: ran GenerateOtelConfig returned unexpected diff (-got +want):\n%s", tc.name, diff)
+			if diff := cmp.Diff(tc.want, got); diff != "" {
+				t.Errorf("test %q: ran GenerateOtelConfig returned unexpected diff (-want +got):\n%s", tc.name, diff)
 			}
 		})
 	}

From d5e9046d23f1d4ab47323f7dd0f5c473ef0941d0 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Thu, 5 Aug 2021 23:51:17 -0400
Subject: [PATCH 38/41] Unify fluentbit's Tail, Syslog, and WindowsEventlog
 structs under the fluentbit.Input interface. Unify fluentbit's filters under
 the fluentbit.Filter interface, and fluentbit's parsers under the
 fluentbit.Parser interface. Wrap fluentbit's Stackdriver into the
 fluentbit.Output interface. Use the fluentbit.Input, fluentbit.Filter,
 fluentbit.Output, and fluentbit.Parser interfaces instead of the individual
 structs.

Template expansion whitespace control is hell, so there are no longer two blank lines at the end of the config. No other changes to the generated configs.
---
 confgenerator/confgenerator.go                |  53 +++++-
 confgenerator/fluentbit/conf.go               | 127 ++++++-------
 confgenerator/fluentbit/conf_test.go          | 178 +++++++++---------
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 .../golden_fluent_bit_main.conf               |   3 +-
 29 files changed, 210 insertions(+), 226 deletions(-)

diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go
index 942d3427e9..62e2113332 100644
--- a/confgenerator/confgenerator.go
+++ b/confgenerator/confgenerator.go
@@ -298,17 +298,50 @@ func (uc *UnifiedConfig) GenerateFluentBitConfigs(logsDir string, stateDir strin
 	sort.Slice(fbFilterRewriteTags, func(i, j int) bool { return fbFilterRewriteTags[i].Match < fbFilterRewriteTags[j].Match })
 	sort.Slice(fbStackdrivers, func(i, j int) bool { return fbStackdrivers[i].Match < fbStackdrivers[j].Match })
 
+	inputs := []fluentbit.Input{}
+	for _, i := range fbTails {
+		inputs = append(inputs, i)
+	}
+	for _, i := range fbSyslogs {
+		inputs = append(inputs, i)
+	}
+	for _, i := range fbWinEventlogs {
+		inputs = append(inputs, i)
+	}
+
+	filters := []fluentbit.Filter{}
+	for _, f := range fbFilterParserGroups {
+		filters = append(filters, f)
+	}
+	for _, f := range fbFilterAddLogNames {
+		filters = append(filters, f)
+	}
+	for _, f := range fbFilterRewriteTags {
+		filters = append(filters, f)
+	}
+	for _, f := range fbFilterRemoveLogNames {
+		filters = append(filters, f)
+	}
+
+	outputs := []fluentbit.Output{}
+	for _, o := range fbStackdrivers {
+		o.UserAgent = userAgent
+		outputs = append(outputs, o)
+	}
+
+	parsers := []fluentbit.Parser{}
+	for _, p := range jsonParsers {
+		parsers = append(parsers, p)
+	}
+	for _, p := range regexParsers {
+		parsers = append(parsers, p)
+	}
+
 	mainConfig, parserConfig, err := fluentbit.Config{
-		Tails:                      fbTails,
-		Syslogs:                    fbSyslogs,
-		Wineventlogs:               fbWinEventlogs,
-		FilterParserGroups:         fbFilterParserGroups,
-		FilterModifyAddLogNames:    fbFilterAddLogNames,
-		FilterModifyRemoveLogNames: fbFilterRemoveLogNames,
-		FilterRewriteTags:          fbFilterRewriteTags,
-		Stackdrivers:               fbStackdrivers,
-		JsonParsers:                jsonParsers,
-		RegexParsers:               regexParsers,
+		Inputs:  inputs,
+		Outputs: outputs,
+		Filters: filters,
+		Parsers: parsers,
 
 		UserAgent: userAgent,
 	}.Generate()
diff --git a/confgenerator/fluentbit/conf.go b/confgenerator/fluentbit/conf.go
index 64899305ca..1da6afcfb6 100644
--- a/confgenerator/fluentbit/conf.go
+++ b/confgenerator/fluentbit/conf.go
@@ -49,42 +49,18 @@ var mainConfTemplate = template.Must(template.New("fluentBitMainConf").Parse(`[S
     # we don't want them to all be loaded into the memory.
     storage.max_chunks_up      128
 
-{{range .Tails -}}
-{{template "tail" .}}
+{{- range .Inputs}}
 
-{{end}}
-{{- range .Syslogs -}}
-{{template "syslog" .}}
-
-{{end}}
-{{- range .Wineventlogs -}}
-{{template "wineventlog" .}}
-
-{{end}}
-{{- range .FilterParserGroups -}}
-{{- range . -}}
-{{template "filter_parser" .}}
-
-{{end -}}
-
-{{end}}
-
-{{- range .FilterModifyAddLogNames -}}
-{{template "filter_modify_add_log_name" .}}
-
-{{end}}
-{{- range .FilterRewriteTags -}}
-{{template "filter_rewrite_tag" .}}
-
-{{end}}
-{{- range .FilterModifyRemoveLogNames -}}
-{{template "filter_modify_remove_log_name" .}}
+{{.Generate}}
+{{- end}}
+{{- range .Filters}}
 
-{{end}}
-{{- range .Stackdrivers -}}
-{{template "stackdriver" .}}
+{{.Generate}}
+{{- end}}
+{{- range .Outputs}}
 
-{{end}}
+{{.Generate}}
+{{- end}}
 {{- define "filter_modify_add_log_name" -}}
 [FILTER]
     Name  modify
@@ -251,15 +227,12 @@ var parserConfTemplate = template.Must(template.New("fluentBitParserConf").Parse
     Time_Key    time
     Time_Format %b %d %H:%M:%S
 
-{{range .JsonParsers -}}
-{{template "parserJSON" .}}
+{{- range .Parsers}}
 
-{{end}}
-{{- range .RegexParsers -}}
-{{template "parserRegex" .}}
+{{.Generate}}
+{{- end}}
 
-{{end}}
-{{- define "parserJSON" -}}
+{{define "parserJSON" -}}
 [PARSER]
     Name        {{.Name}}
     Format      json
@@ -285,24 +258,14 @@ var parserConfTemplate = template.Must(template.New("fluentBitParserConf").Parse
 `))
 
 type Config struct {
-	Tails                      []*Tail
-	Syslogs                    []*Syslog
-	Wineventlogs               []*WindowsEventlog
-	FilterParserGroups         []FilterParserGroup
-	FilterModifyAddLogNames    []*FilterModifyAddLogName
-	FilterRewriteTags          []*FilterRewriteTag
-	FilterModifyRemoveLogNames []*FilterModifyRemoveLogName
-	Stackdrivers               []*Stackdriver
-	JsonParsers                []*ParserJSON
-	RegexParsers               []*ParserRegex
-	UserAgent                  string
+	Inputs    []Input
+	Filters   []Filter
+	Outputs   []Output
+	Parsers   []Parser
+	UserAgent string
 }
 
 func (c Config) generateMain() (string, error) {
-	for _, s := range c.Stackdrivers {
-		s.UserAgent = c.UserAgent
-	}
-
 	var mainConfigBuilder strings.Builder
 	if err := mainConfTemplate.Execute(&mainConfigBuilder, c); err != nil {
 		return "", err
@@ -332,9 +295,25 @@ func (c Config) Generate() (mainConfig string, parserConfig string, err error) {
 	return mainConfig, parserConfig, nil
 }
 
+type Filter interface {
+	Generate() (string, error)
+}
+
 // A FilterParserGroup represents a list of filters to be applied in order.
 type FilterParserGroup []*FilterParser
 
+func (g FilterParserGroup) Generate() (string, error) {
+	var filters []string
+	for _, f := range g {
+		configSection, err := f.Generate()
+		if err != nil {
+			return "", err
+		}
+		filters = append(filters, configSection)
+	}
+	return strings.Join(filters, "\n\n"), nil
+}
+
 // A FilterParser represents the configuration data for fluentBit's filter parser plugin.
 type FilterParser struct {
 	Match   string
@@ -342,7 +321,7 @@ type FilterParser struct {
 	Parser  string
 }
 
-func (f FilterParser) renderConfig() (string, error) {
+func (f FilterParser) Generate() (string, error) {
 	var renderedFilterParserConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterParserConfig, "filter_parser", f); err != nil {
 		return "", err
@@ -357,7 +336,7 @@ type FilterModifyAddLogName struct {
 	LogName string
 }
 
-func (f FilterModifyAddLogName) renderConfig() (string, error) {
+func (f FilterModifyAddLogName) Generate() (string, error) {
 	var renderedFilterModifyAddLogNameConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterModifyAddLogNameConfig, "filter_modify_add_log_name", f); err != nil {
 		return "", err
@@ -371,7 +350,7 @@ type FilterModifyRemoveLogName struct {
 	Match string
 }
 
-func (f FilterModifyRemoveLogName) renderConfig() (string, error) {
+func (f FilterModifyRemoveLogName) Generate() (string, error) {
 	var renderedFilterModifyRemoveLogNameConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterModifyRemoveLogNameConfig, "filter_modify_remove_log_name", f); err != nil {
 		return "", err
@@ -384,7 +363,7 @@ type FilterRewriteTag struct {
 	Match string
 }
 
-func (f FilterRewriteTag) renderConfig() (string, error) {
+func (f FilterRewriteTag) Generate() (string, error) {
 	var renderedFilterRewriteTagConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedFilterRewriteTagConfig, "filter_rewrite_tag", f); err != nil {
 		return "", err
@@ -392,6 +371,10 @@ func (f FilterRewriteTag) renderConfig() (string, error) {
 	return renderedFilterRewriteTagConfig.String(), nil
 }
 
+type Parser interface {
+	Generate() (string, error)
+}
+
 // A ParserJSON represents the configuration data for fluentBit's JSON parser.
 type ParserJSON struct {
 	Name       string
@@ -399,8 +382,7 @@ type ParserJSON struct {
 	TimeFormat string
 }
 
-// renderConfig generates a section for configure fluentBit JSON parser.
-func (p ParserJSON) renderConfig() (string, error) {
+func (p ParserJSON) Generate() (string, error) {
 	var b strings.Builder
 	if err := parserConfTemplate.ExecuteTemplate(&b, "parserJSON", p); err != nil {
 		return "", err
@@ -416,8 +398,7 @@ type ParserRegex struct {
 	TimeFormat string
 }
 
-// renderConfig generates a section for configure fluentBit Regex parser.
-func (p ParserRegex) renderConfig() (string, error) {
+func (p ParserRegex) Generate() (string, error) {
 	var b strings.Builder
 	if err := parserConfTemplate.ExecuteTemplate(&b, "parserRegex", p); err != nil {
 		return "", err
@@ -425,6 +406,10 @@ func (p ParserRegex) renderConfig() (string, error) {
 	return b.String(), nil
 }
 
+type Input interface {
+	Generate() (string, error)
+}
+
 // A Tail represents the configuration data for fluentBit's tail input plugin.
 type Tail struct {
 	Tag         string
@@ -433,8 +418,7 @@ type Tail struct {
 	DB          string
 }
 
-// renderConfig generates a section for configure fluentBit tail parser.
-func (t Tail) renderConfig() (string, error) {
+func (t Tail) Generate() (string, error) {
 	var renderedTailConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedTailConfig, "tail", t); err != nil {
 		return "", err
@@ -450,8 +434,7 @@ type Syslog struct {
 	Tag    string
 }
 
-// renderConfig generates a section for configure fluentBit syslog input plugin.
-func (s Syslog) renderConfig() (string, error) {
+func (s Syslog) Generate() (string, error) {
 	var renderedSyslogConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedSyslogConfig, "syslog", s); err != nil {
 		return "", err
@@ -467,8 +450,7 @@ type WindowsEventlog struct {
 	DB           string
 }
 
-// renderConfig generates a section for configure fluentBit wineventlog input plugin.
-func (w WindowsEventlog) renderConfig() (string, error) {
+func (w WindowsEventlog) Generate() (string, error) {
 	var renderedWineventlogConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedWineventlogConfig, "wineventlog", w); err != nil {
 		return "", err
@@ -476,6 +458,10 @@ func (w WindowsEventlog) renderConfig() (string, error) {
 	return renderedWineventlogConfig.String(), nil
 }
 
+type Output interface {
+	Generate() (string, error)
+}
+
 // A Stackdriver represents the configurable data for fluentBit's stackdriver output plugin.
 type Stackdriver struct {
 	Match     string
@@ -483,8 +469,7 @@ type Stackdriver struct {
 	Workers   int
 }
 
-// renderConfig generates a section for configure fluentBit stackdriver output plugin.
-func (s Stackdriver) renderConfig() (string, error) {
+func (s Stackdriver) Generate() (string, error) {
 	var renderedStackdriverConfig strings.Builder
 	if err := mainConfTemplate.ExecuteTemplate(&renderedStackdriverConfig, "stackdriver", s); err != nil {
 		return "", err
diff --git a/confgenerator/fluentbit/conf_test.go b/confgenerator/fluentbit/conf_test.go
index f7cd99b882..ea11e63b89 100644
--- a/confgenerator/fluentbit/conf_test.go
+++ b/confgenerator/fluentbit/conf_test.go
@@ -31,13 +31,13 @@ func TestFilterParser(t *testing.T) {
     Match    test_match
     Key_Name test_key_name
     Parser   test_parser`
-	got, err := f.renderConfig()
+	got, err := f.Generate()
 	if err != nil {
 		t.Errorf("got error: %v, want no error", err)
 		return
 	}
 	if diff := cmp.Diff(want, got); diff != "" {
-		t.Errorf("FilterParser %v: FilterParser.renderConfig() returned unexpected diff (-want +got):\n%s", want, diff)
+		t.Errorf("FilterParser %v: FilterParser.Generate() returned unexpected diff (-want +got):\n%s", want, diff)
 	}
 }
 
@@ -51,13 +51,13 @@ func TestFilterRewriteTag(t *testing.T) {
     Rule                  $logName .* $logName false
     Emitter_Storage.type  filesystem
     Emitter_Mem_Buf_Limit 10M`
-	got, err := f.renderConfig()
+	got, err := f.Generate()
 	if err != nil {
 		t.Errorf("got error: %v, want no error", err)
 		return
 	}
 	if diff := cmp.Diff(want, got); diff != "" {
-		t.Errorf("FilterRewriteTag %v: FilterRewriteTag.renderConfig() returned unexpected diff (-want +got):\n%s", want, diff)
+		t.Errorf("FilterRewriteTag %v: FilterRewriteTag.Generate() returned unexpected diff (-want +got):\n%s", want, diff)
 	}
 }
 
@@ -100,13 +100,13 @@ func TestParserJSON(t *testing.T) {
 		},
 	}
 	for _, tc := range tests {
-		got, err := tc.parserJSON.renderConfig()
+		got, err := tc.parserJSON.Generate()
 		if err != nil {
 			t.Errorf("got error: %v, want no error", err)
 			return
 		}
 		if diff := cmp.Diff(tc.expectedTailConfig, got); diff != "" {
-			t.Errorf("ParserJSON %v: ParserJSON.renderConfig() returned unexpected diff (-want +got):\n%s", tc.parserJSON, diff)
+			t.Errorf("ParserJSON %v: ParserJSON.Generate() returned unexpected diff (-want +got):\n%s", tc.parserJSON, diff)
 		}
 	}
 }
@@ -166,14 +166,14 @@ func TestParserRegex(t *testing.T) {
 		},
 	}
 	for _, tc := range tests {
-		got, err := tc.parserRegex.renderConfig()
+		got, err := tc.parserRegex.Generate()
 		if err != nil {
 			t.Errorf("got error: %v, want no error", err)
 			return
 
 		}
 		if diff := cmp.Diff(tc.expectedTailConfig, got); diff != "" {
-			t.Errorf("ParserRegex %v: ParserRegex.renderConfig() returned unexpected diff (-want +got):\n%s", tc.parserRegex, diff)
+			t.Errorf("ParserRegex %v: ParserRegex.Generate() returned unexpected diff (-want +got):\n%s", tc.parserRegex, diff)
 		}
 	}
 }
@@ -296,13 +296,13 @@ func TestTail(t *testing.T) {
 		},
 	}
 	for _, tc := range tests {
-		got, err := tc.tail.renderConfig()
+		got, err := tc.tail.Generate()
 		if err != nil {
 			t.Errorf("got error: %v, want no error", err)
 			return
 		}
 		if diff := cmp.Diff(tc.expectedTailConfig, got); diff != "" {
-			t.Errorf("Tail %v: ran Tail.renderConfig() returned unexpected diff (-want +got):\n%s", tc.tail, diff)
+			t.Errorf("Tail %v: ran Tail.Generate() returned unexpected diff (-want +got):\n%s", tc.tail, diff)
 		}
 	}
 }
@@ -341,13 +341,13 @@ func TestSyslog(t *testing.T) {
 		},
 	}
 	for _, tc := range tests {
-		got, err := tc.syslog.renderConfig()
+		got, err := tc.syslog.Generate()
 		if err != nil {
 			t.Errorf("got error: %v, want no error", err)
 			return
 		}
 		if diff := cmp.Diff(tc.expectedSyslogConfig, got); diff != "" {
-			t.Errorf("Tail %v: ran syslog.renderConfig() returned unexpected diff (-want +got):\n%s", tc.syslog, diff)
+			t.Errorf("Tail %v: ran syslog.Generate() returned unexpected diff (-want +got):\n%s", tc.syslog, diff)
 		}
 	}
 }
@@ -374,13 +374,13 @@ func TestWinlog(t *testing.T) {
 		},
 	}
 	for _, tc := range tests {
-		got, err := tc.wineventlog.renderConfig()
+		got, err := tc.wineventlog.Generate()
 		if err != nil {
 			t.Errorf("got error: %v, want no error", err)
 			return
 		}
 		if diff := cmp.Diff(tc.expectedWinlogConfig, got); diff != "" {
-			t.Errorf("Tail %v: ran wineventlog.renderConfig() returned unexpected diff (-want +got):\n%s", tc.wineventlog, diff)
+			t.Errorf("Tail %v: ran wineventlog.Generate() returned unexpected diff (-want +got):\n%s", tc.wineventlog, diff)
 		}
 	}
 }
@@ -408,22 +408,21 @@ func TestStackdriver(t *testing.T) {
     tls         On
     # Do not force certificate validation.
     tls.verify  Off`
-	got, err := s.renderConfig()
+	got, err := s.Generate()
 	if err != nil {
 		t.Errorf("got error: %v, want no error", err)
 		return
 	}
 	if diff := cmp.Diff(want, got); diff != "" {
-		t.Errorf("Stackdriver %v: Stackdriver.renderConfig() returned unexpected diff (-want +got):\n%s", want, diff)
+		t.Errorf("Stackdriver %v: Stackdriver.Generate() returned unexpected diff (-want +got):\n%s", want, diff)
 	}
 }
 
 func TestGenerateFluentBitMainConfig(t *testing.T) {
 	tests := []struct {
-		name    string
-		tails   []*Tail
-		syslogs []*Syslog
-		want    string
+		name   string
+		inputs []Input
+		want   string
 	}{
 		{
 			name: "zero plugins",
@@ -454,32 +453,32 @@ func TestGenerateFluentBitMainConfig(t *testing.T) {
     # This is exclusive to filesystem storage type. It specifies the number of chunks (every chunk is a file) that can be up in memory.
     # Every chunk is a file, so having it up in memory means having an open file descriptor. In case there are thousands of chunks,
     # we don't want them to all be loaded into the memory.
-    storage.max_chunks_up      128
-
-`,
+    storage.max_chunks_up      128`,
 		},
 		{
 			name: "multiple tail and syslog plugins",
-			tails: []*Tail{{
-				Tag:  "test_tag1",
-				DB:   "test_db1",
-				Path: "test_path1",
-			}, {
-				Tag:  "test_tag2",
-				DB:   "test_db2",
-				Path: "test_path2",
-			}},
-			syslogs: []*Syslog{{
-				Mode:   "tcp",
-				Listen: "0.0.0.0",
-				Port:   1234,
-				Tag:    "test_tag1",
-			}, {
-				Mode:   "udp",
-				Listen: "0.0.0.0",
-				Port:   5678,
-				Tag:    "test_tag2",
-			}},
+			inputs: []Input{
+				&Tail{
+					Tag:  "test_tag1",
+					DB:   "test_db1",
+					Path: "test_path1",
+				}, &Tail{
+					Tag:  "test_tag2",
+					DB:   "test_db2",
+					Path: "test_path2",
+				},
+				&Syslog{
+					Mode:   "tcp",
+					Listen: "0.0.0.0",
+					Port:   1234,
+					Tag:    "test_tag1",
+				}, &Syslog{
+					Mode:   "udp",
+					Listen: "0.0.0.0",
+					Port:   5678,
+					Tag:    "test_tag2",
+				},
+			},
 			want: `[SERVICE]
     # https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file#config_section
     # Flush logs every 1 second, even if the buffer is not full to minimize log entry arrival delay.
@@ -605,15 +604,12 @@ func TestGenerateFluentBitMainConfig(t *testing.T) {
     # This is used to deal with backpressure scenarios (e.g: cannot flush data for some reason).
     # When the input plugin hits "mem_buf_limit", because we have enabled filesystem storage type, mem_buf_limit acts
     # as a hint to set "how much data can be up in memory", once the limit is reached it continues writing to disk.
-    Mem_Buf_Limit  10M
-
-`,
+    Mem_Buf_Limit  10M`,
 		},
 	}
 	for _, tc := range tests {
 		got, _, err := Config{
-			Tails:   tc.tails,
-			Syslogs: tc.syslogs,
+			Inputs: tc.inputs,
 		}.Generate()
 		if err != nil {
 			t.Errorf("got error: %v, want no error", err)
@@ -627,33 +623,34 @@ func TestGenerateFluentBitMainConfig(t *testing.T) {
 
 func TestGenerateFluentBitMainConfigWindows(t *testing.T) {
 	tests := []struct {
-		name         string
-		tails        []*Tail
-		wineventlogs []*WindowsEventlog
-		want         string
+		name   string
+		inputs []Input
+		want   string
 	}{
 		{
 			name: "multiple tail and winlog plugins",
-			tails: []*Tail{{
-				Tag:  "test_tag1",
-				DB:   "test_db1",
-				Path: "test_path1",
-			}, {
-				Tag:  "test_tag2",
-				DB:   "test_db2",
-				Path: "test_path2",
-			}},
-			wineventlogs: []*WindowsEventlog{{
-				Tag:          "win_tag1",
-				Channels:     "chl1",
-				Interval_Sec: "1",
-				DB:           "test_DB1",
-			}, {
-				Tag:          "win_tag2",
-				Channels:     "chl2",
-				Interval_Sec: "1",
-				DB:           "test_DB2",
-			}},
+			inputs: []Input{
+				&Tail{
+					Tag:  "test_tag1",
+					DB:   "test_db1",
+					Path: "test_path1",
+				}, &Tail{
+					Tag:  "test_tag2",
+					DB:   "test_db2",
+					Path: "test_path2",
+				},
+				&WindowsEventlog{
+					Tag:          "win_tag1",
+					Channels:     "chl1",
+					Interval_Sec: "1",
+					DB:           "test_DB1",
+				}, &WindowsEventlog{
+					Tag:          "win_tag2",
+					Channels:     "chl2",
+					Interval_Sec: "1",
+					DB:           "test_DB2",
+				},
+			},
 			want: `[SERVICE]
     # https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file#config_section
     # Flush logs every 1 second, even if the buffer is not full to minimize log entry arrival delay.
@@ -755,15 +752,12 @@ func TestGenerateFluentBitMainConfigWindows(t *testing.T) {
     Tag            win_tag2
     Channels       chl2
     Interval_Sec   1
-    DB             test_DB2
-
-`,
+    DB             test_DB2`,
 		},
 	}
 	for _, tc := range tests {
 		got, _, err := Config{
-			Tails:        tc.tails,
-			Wineventlogs: tc.wineventlogs,
+			Inputs: tc.inputs,
 		}.Generate()
 		if err != nil {
 			t.Errorf("got error: %v, want no error", err)
@@ -777,10 +771,9 @@ func TestGenerateFluentBitMainConfigWindows(t *testing.T) {
 
 func TestGenerateFluentBitParserConfig(t *testing.T) {
 	tests := []struct {
-		name         string
-		jsonParsers  []*ParserJSON
-		regexParsers []*ParserRegex
-		want         string
+		name    string
+		parsers []Parser
+		want    string
 	}{
 		{
 			name: "empty JSON Parsers and Regex Parsers",
@@ -840,24 +833,24 @@ func TestGenerateFluentBitParserConfig(t *testing.T) {
 		},
 		{
 			name: "multiple JSON Parsers and Regex Parsers",
-			jsonParsers: []*ParserJSON{
-				{
+			parsers: []Parser{
+				&ParserJSON{
 					Name:       "test_name1",
 					TimeKey:    "test_time_key1",
 					TimeFormat: "test_time_format1",
-				}, {
+				}, &ParserJSON{
 					Name:       "test_name2",
 					TimeKey:    "test_time_key2",
 					TimeFormat: "test_time_format2",
 				},
+				&ParserRegex{
+					Name:  "test_name1",
+					Regex: "test_regex1",
+				}, &ParserRegex{
+					Name:  "test_name2",
+					Regex: "test_regex2",
+				},
 			},
-			regexParsers: []*ParserRegex{{
-				Name:  "test_name1",
-				Regex: "test_regex1",
-			}, {
-				Name:  "test_name2",
-				Regex: "test_regex2",
-			}},
 			want: `[PARSER]
     Name        lib:default_message_parser
     Format      regex
@@ -937,8 +930,7 @@ func TestGenerateFluentBitParserConfig(t *testing.T) {
 	}
 	for _, tc := range tests {
 		_, got, err := Config{
-			JsonParsers:  tc.jsonParsers,
-			RegexParsers: tc.regexParsers,
+			Parsers: tc.parsers,
 		}.Generate()
 		if err != nil {
 			t.Errorf("test %q got error: %v, want no error", tc.name, err)
diff --git a/confgenerator/testdata/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
index 075a5896ce..73e3a275a4 100644
--- a/confgenerator/testdata/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
@@ -165,5 +165,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf
index 075a5896ce..73e3a275a4 100644
--- a/confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf
@@ -165,5 +165,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf
index 075a5896ce..73e3a275a4 100644
--- a/confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf
@@ -165,5 +165,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/logging-default_overrides_disable_all/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
index f42d5f08a5..14806e2693 100644
--- a/confgenerator/testdata/valid/linux/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
@@ -101,5 +101,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
index 1a9957e268..7eb8c9c716 100644
--- a/confgenerator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
@@ -301,5 +301,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf
index e91df3347a..64544cd142 100644
--- a/confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf
@@ -281,5 +281,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/logging-processor_parse_json_and_parse_regex_types/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-processor_parse_json_and_parse_regex_types/golden_fluent_bit_main.conf
index b4fa6b080c..ceb2d946e4 100644
--- a/confgenerator/testdata/valid/linux/logging-processor_parse_json_and_parse_regex_types/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-processor_parse_json_and_parse_regex_types/golden_fluent_bit_main.conf
@@ -313,5 +313,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/logging-processor_parse_regex_type_on_default_pipeline/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-processor_parse_regex_type_on_default_pipeline/golden_fluent_bit_main.conf
index e8a9b8c736..f7a7c28836 100644
--- a/confgenerator/testdata/valid/linux/logging-processor_parse_regex_type_on_default_pipeline/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-processor_parse_regex_type_on_default_pipeline/golden_fluent_bit_main.conf
@@ -171,5 +171,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
index c7a831dc04..7b6bb6feb9 100644
--- a/confgenerator/testdata/valid/linux/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
@@ -359,5 +359,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/logging-receiver_syslog_type_multiple_receivers/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_syslog_type_multiple_receivers/golden_fluent_bit_main.conf
index 1c6163a41d..1602fb25f1 100644
--- a/confgenerator/testdata/valid/linux/logging-receiver_syslog_type_multiple_receivers/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/logging-receiver_syslog_type_multiple_receivers/golden_fluent_bit_main.conf
@@ -205,5 +205,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
index 075a5896ce..73e3a275a4 100644
--- a/confgenerator/testdata/valid/linux/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
@@ -165,5 +165,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
index 075a5896ce..73e3a275a4 100644
--- a/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
@@ -165,5 +165,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
index 075a5896ce..73e3a275a4 100644
--- a/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
@@ -165,5 +165,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/linux/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
index 075a5896ce..73e3a275a4 100644
--- a/confgenerator/testdata/valid/linux/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/linux/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
@@ -165,5 +165,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
index 2584aaaa3d..8e759a5b28 100644
--- a/confgenerator/testdata/valid/windows/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
@@ -113,5 +113,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/all-built_in_config/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/all-built_in_config/golden_fluent_bit_main.conf
index 2584aaaa3d..8e759a5b28 100644
--- a/confgenerator/testdata/valid/windows/all-built_in_config/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/all-built_in_config/golden_fluent_bit_main.conf
@@ -113,5 +113,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/all-user_config_file_deleted/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/all-user_config_file_deleted/golden_fluent_bit_main.conf
index 2584aaaa3d..8e759a5b28 100644
--- a/confgenerator/testdata/valid/windows/all-user_config_file_deleted/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/all-user_config_file_deleted/golden_fluent_bit_main.conf
@@ -113,5 +113,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/logging-default_overrides_disable_all/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
index 358efb79df..d71746d4c2 100644
--- a/confgenerator/testdata/valid/windows/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
@@ -71,5 +71,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
index d55e5b2dce..725e65a6a1 100644
--- a/confgenerator/testdata/valid/windows/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
@@ -211,5 +211,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
index 2584aaaa3d..8e759a5b28 100644
--- a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_all/golden_fluent_bit_main.conf
@@ -113,5 +113,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_iis/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_iis/golden_fluent_bit_main.conf
index 2584aaaa3d..8e759a5b28 100644
--- a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_iis/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_iis/golden_fluent_bit_main.conf
@@ -113,5 +113,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_mssql/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_mssql/golden_fluent_bit_main.conf
index 2584aaaa3d..8e759a5b28 100644
--- a/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_mssql/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-default_overrides_disable_mssql/golden_fluent_bit_main.conf
@@ -113,5 +113,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/metrics-pipeline_multiple_pipelines/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
index 2584aaaa3d..8e759a5b28 100644
--- a/confgenerator/testdata/valid/windows/metrics-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
@@ -113,5 +113,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
index 2584aaaa3d..8e759a5b28 100644
--- a/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_by_prefixes/golden_fluent_bit_main.conf
@@ -113,5 +113,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
index 2584aaaa3d..8e759a5b28 100644
--- a/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-processor_exclude_metrics_type_filter_with_globs/golden_fluent_bit_main.conf
@@ -113,5 +113,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file
diff --git a/confgenerator/testdata/valid/windows/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/windows/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
index 2584aaaa3d..8e759a5b28 100644
--- a/confgenerator/testdata/valid/windows/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
+++ b/confgenerator/testdata/valid/windows/metrics-receiver_custom_collection_interval/golden_fluent_bit_main.conf
@@ -113,5 +113,4 @@
     # Enable TLS support.
     tls         On
     # Do not force certificate validation.
-    tls.verify  Off
-
+    tls.verify  Off
\ No newline at end of file

From ef84963586d87f199167e3835b61e35381888bc6 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Fri, 6 Aug 2021 12:41:33 -0400
Subject: [PATCH 39/41] Use `validate:"required"` to indicate required fields
 (instead of `config:"optional"`).

---
 confgenerator/config.go | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/confgenerator/config.go b/confgenerator/config.go
index 99f3e149d7..de53a2b3f5 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -80,7 +80,7 @@ type component interface {
 }
 
 type configComponent struct {
-	ComponentType string `yaml:"type"`
+	ComponentType string `yaml:"type" validate:"required"`
 }
 
 func (c *configComponent) Type() string {
@@ -131,8 +131,8 @@ type LoggingReceiver interface {
 type LoggingReceiverFiles struct {
 	configComponent `yaml:",inline"`
 
-	IncludePaths []string `yaml:"include_paths,omitempty"`
-	ExcludePaths []string `yaml:"exclude_paths,omitempty" config:"optional"`
+	IncludePaths []string `yaml:"include_paths,omitempty" validate:"required"`
+	ExcludePaths []string `yaml:"exclude_paths,omitempty"`
 }
 
 func (r LoggingReceiverFiles) Type() string {
@@ -146,9 +146,9 @@ func init() {
 type LoggingReceiverSyslog struct {
 	configComponent `yaml:",inline"`
 
-	TransportProtocol string `yaml:"transport_protocol,omitempty" config:"optional"` // one of "tcp" or "udp"
-	ListenHost        string `yaml:"listen_host,omitempty"`
-	ListenPort        uint16 `yaml:"listen_port,omitempty"`
+	TransportProtocol string `yaml:"transport_protocol,omitempty"` // one of "tcp" or "udp"
+	ListenHost        string `yaml:"listen_host,omitempty" validate:"required"`
+	ListenPort        uint16 `yaml:"listen_port,omitempty" validate:"required"`
 }
 
 func (r LoggingReceiverSyslog) Type() string {
@@ -162,7 +162,7 @@ func init() {
 type LoggingReceiverWinevtlog struct {
 	configComponent `yaml:",inline"`
 
-	Channels []string `yaml:"channels,omitempty,flow"`
+	Channels []string `yaml:"channels,omitempty,flow" validate:"required"`
 }
 
 func (r LoggingReceiverWinevtlog) Type() string {
@@ -207,9 +207,9 @@ type LoggingProcessor interface {
 }
 
 type loggingProcessorParseShared struct {
-	Field      string `yaml:"field,omitempty" config:"optional"`       // default to "message"
-	TimeKey    string `yaml:"time_key,omitempty" config:"optional"`    // by default does not parse timestamp
-	TimeFormat string `yaml:"time_format,omitempty" config:"optional"` // must be provided if time_key is present
+	Field      string `yaml:"field,omitempty"`       // default to "message"
+	TimeKey    string `yaml:"time_key,omitempty"`    // by default does not parse timestamp
+	TimeFormat string `yaml:"time_format,omitempty"` // must be provided if time_key is present
 }
 
 func (p loggingProcessorParseShared) GetField() string {
@@ -233,7 +233,7 @@ type LoggingProcessorParseRegex struct {
 	configComponent             `yaml:",inline"`
 	loggingProcessorParseShared `yaml:",inline"`
 
-	Regex string `yaml:"regex,omitempty"`
+	Regex string `yaml:"regex,omitempty" validate:"required"`
 }
 
 func (r LoggingProcessorParseRegex) Type() string {
@@ -343,7 +343,7 @@ type MetricsReceiver interface {
 }
 
 type metricsReceiverShared struct {
-	CollectionInterval string `yaml:"collection_interval"` // time.Duration format
+	CollectionInterval string `yaml:"collection_interval" validate:"required"` // time.Duration format
 }
 
 func (r *metricsReceiverShared) GetCollectionInterval() string {
@@ -427,7 +427,7 @@ type MetricsProcessor interface {
 type MetricsProcessorExcludeMetrics struct {
 	configComponent `yaml:",inline"`
 
-	MetricsPattern []string `yaml:"metrics_pattern,flow"`
+	MetricsPattern []string `yaml:"metrics_pattern,flow" validate:"required"`
 }
 
 func (r MetricsProcessorExcludeMetrics) Type() string {
@@ -809,13 +809,14 @@ func collectYamlFields(s interface{}) []yamlField {
 				// Expand inline structs.
 				parameters = append(parameters, recurse(v)...)
 			} else if f.PkgPath == "" { // skip private non-struct fields
-				t, e := f.Tag.Lookup("config")
+				t, e := f.Tag.Lookup("validate")
 				if !e {
 					t = ""
 				}
+				validation := strings.Split(t, ",")
 				parameters = append(parameters, yamlField{
 					Name:     n,
-					Required: t != "optional",
+					Required: sliceContains(validation, "required"),
 					Value:    v.Interface(),
 					IsZero:   v.IsZero(),
 				})

From 14d94c36369416d96245ea6d14856f04085a54d6 Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Fri, 30 Jul 2021 19:20:33 -0400
Subject: [PATCH 40/41] WIP: Attempt to reconstruct previous validation error
 messages.

---
 confgenerator/config.go                       | 35 +++++++++++++++++--
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 .../golden_error                              |  2 +-
 21 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/confgenerator/config.go b/confgenerator/config.go
index de53a2b3f5..7d7aff326e 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -19,6 +19,7 @@ import (
 	"log"
 	"net"
 	"reflect"
+	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -102,15 +103,39 @@ func (r *componentTypeRegistry) registerType(constructor func() component) error
 	return nil
 }
 
+var parameterErrRe = regexp.MustCompile(`field (\S+) not found in type \S+`)
+
 func (r *componentTypeRegistry) unmarshalComponentYaml(inner *interface{}, unmarshal func(interface{}) error) error {
 	c := configComponent{}
 	unmarshal(&c) // Get the type; ignore the error
 	f := r.TypeMap[c.Type()]
 	if f == nil {
-		return fmt.Errorf("Unknown %s %s type %q", r.Subagent, r.Kind, c.Type())
+		// TODO: propagate platform into the validation code and use the supported types per platform.
+		supportedTypes := sortedKeys(r.TypeMap)
+		return fmt.Errorf(`%s %s %q with type %q is not supported. Supported %s %s types: [%s].`,
+			r.Subagent, r.Kind, "???", c.Type(),
+			r.Subagent, r.Kind, strings.Join(supportedTypes, ", "))
+	}
+	o := f()
+	*inner = o
+	if err := unmarshal(*inner); err != nil {
+		m := parameterErrRe.FindStringSubmatchIndex(err.Error())
+		if m == nil {
+			return err
+		}
+		t := o.Type()
+		fields := collectYamlFields(o)
+		supportedParameters := []string{}
+		for _, f := range fields[1:] { // The first field is always "type".
+			supportedParameters = append(supportedParameters, f.Name)
+		}
+		n := err.Error()[m[2]:m[3]]
+		return fmt.Errorf(
+			`%s%s is not supported. Supported parameters: [%s].`, err.Error()[:m[0]],
+			parameterErrorPrefix(r.Subagent, r.Kind, "???", t, n),
+			strings.Join(supportedParameters, ", "))
 	}
-	*inner = f()
-	return unmarshal(*inner)
+	return nil
 }
 
 // Ops Agent logging config.
@@ -931,6 +956,10 @@ func mapKeys(m interface{}) map[string]bool {
 		for k := range m {
 			keys[k] = true
 		}
+	case map[string]func() component:
+		for k := range m {
+			keys[k] = true
+		}
 	default:
 		panic(fmt.Sprintf("Unknown type: %T", m))
 	}
diff --git a/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
index a9b6d3d93a..fdb78c8f77 100644
--- a/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: Unknown logging processor type "unsupported_type"
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: logging processor "???" with type "unsupported_type" is not supported. Supported logging processor types: [parse_json, parse_regex].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error
index e31d67aed5..895403d9d0 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field channels not found in type confgenerator.LoggingReceiverFiles
\ No newline at end of file
+  line 8: parameter "channels" in "files" type logging receiver "???" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error
index c3cce76ba1..9061bb5b7d 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field listen_host not found in type confgenerator.LoggingReceiverFiles
\ No newline at end of file
+  line 8: parameter "listen_host" in "files" type logging receiver "???" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error
index b27adabc01..d9f61d1301 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field listen_port not found in type confgenerator.LoggingReceiverFiles
\ No newline at end of file
+  line 8: parameter "listen_port" in "files" type logging receiver "???" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error
index c8851b03ee..8563cda76b 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field field_1 not found in type confgenerator.LoggingReceiverFiles
\ No newline at end of file
+  line 8: parameter "field_1" in "files" type logging receiver "???" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error
index 854454d3de..a398d0a28e 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field transport_protocol not found in type confgenerator.LoggingReceiverFiles
\ No newline at end of file
+  line 8: parameter "transport_protocol" in "files" type logging receiver "???" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error
index 43b582afee..1d54ff82a2 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field channels not found in type confgenerator.LoggingReceiverSyslog
\ No newline at end of file
+  line 8: parameter "channels" in "syslog" type logging receiver "???" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error
index 18dd4af940..57483ac3dc 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field exclude_paths not found in type confgenerator.LoggingReceiverSyslog
\ No newline at end of file
+  line 8: parameter "exclude_paths" in "syslog" type logging receiver "???" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error
index d6fa1c9d70..ae88dfa29a 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field include_paths not found in type confgenerator.LoggingReceiverSyslog
\ No newline at end of file
+  line 8: parameter "include_paths" in "syslog" type logging receiver "???" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
index a0fb365def..099756e0ab 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 8: field unsupported_parameter not found in type confgenerator.LoggingReceiverSyslog
\ No newline at end of file
+  line 8: parameter "unsupported_parameter" in "syslog" type logging receiver "???" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
index c1239deb57..1fc0925a82 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: Unknown logging receiver type "unsupported_type"
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: logging receiver "???" with type "unsupported_type" is not supported. Supported logging receiver types: [files, syslog, windows_event_log].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
index 794ceaf2ee..2e83522e58 100644
--- a/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: Unknown metrics processor type "unsupported_type"
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: metrics processor "???" with type "unsupported_type" is not supported. Supported metrics processor types: [exclude_metrics].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
index fc86952cee..18aedb0b95 100644
--- a/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: Unknown metrics receiver type "unsupported_type"
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: metrics receiver "???" with type "unsupported_type" is not supported. Supported metrics receiver types: [hostmetrics, iis, mssql].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error
index 55455817c4..ebcd19f457 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 6: field exclude_paths not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
+  line 6: parameter "exclude_paths" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error
index bf4b26635f..19e0d2fda2 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 6: field include_paths not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
+  line 6: parameter "include_paths" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error
index 4d68bd71c9..8abf1fd8bf 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 6: field listen_host not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
+  line 6: parameter "listen_host" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error
index 1c3456feb4..3def597521 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 6: field listen_port not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
+  line 6: parameter "listen_port" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error
index 191fc797ac..80f724c1d9 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 6: field unsupported_parameter not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
+  line 6: parameter "unsupported_parameter" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error
index e3cdc24243..5bdc67f994 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error
@@ -1,2 +1,2 @@
 the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
-  line 6: field transport_protocol not found in type confgenerator.LoggingReceiverWinevtlog
\ No newline at end of file
+  line 6: parameter "transport_protocol" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
index fc86952cee..18aedb0b95 100644
--- a/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: Unknown metrics receiver type "unsupported_type"
\ No newline at end of file
+the agent config file is not valid YAML. detailed error: metrics receiver "???" with type "unsupported_type" is not supported. Supported metrics receiver types: [hostmetrics, iis, mssql].
\ No newline at end of file

From 7e78333d39bdf945819f7b8530f47a4c184fc02a Mon Sep 17 00:00:00 2001
From: Igor Peshansky <igorp@google.com>
Date: Fri, 6 Aug 2021 13:01:06 -0400
Subject: [PATCH 41/41] Emit a more neutral error message on validation errors
 at unmarshal time.

---
 confgenerator/config.go                                         | 2 +-
 .../testdata/invalid/linux/all-not_in_yaml_format/golden_error  | 2 +-
 .../linux/logging-processor_unsupported_type/golden_error       | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../linux/logging-receiver_unsupported_type/golden_error        | 2 +-
 .../linux/metrics-processor_unsupported_type/golden_error       | 2 +-
 .../linux/metrics-receiver_unsupported_type/golden_error        | 2 +-
 .../invalid/windows/all-not_in_yaml_format/golden_error         | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../golden_error                                                | 2 +-
 .../windows/metrics-receiver_unsupported_type/golden_error      | 2 +-
 25 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/confgenerator/config.go b/confgenerator/config.go
index 7d7aff326e..c840ca6a96 100644
--- a/confgenerator/config.go
+++ b/confgenerator/config.go
@@ -57,7 +57,7 @@ func (uc *UnifiedConfig) DeepCopy() (UnifiedConfig, error) {
 func UnmarshalYamlToUnifiedConfig(input []byte) (UnifiedConfig, error) {
 	config := UnifiedConfig{}
 	if err := yaml.UnmarshalStrict(input, &config); err != nil {
-		return UnifiedConfig{}, fmt.Errorf("the agent config file is not valid YAML. detailed error: %s", err)
+		return UnifiedConfig{}, fmt.Errorf("the agent config file is not valid. detailed error: %s", err)
 	}
 	return config, nil
 }
diff --git a/confgenerator/testdata/invalid/linux/all-not_in_yaml_format/golden_error b/confgenerator/testdata/invalid/linux/all-not_in_yaml_format/golden_error
index f7a8def6dc..a7851222d7 100644
--- a/confgenerator/testdata/invalid/linux/all-not_in_yaml_format/golden_error
+++ b/confgenerator/testdata/invalid/linux/all-not_in_yaml_format/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: yaml: line 1: did not find expected key
\ No newline at end of file
+the agent config file is not valid. detailed error: yaml: line 1: did not find expected key
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
index fdb78c8f77..afa276fc40 100644
--- a/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-processor_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: logging processor "???" with type "unsupported_type" is not supported. Supported logging processor types: [parse_json, parse_regex].
\ No newline at end of file
+the agent config file is not valid. detailed error: logging processor "???" with type "unsupported_type" is not supported. Supported logging processor types: [parse_json, parse_regex].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_invalid_parameter_include_paths_is_empty_string/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_invalid_parameter_include_paths_is_empty_string/golden_error
index a9c8a9ea08..9e1ad9dbb1 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_invalid_parameter_include_paths_is_empty_string/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_invalid_parameter_include_paths_is_empty_string/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 5: cannot unmarshal !!str `` into []string
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error
index 895403d9d0..31ef6db2f1 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_channels/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 8: parameter "channels" in "files" type logging receiver "???" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error
index 9061bb5b7d..2ff94fea40 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_host/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 8: parameter "listen_host" in "files" type logging receiver "???" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error
index d9f61d1301..035b87bc04 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_listen_port/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 8: parameter "listen_port" in "files" type logging receiver "???" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error
index 8563cda76b..3aa09743d0 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_random/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 8: parameter "field_1" in "files" type logging receiver "???" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error
index a398d0a28e..a7301390f1 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_files_type_unsupported_parameter_transport_protocol/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 8: parameter "transport_protocol" in "files" type logging receiver "???" is not supported. Supported parameters: [include_paths, exclude_paths].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_invalid_parameter_listen_port_is_not_an_int/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_invalid_parameter_listen_port_is_not_an_int/golden_error
index 76e78350d5..a7c5c54360 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_invalid_parameter_listen_port_is_not_an_int/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_invalid_parameter_listen_port_is_not_an_int/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 6: cannot unmarshal !!str `abc` into uint16
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error
index 1d54ff82a2..9b5cee8604 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_channels/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 8: parameter "channels" in "syslog" type logging receiver "???" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error
index 57483ac3dc..3f8ce6cea8 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_exclude_paths/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 8: parameter "exclude_paths" in "syslog" type logging receiver "???" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error
index ae88dfa29a..6481cbbc96 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_include_paths/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 8: parameter "include_paths" in "syslog" type logging receiver "???" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
index 099756e0ab..8387354c43 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_syslog_type_unsupported_parameter_random/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 8: parameter "unsupported_parameter" in "syslog" type logging receiver "???" is not supported. Supported parameters: [transport_protocol, listen_host, listen_port].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
index 1fc0925a82..60d34e643a 100644
--- a/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/logging-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: logging receiver "???" with type "unsupported_type" is not supported. Supported logging receiver types: [files, syslog, windows_event_log].
\ No newline at end of file
+the agent config file is not valid. detailed error: logging receiver "???" with type "unsupported_type" is not supported. Supported logging receiver types: [files, syslog, windows_event_log].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
index 2e83522e58..5f34735e91 100644
--- a/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/metrics-processor_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: metrics processor "???" with type "unsupported_type" is not supported. Supported metrics processor types: [exclude_metrics].
\ No newline at end of file
+the agent config file is not valid. detailed error: metrics processor "???" with type "unsupported_type" is not supported. Supported metrics processor types: [exclude_metrics].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
index 18aedb0b95..2eed15eeb2 100644
--- a/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/linux/metrics-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: metrics receiver "???" with type "unsupported_type" is not supported. Supported metrics receiver types: [hostmetrics, iis, mssql].
\ No newline at end of file
+the agent config file is not valid. detailed error: metrics receiver "???" with type "unsupported_type" is not supported. Supported metrics receiver types: [hostmetrics, iis, mssql].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/all-not_in_yaml_format/golden_error b/confgenerator/testdata/invalid/windows/all-not_in_yaml_format/golden_error
index 3272ae8938..8c79dea5b7 100644
--- a/confgenerator/testdata/invalid/windows/all-not_in_yaml_format/golden_error
+++ b/confgenerator/testdata/invalid/windows/all-not_in_yaml_format/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 1: cannot unmarshal !!str `sdfd` into confgenerator.UnifiedConfig
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error
index ebcd19f457..986936820f 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_exclude_paths/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 6: parameter "exclude_paths" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error
index 19e0d2fda2..c3dd097f6a 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_include_paths/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 6: parameter "include_paths" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error
index 8abf1fd8bf..b5c801b19e 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_host/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 6: parameter "listen_host" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error
index 3def597521..4dd1127e35 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_listen_port/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 6: parameter "listen_port" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error
index 80f724c1d9..48eee4eab2 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_random/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 6: parameter "unsupported_parameter" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error
index 5bdc67f994..1a9755daab 100644
--- a/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error
+++ b/confgenerator/testdata/invalid/windows/logging-receiver_windows_event_log_type_unsupported_parameter_transport_protocol/golden_error
@@ -1,2 +1,2 @@
-the agent config file is not valid YAML. detailed error: yaml: unmarshal errors:
+the agent config file is not valid. detailed error: yaml: unmarshal errors:
   line 6: parameter "transport_protocol" in "windows_event_log" type logging receiver "???" is not supported. Supported parameters: [channels].
\ No newline at end of file
diff --git a/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error b/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
index 18aedb0b95..2eed15eeb2 100644
--- a/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
+++ b/confgenerator/testdata/invalid/windows/metrics-receiver_unsupported_type/golden_error
@@ -1 +1 @@
-the agent config file is not valid YAML. detailed error: metrics receiver "???" with type "unsupported_type" is not supported. Supported metrics receiver types: [hostmetrics, iis, mssql].
\ No newline at end of file
+the agent config file is not valid. detailed error: metrics receiver "???" with type "unsupported_type" is not supported. Supported metrics receiver types: [hostmetrics, iis, mssql].
\ No newline at end of file