feat: syslog-ng custom metrics, add logging to output metrics as well

Signed-off-by: Peter Wilcsinszky <[email protected]>
kube-logging · Oct 31, 2023 · e00a6de · e00a6de
1 parent 69cf2af
commit e00a6de
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 6 deletions.
diff --git a/pkg/sdk/logging/model/syslogng/config/config.go b/pkg/sdk/logging/model/syslogng/config/config.go
@@ -100,13 +100,13 @@ func configRenderer(in Input) (render.Renderer, error) {
 		if err := validateClusterOutputs(clusterOutputRefs, client.ObjectKeyFromObject(&cf).String(), cf.Spec.GlobalOutputRefs); err != nil {
 			errs = errors.Append(errs, err)
 		}
-		logDefs = append(logDefs, renderClusterFlow(clusterOutputRefs, sourceName, cf, in.SecretLoaderFactory))
+		logDefs = append(logDefs, renderClusterFlow(in.Logging.Name, clusterOutputRefs, sourceName, cf, in.SecretLoaderFactory))
 	}
 	for _, f := range in.Flows {
 		if err := validateClusterOutputs(clusterOutputRefs, client.ObjectKeyFromObject(&f).String(), f.Spec.GlobalOutputRefs); err != nil {
 			errs = errors.Append(errs, err)
 		}
-		logDefs = append(logDefs, renderFlow(clusterOutputRefs, sourceName, keyDelim(in.Logging.Spec.SyslogNGSpec.JSONKeyDelimiter), f, in.SecretLoaderFactory))
+		logDefs = append(logDefs, renderFlow(in.Logging.Name, clusterOutputRefs, sourceName, keyDelim(in.Logging.Spec.SyslogNGSpec.JSONKeyDelimiter), f, in.SecretLoaderFactory))
 	}
 
 	if in.Logging.Spec.SyslogNGSpec.JSONKeyPrefix == "" {

diff --git a/pkg/sdk/logging/model/syslogng/config/config_test.go b/pkg/sdk/logging/model/syslogng/config/config_test.go
@@ -266,7 +266,8 @@ log {
     log {
         parser {
             metrics-probe(key("example") labels(
-                "output_name" => "test-syslog-out-global"
+                "logging" => "test"
+				"output_name" => "test-syslog-out-global"
 				"output_namespace" => "config-test"
 				"output_scope" => "global"
             ));
@@ -276,6 +277,7 @@ log {
     log {
         parser {
             metrics-probe(key("example") labels(
+				"logging" => "test"
                 "output_name" => "test-syslog-out"
 				"output_namespace" => "default"
 				"output_scope" => "local"
@@ -1001,9 +1003,11 @@ source "main_input" {
             json-parser(prefix("json."));
             metrics-probe(key("example") labels(
 				"a" => "b"
+				"logging" => "test"
 			) level(2));
             metrics-probe(key("example2") labels(
 				"c" => "d"
+				"logging" => "test"
 			) level(3));
         };
     };

diff --git a/pkg/sdk/logging/model/syslogng/config/flow.go b/pkg/sdk/logging/model/syslogng/config/flow.go
@@ -41,7 +41,7 @@ func validateClusterOutputs(clusterOutputRefs map[string]types.NamespacedName, f
 	})
 }
 
-func renderClusterFlow(clusterOutputRefs map[string]types.NamespacedName, sourceName string, f v1beta1.SyslogNGClusterFlow, secretLoaderFactory SecretLoaderFactory) render.Renderer {
+func renderClusterFlow(logging string, clusterOutputRefs map[string]types.NamespacedName, sourceName string, f v1beta1.SyslogNGClusterFlow, secretLoaderFactory SecretLoaderFactory) render.Renderer {
 	baseName := fmt.Sprintf("clusterflow_%s_%s", f.Namespace, f.Name)
 	matchName := fmt.Sprintf("%s_match", baseName)
 	filterDefs := seqs.MapWithIndex(seqs.FromSlice(f.Spec.Filters), func(idx int, flt v1beta1.SyslogNGFilter) render.Renderer {
@@ -62,6 +62,7 @@ func renderClusterFlow(clusterOutputRefs map[string]types.NamespacedName, source
 			)),
 			seqs.ToSlice(seqs.Map(seqs.FromSlice(f.Spec.GlobalOutputRefs), func(ref string) destination {
 				return destination{
+					logging:          logging,
 					renderedDestName: clusterOutputDestName(clusterOutputRefs[ref].Namespace, ref),
 					namespace:        clusterOutputRefs[ref].Namespace,
 					name:             ref,
@@ -73,7 +74,7 @@ func renderClusterFlow(clusterOutputRefs map[string]types.NamespacedName, source
 	)
 }
 
-func renderFlow(clusterOutputRefs map[string]types.NamespacedName, sourceName string, keyDelim string, f v1beta1.SyslogNGFlow, secretLoaderFactory SecretLoaderFactory) render.Renderer {
+func renderFlow(logging string, clusterOutputRefs map[string]types.NamespacedName, sourceName string, keyDelim string, f v1beta1.SyslogNGFlow, secretLoaderFactory SecretLoaderFactory) render.Renderer {
 	baseName := fmt.Sprintf("flow_%s_%s", f.Namespace, f.Name)
 	matchName := fmt.Sprintf("%s_match", baseName)
 	nsFilterName := fmt.Sprintf("%s_ns_filter", baseName)
@@ -102,6 +103,7 @@ func renderFlow(clusterOutputRefs map[string]types.NamespacedName, sourceName st
 			seqs.ToSlice(seqs.Concat(
 				seqs.Map(seqs.FromSlice(f.Spec.GlobalOutputRefs), func(ref string) destination {
 					return destination{
+						logging:          logging,
 						renderedDestName: clusterOutputDestName(clusterOutputRefs[ref].Namespace, ref),
 						namespace:        clusterOutputRefs[ref].Namespace,
 						name:             ref,
@@ -111,6 +113,7 @@ func renderFlow(clusterOutputRefs map[string]types.NamespacedName, sourceName st
 				}),
 				seqs.Map(seqs.FromSlice(f.Spec.LocalOutputRefs), func(ref string) destination {
 					return destination{
+						logging:          logging,
 						renderedDestName: outputDestName(f.Namespace, ref),
 						namespace:        f.Namespace,
 						name:             ref,

diff --git a/pkg/sdk/logging/model/syslogng/config/flow_test.go b/pkg/sdk/logging/model/syslogng/config/flow_test.go
@@ -101,7 +101,7 @@ parser("clusterflow_test_ns_test_clusterflow_filters_0");
 		testCase := testCase
 		t.Run(name, func(t *testing.T) {
 			out := strings.Builder{}
-			require.NoError(t, renderClusterFlow(nil, "test_input", testCase.clusterFlow, &TestSecretLoaderFactory{})(render.RenderContext{
+			require.NoError(t, renderClusterFlow("", nil, "test_input", testCase.clusterFlow, &TestSecretLoaderFactory{})(render.RenderContext{
 				Out: &out,
 			}))
 			assert.Equal(t, testCase.expected, out.String())

diff --git a/pkg/sdk/logging/model/syslogng/config/log.go b/pkg/sdk/logging/model/syslogng/config/log.go
@@ -31,6 +31,7 @@ const (
 )
 
 type destination struct {
+	logging          string
 	renderedDestName string
 	namespace        string
 	name             string
@@ -68,6 +69,7 @@ func destinationLogPath(dest destination) render.Renderer {
 		m.Labels["output_name"] = dest.name
 		m.Labels["output_namespace"] = dest.namespace
 		m.Labels["output_scope"] = string(dest.scope)
+		m.Labels["logging"] = dest.logging
 		metricsProbesRenderer = append(metricsProbesRenderer, renderDriver(Field{
 			Value: reflect.ValueOf(m),
 		}, nil))