fix(sourceprocessor): don't prevent using attributes from Sumo schema…

… in source templates (#227)

pkg/exporter/sumologicexporter/README.md

@@ -159,6 +159,36 @@ Below is a list of all attribute keys that are being translated.
 
 ## Source Templates
 
+> **IMPORTANT NOTE**:
+>
+> When using non-`OTLP` based format (e.g. `JSON` for logs) metadata attributes
+> used in source templates have to have a regex defined in
+> `metadata_attributes` that would match them.
+>
+> Otherwise the attributes will not be available during source templates rendering.
+> Hence this is correct:
+>
+> ```
+> source_name: "%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}"
+> source_category: "%{k8s.namespace.name}/%{k8s.pod.pod_name}"
+> source_host: '%{k8s.pod.hostname}'
+> metadata_attributes:
+>   - k8s.*
+>   - some_other_metadata_regex.*
+> ```
+>
+> While is **not**:
+>
+> ```
+> source_name: "%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}"
+> source_category: "%{k8s.namespace.name}/%{k8s.pod.pod_name}"
+> source_host: '%{k8s.pod.hostname}'
+> metadata_attributes:
+>   - host
+>   - pod
+>   - some_other_metadata_regex.*
+> ```
+
 You can specify a template with an attribute for `source_category`, `source_name`,
 `source_host` or `graphite_template` using `%{attr_name}`.
 
@@ -194,7 +224,7 @@ exporters:
   sumologic:
     source_category: "custom category"
     source_name: "custom name"
-    source_host: "custom host"
+    source_host: "%{k8s.pod.name}"
     metadata_attributes:
       - k8s.*
 

pkg/exporter/sumologicexporter/exporter_test.go

@@ -427,7 +427,11 @@ func TestPushTextLogsWithAttributeTranslation(t *testing.T) {
 			assert.Equal(t, `Example log`, body)
 			assert.Equal(t, "host=harry-potter", req.Header.Get("X-Sumo-Fields"))
 			assert.Equal(t, "harry-potter", req.Header.Get("X-Sumo-Category"))
-			assert.Equal(t, "undefined", req.Header.Get("X-Sumo-Host"))
+
+			// This gets the value from 'host.name' because we do not disallow
+			// using Sumo schema and 'host.name' from OT convention
+			// translates into 'host' in Sumo convention
+			assert.Equal(t, "harry-potter", req.Header.Get("X-Sumo-Host"))
 		},
 	}
 	test := prepareExporterTest(t, config, expectedRequests)
@@ -479,7 +483,11 @@ func TestPushJSONLogsWithAttributeTranslation(t *testing.T) {
 			assert.Equal(t, `{"InstanceType":"wizard","host":"harry-potter","log":"Example log"}`, body)
 			assert.Equal(t, "host=harry-potter", req.Header.Get("X-Sumo-Fields"))
 			assert.Equal(t, "harry-potter", req.Header.Get("X-Sumo-Category"))
-			assert.Equal(t, "undefined", req.Header.Get("X-Sumo-Host"))
+
+			// This gets the value from 'host.name' because we do not disallow
+			// using Sumo schema and 'host.name' from OT convention
+			// translates into 'host' in Sumo convention.
+			assert.Equal(t, "harry-potter", req.Header.Get("X-Sumo-Host"))
 		},
 	}
 	test := prepareExporterTest(t, config, expectedRequests)
@@ -889,7 +897,11 @@ func TestPushMetrics_AttributeTranslation(t *testing.T) {
 			expectedHeaders: map[string]string{
 				"Content-Type":    "application/vnd.sumologic.prometheus",
 				"X-Sumo-Category": "harry-potter",
-				"X-Sumo-Host":     "undefined",
+
+				// This gets the value from 'host.name' because we do not disallow
+				// using Sumo schema and 'host.name' from OT convention
+				// translates into 'host' in Sumo convention
+				"X-Sumo-Host": "harry-potter",
 			},
 			expectedBody: `test.metric.data{test="test_value",test2="second_value",host="harry-potter",InstanceType="wizard"} 14500 1605534165000`,
 		},

pkg/exporter/sumologicexporter/translate_attributes.go

@@ -62,10 +62,11 @@ func translateAttributes(attributes pdata.AttributeMap) {
 }
 
 // translateConfigValue renames attribute keys in config values according to attributeTranslations.
+// Example:
+// * '%{k8s.container.name}' would translate to '%{container}'
+// * '%{k8s.pod.name}-%{custom_attr}' would translate to '%{pod}-%{custom_attr}'
+// * '%{pod}' would translate to '%{pod}'
 func translateConfigValue(value string) string {
-	for _, sumoKey := range attributeTranslations {
-		value = strings.ReplaceAll(value, fmt.Sprintf("%%{%v}", sumoKey), unrecognizedAttributeValue)
-	}
 	for otKey, sumoKey := range attributeTranslations {
 		value = strings.ReplaceAll(value, fmt.Sprintf("%%{%v}", otKey), fmt.Sprintf("%%{%v}", sumoKey))
 	}

pkg/exporter/sumologicexporter/translate_attributes_test.go

@@ -127,7 +127,31 @@ func assertAttribute(t *testing.T, metadata pdata.AttributeMap, attributeName st
 }
 
 func TestTranslateConfigValue(t *testing.T) {
-	translatedValue := translateConfigValue("%{k8s.pod.name}-%{host.name}/%{pod}-%{host}")
+	testcases := []struct {
+		name  string
+		input string
+		want  string
+	}{
+		{
+			name:  "basic",
+			input: "%{k8s.pod.name}-%{host.name}",
+			want:  "%{pod}-%{host}",
+		},
+		{
+			name:  "basic with sumo convention tags",
+			input: "%{k8s.pod.name}-%{host.name}/%{pod}-%{host}",
+			want:  "%{pod}-%{host}/%{pod}-%{host}",
+		},
+		{
+			name:  "custom attributes",
+			input: "%{_sourceCategory}-%{my_custom_vendor_attr}",
+			want:  "%{_sourceCategory}-%{my_custom_vendor_attr}",
+		},
+	}
 
-	assert.Equal(t, "%{pod}-%{host}/undefined-undefined", translatedValue)
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, translateConfigValue(tc.input))
+		})
+	}
 }