Add wildcard tags json parser support (#8579)

plugins/parsers/json/README.md

@@ -30,10 +30,12 @@ ignored unless specified in the `tag_key` or `json_string_fields` options.
   json_query = ""
 
   ## Tag keys is an array of keys that should be added as tags.  Matching keys
-  ## are no longer saved as fields.
+  ## are no longer saved as fields. Supports wildcard glob matching.
   tag_keys = [
     "my_tag_1",
-    "my_tag_2"
+    "my_tag_2",
+    "tags_*",
+    "tag*"
   ]
 
   ## Array of glob pattern strings or booleans keys that should be added as string fields.

plugins/parsers/json/parser.go

@@ -36,7 +36,7 @@ type Config struct {
 
 type Parser struct {
 	metricName   string
-	tagKeys      []string
+	tagKeys      filter.Filter
 	stringFields filter.Filter
 	nameKey      string
 	query        string
@@ -53,9 +53,14 @@ func New(config *Config) (*Parser, error) {
 		return nil, err
 	}
 
+	tagKeyFilter, err := filter.Compile(config.TagKeys)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Parser{
 		metricName:   config.MetricName,
-		tagKeys:      config.TagKeys,
+		tagKeys:      tagKeyFilter,
 		nameKey:      config.NameKey,
 		stringFields: stringFilter,
 		query:        config.Query,
@@ -104,7 +109,7 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) (
 
 	name := p.metricName
 
-	//checks if json_name_key is set
+	// checks if json_name_key is set
 	if p.nameKey != "" {
 		switch field := f.Fields[p.nameKey].(type) {
 		case string:
@@ -112,7 +117,7 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) (
 		}
 	}
 
-	//if time key is specified, set timestamp to it
+	// if time key is specified, set timestamp to it
 	if p.timeKey != "" {
 		if p.timeFormat == "" {
 			err := fmt.Errorf("use of 'json_time_key' requires 'json_time_format'")
@@ -131,7 +136,7 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) (
 
 		delete(f.Fields, p.timeKey)
 
-		//if the year is 0, set to current year
+		// if the year is 0, set to current year
 		if timestamp.Year() == 0 {
 			timestamp = timestamp.AddDate(time.Now().Year(), 0, 0)
 		}
@@ -145,32 +150,37 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) (
 	return []telegraf.Metric{metric}, nil
 }
 
-//will take in field map with strings and bools,
+// will take in field map with strings and bools,
-//search for TagKeys that match fieldnames and add them to tags
+// search for TagKeys that match fieldnames and add them to tags
-//will delete any strings/bools that shouldn't be fields
+// will delete any strings/bools that shouldn't be fields
-//assumes that any non-numeric values in TagKeys should be displayed as tags
+// assumes that any non-numeric values in TagKeys should be displayed as tags
 func (p *Parser) switchFieldToTag(tags map[string]string, fields map[string]interface{}) (map[string]string, map[string]interface{}) {
-	for _, name := range p.tagKeys {
+
-		//switch any fields in tagkeys into tags
+	for name, value := range fields {
-		if fields[name] == nil {
+		if p.tagKeys == nil {
 			continue
 		}
-		switch value := fields[name].(type) {
+		// skip switch statement if tagkey doesn't match fieldname
+		if !p.tagKeys.Match(name) {
+			continue
+		}
+		// switch any fields in tagkeys into tags
+		switch t := value.(type) {
 		case string:
-			tags[name] = value
+			tags[name] = t
 			delete(fields, name)
 		case bool:
-			tags[name] = strconv.FormatBool(value)
+			tags[name] = strconv.FormatBool(t)
 			delete(fields, name)
 		case float64:
-			tags[name] = strconv.FormatFloat(value, 'f', -1, 64)
+			tags[name] = strconv.FormatFloat(t, 'f', -1, 64)
 			delete(fields, name)
 		default:
 			log.Printf("E! [parsers.json] Unrecognized type %T", value)
 		}
 	}
 
-	//remove any additional string/bool values from fields
+	// remove any additional string/bool values from fields
 	for fk := range fields {
 		switch fields[fk].(type) {
 		case string, bool:

plugins/parsers/json/parser_test.go

@@ -24,10 +24,14 @@ const validJSONTags = `
 {
     "a": 5,
     "b": {
-        "c": 6
+      "c": 6
     },
     "mytag": "foobar",
-    "othertag": "baz"
+    "othertag": "baz",
+    "tags_object": {
+        "mytag": "foobar",
+        "othertag": "baz"
+    }
 }
 `
 
@@ -39,7 +43,16 @@ const validJSONArrayTags = `
         "c": 6
     },
     "mytag": "foo",
-    "othertag": "baz"
+    "othertag": "baz",
+    "tags_array": [
+        {
+        "mytag": "foo"
+        },
+        {
+        "othertag": "baz"
+        }
+    ],
+    "anothert": "foo"
 },
 {
     "a": 7,
@@ -47,8 +60,17 @@ const validJSONArrayTags = `
         "c": 8
     },
     "mytag": "bar",
+    "othertag": "baz",
+    "tags_array": [
+    {
+    "mytag": "bar"
+    },
+    {
     "othertag": "baz"
-}
+    }
+    ],
+    "anothert": "bar"
+    }
 ]
 `
 
@@ -948,3 +970,392 @@ func TestParse(t *testing.T) {
 		})
 	}
 }
+
+func TestParseWithWildcardTagKeys(t *testing.T) {
+	var tests = []struct {
+		name     string
+		config   *Config
+		input    []byte
+		expected []telegraf.Metric
+	}{
+		{
+			name: "wildcard matching with tags nested within object",
+			config: &Config{
+				MetricName: "json_test",
+				TagKeys:    []string{"tags_object_*"},
+			},
+			input: []byte(validJSONTags),
+			expected: []telegraf.Metric{
+				testutil.MustMetric(
+					"json_test",
+					map[string]string{
+						"tags_object_mytag":    "foobar",
+						"tags_object_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(5),
+						"b_c": float64(6),
+					},
+					time.Unix(0, 0),
+				),
+			},
+		},
+		{
+			name: "wildcard matching with keys containing tag",
+			config: &Config{
+				MetricName: "json_test",
+				TagKeys:    []string{"*tag"},
+			},
+			input: []byte(validJSONTags),
+			expected: []telegraf.Metric{
+				testutil.MustMetric(
+					"json_test",
+					map[string]string{
+						"mytag":                "foobar",
+						"othertag":             "baz",
+						"tags_object_mytag":    "foobar",
+						"tags_object_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(5),
+						"b_c": float64(6),
+					},
+					time.Unix(0, 0),
+				),
+			},
+		},
+		{
+			name: "strings not matching tag keys are still also ignored",
+			config: &Config{
+				MetricName: "json_test",
+				TagKeys:    []string{"wrongtagkey", "tags_object_*"},
+			},
+			input: []byte(validJSONTags),
+			expected: []telegraf.Metric{
+				testutil.MustMetric(
+					"json_test",
+					map[string]string{
+						"tags_object_mytag":    "foobar",
+						"tags_object_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(5),
+						"b_c": float64(6),
+					},
+					time.Unix(0, 0),
+				),
+			},
+		},
+		{
+			name: "single tag key is also found and applied",
+			config: &Config{
+				MetricName: "json_test",
+				TagKeys:    []string{"mytag", "tags_object_*"},
+			},
+			input: []byte(validJSONTags),
+			expected: []telegraf.Metric{
+				testutil.MustMetric(
+					"json_test",
+					map[string]string{
+						"mytag":                "foobar",
+						"tags_object_mytag":    "foobar",
+						"tags_object_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(5),
+						"b_c": float64(6),
+					},
+					time.Unix(0, 0),
+				),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			parser, err := New(tt.config)
+			require.NoError(t, err)
+
+			actual, err := parser.Parse(tt.input)
+			require.NoError(t, err)
+			testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime())
+		})
+	}
+}
+
+func TestParseLineWithWildcardTagKeys(t *testing.T) {
+	var tests = []struct {
+		name     string
+		config   *Config
+		input    string
+		expected telegraf.Metric
+	}{
+		{
+			name: "wildcard matching with tags nested within object",
+			config: &Config{
+				MetricName: "json_test",
+				TagKeys:    []string{"tags_object_*"},
+			},
+			input: validJSONTags,
+			expected: testutil.MustMetric(
+				"json_test",
+				map[string]string{
+					"tags_object_mytag":    "foobar",
+					"tags_object_othertag": "baz",
+				},
+				map[string]interface{}{
+					"a":   float64(5),
+					"b_c": float64(6),
+				},
+				time.Unix(0, 0),
+			),
+		},
+		{
+			name: "wildcard matching with keys containing tag",
+			config: &Config{
+				MetricName: "json_test",
+				TagKeys:    []string{"*tag"},
+			},
+			input: validJSONTags,
+			expected: testutil.MustMetric(
+				"json_test",
+				map[string]string{
+					"mytag":                "foobar",
+					"othertag":             "baz",
+					"tags_object_mytag":    "foobar",
+					"tags_object_othertag": "baz",
+				},
+				map[string]interface{}{
+					"a":   float64(5),
+					"b_c": float64(6),
+				},
+				time.Unix(0, 0),
+			),
+		},
+		{
+			name: "strings not matching tag keys are ignored",
+			config: &Config{
+				MetricName: "json_test",
+				TagKeys:    []string{"wrongtagkey", "tags_object_*"},
+			},
+			input: validJSONTags,
+			expected: testutil.MustMetric(
+				"json_test",
+				map[string]string{
+					"tags_object_mytag":    "foobar",
+					"tags_object_othertag": "baz",
+				},
+				map[string]interface{}{
+					"a":   float64(5),
+					"b_c": float64(6),
+				},
+				time.Unix(0, 0),
+			),
+		},
+		{
+			name: "single tag key is also found and applied",
+			config: &Config{
+				MetricName: "json_test",
+				TagKeys:    []string{"mytag", "tags_object_*"},
+			},
+			input: validJSONTags,
+			expected: testutil.MustMetric(
+				"json_test",
+				map[string]string{
+					"mytag":                "foobar",
+					"tags_object_mytag":    "foobar",
+					"tags_object_othertag": "baz",
+				},
+				map[string]interface{}{
+					"a":   float64(5),
+					"b_c": float64(6),
+				},
+				time.Unix(0, 0),
+			),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			parser, err := New(tt.config)
+			require.NoError(t, err)
+
+			actual, err := parser.ParseLine(tt.input)
+			require.NoError(t, err)
+
+			testutil.RequireMetricEqual(t, tt.expected, actual, testutil.IgnoreTime())
+		})
+	}
+}
+
+func TestParseArrayWithWildcardTagKeys(t *testing.T) {
+	var tests = []struct {
+		name     string
+		config   *Config
+		input    []byte
+		expected []telegraf.Metric
+	}{
+		{
+			name: "wildcard matching with keys containing tag within array works",
+			config: &Config{
+				MetricName: "json_array_test",
+				TagKeys:    []string{"*tag"},
+			},
+			input: []byte(validJSONArrayTags),
+			expected: []telegraf.Metric{
+				testutil.MustMetric(
+					"json_array_test",
+					map[string]string{
+						"mytag":                 "foo",
+						"othertag":              "baz",
+						"tags_array_0_mytag":    "foo",
+						"tags_array_1_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(5),
+						"b_c": float64(6),
+					},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric(
+					"json_array_test",
+					map[string]string{
+						"mytag":                 "bar",
+						"othertag":              "baz",
+						"tags_array_0_mytag":    "bar",
+						"tags_array_1_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(7),
+						"b_c": float64(8),
+					},
+					time.Unix(0, 0),
+				),
+			},
+		},
+		{
+			name: " wildcard matching with tags nested array within object works",
+			config: &Config{
+				MetricName: "json_array_test",
+				TagKeys:    []string{"tags_array_*"},
+			},
+			input: []byte(validJSONArrayTags),
+			expected: []telegraf.Metric{
+				testutil.MustMetric(
+					"json_array_test",
+					map[string]string{
+						"tags_array_0_mytag":    "foo",
+						"tags_array_1_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(5),
+						"b_c": float64(6),
+					},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric(
+					"json_array_test",
+					map[string]string{
+						"tags_array_0_mytag":    "bar",
+						"tags_array_1_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(7),
+						"b_c": float64(8),
+					},
+					time.Unix(0, 0),
+				),
+			},
+		},
+		{
+			name: "strings not matching tag keys are still also ignored",
+			config: &Config{
+				MetricName: "json_array_test",
+				TagKeys:    []string{"mytag", "*tag"},
+			},
+			input: []byte(validJSONArrayTags),
+			expected: []telegraf.Metric{
+				testutil.MustMetric(
+					"json_array_test",
+					map[string]string{
+						"mytag":                 "foo",
+						"othertag":              "baz",
+						"tags_array_0_mytag":    "foo",
+						"tags_array_1_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(5),
+						"b_c": float64(6),
+					},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric(
+					"json_array_test",
+					map[string]string{
+						"mytag":                 "bar",
+						"othertag":              "baz",
+						"tags_array_0_mytag":    "bar",
+						"tags_array_1_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(7),
+						"b_c": float64(8),
+					},
+					time.Unix(0, 0),
+				),
+			},
+		},
+		{
+			name: "single tag key is also found and applied",
+			config: &Config{
+				MetricName: "json_array_test",
+				TagKeys:    []string{"anothert", "*tag"},
+			},
+			input: []byte(validJSONArrayTags),
+			expected: []telegraf.Metric{
+				testutil.MustMetric(
+					"json_array_test",
+					map[string]string{
+						"anothert":              "foo",
+						"mytag":                 "foo",
+						"othertag":              "baz",
+						"tags_array_0_mytag":    "foo",
+						"tags_array_1_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(5),
+						"b_c": float64(6),
+					},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric(
+					"json_array_test",
+					map[string]string{
+						"anothert":              "bar",
+						"mytag":                 "bar",
+						"othertag":              "baz",
+						"tags_array_0_mytag":    "bar",
+						"tags_array_1_othertag": "baz",
+					},
+					map[string]interface{}{
+						"a":   float64(7),
+						"b_c": float64(8),
+					},
+					time.Unix(0, 0),
+				),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			parser, err := New(tt.config)
+			require.NoError(t, err)
+
+			actual, err := parser.Parse(tt.input)
+			require.NoError(t, err)
+
+			testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime())
+		})
+	}
+
+}