From 94eb8f2e429c66c78c3bbef425e0fb7e13a2ff96 Mon Sep 17 00:00:00 2001 From: Helen Weller <38860767+helenosheaa@users.noreply.github.com> Date: Thu, 17 Dec 2020 19:32:25 -0500 Subject: [PATCH] Add wildcard tags json parser support (#8579) --- plugins/parsers/json/README.md | 6 +- plugins/parsers/json/parser.go | 44 +-- plugins/parsers/json/parser_test.go | 419 +++++++++++++++++++++++++++- 3 files changed, 446 insertions(+), 23 deletions(-) diff --git a/plugins/parsers/json/README.md b/plugins/parsers/json/README.md index 01ddf673e..682a0c62b 100644 --- a/plugins/parsers/json/README.md +++ b/plugins/parsers/json/README.md @@ -30,10 +30,12 @@ ignored unless specified in the `tag_key` or `json_string_fields` options. json_query = "" ## Tag keys is an array of keys that should be added as tags. Matching keys - ## are no longer saved as fields. + ## are no longer saved as fields. Supports wildcard glob matching. tag_keys = [ "my_tag_1", - "my_tag_2" + "my_tag_2", + "tags_*", + "tag*" ] ## Array of glob pattern strings or booleans keys that should be added as string fields. diff --git a/plugins/parsers/json/parser.go b/plugins/parsers/json/parser.go index bd9dee869..e8a748e70 100644 --- a/plugins/parsers/json/parser.go +++ b/plugins/parsers/json/parser.go @@ -36,7 +36,7 @@ type Config struct { type Parser struct { metricName string - tagKeys []string + tagKeys filter.Filter stringFields filter.Filter nameKey string query string @@ -53,9 +53,14 @@ func New(config *Config) (*Parser, error) { return nil, err } + tagKeyFilter, err := filter.Compile(config.TagKeys) + if err != nil { + return nil, err + } + return &Parser{ metricName: config.MetricName, - tagKeys: config.TagKeys, + tagKeys: tagKeyFilter, nameKey: config.NameKey, stringFields: stringFilter, query: config.Query, @@ -104,7 +109,7 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) ( name := p.metricName - //checks if json_name_key is set + // checks if json_name_key is set if p.nameKey != "" { switch field := f.Fields[p.nameKey].(type) { case string: @@ -112,7 +117,7 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) ( } } - //if time key is specified, set timestamp to it + // if time key is specified, set timestamp to it if p.timeKey != "" { if p.timeFormat == "" { err := fmt.Errorf("use of 'json_time_key' requires 'json_time_format'") @@ -131,7 +136,7 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) ( delete(f.Fields, p.timeKey) - //if the year is 0, set to current year + // if the year is 0, set to current year if timestamp.Year() == 0 { timestamp = timestamp.AddDate(time.Now().Year(), 0, 0) } @@ -145,32 +150,37 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) ( return []telegraf.Metric{metric}, nil } -//will take in field map with strings and bools, -//search for TagKeys that match fieldnames and add them to tags -//will delete any strings/bools that shouldn't be fields -//assumes that any non-numeric values in TagKeys should be displayed as tags +// will take in field map with strings and bools, +// search for TagKeys that match fieldnames and add them to tags +// will delete any strings/bools that shouldn't be fields +// assumes that any non-numeric values in TagKeys should be displayed as tags func (p *Parser) switchFieldToTag(tags map[string]string, fields map[string]interface{}) (map[string]string, map[string]interface{}) { - for _, name := range p.tagKeys { - //switch any fields in tagkeys into tags - if fields[name] == nil { + + for name, value := range fields { + if p.tagKeys == nil { continue } - switch value := fields[name].(type) { + // skip switch statement if tagkey doesn't match fieldname + if !p.tagKeys.Match(name) { + continue + } + // switch any fields in tagkeys into tags + switch t := value.(type) { case string: - tags[name] = value + tags[name] = t delete(fields, name) case bool: - tags[name] = strconv.FormatBool(value) + tags[name] = strconv.FormatBool(t) delete(fields, name) case float64: - tags[name] = strconv.FormatFloat(value, 'f', -1, 64) + tags[name] = strconv.FormatFloat(t, 'f', -1, 64) delete(fields, name) default: log.Printf("E! [parsers.json] Unrecognized type %T", value) } } - //remove any additional string/bool values from fields + // remove any additional string/bool values from fields for fk := range fields { switch fields[fk].(type) { case string, bool: diff --git a/plugins/parsers/json/parser_test.go b/plugins/parsers/json/parser_test.go index 31c507e75..525c8fd28 100644 --- a/plugins/parsers/json/parser_test.go +++ b/plugins/parsers/json/parser_test.go @@ -24,10 +24,14 @@ const validJSONTags = ` { "a": 5, "b": { - "c": 6 + "c": 6 }, "mytag": "foobar", - "othertag": "baz" + "othertag": "baz", + "tags_object": { + "mytag": "foobar", + "othertag": "baz" + } } ` @@ -39,7 +43,16 @@ const validJSONArrayTags = ` "c": 6 }, "mytag": "foo", - "othertag": "baz" + "othertag": "baz", + "tags_array": [ + { + "mytag": "foo" + }, + { + "othertag": "baz" + } + ], + "anothert": "foo" }, { "a": 7, @@ -47,8 +60,17 @@ const validJSONArrayTags = ` "c": 8 }, "mytag": "bar", + "othertag": "baz", + "tags_array": [ + { + "mytag": "bar" + }, + { "othertag": "baz" -} + } + ], + "anothert": "bar" + } ] ` @@ -948,3 +970,392 @@ func TestParse(t *testing.T) { }) } } + +func TestParseWithWildcardTagKeys(t *testing.T) { + var tests = []struct { + name string + config *Config + input []byte + expected []telegraf.Metric + }{ + { + name: "wildcard matching with tags nested within object", + config: &Config{ + MetricName: "json_test", + TagKeys: []string{"tags_object_*"}, + }, + input: []byte(validJSONTags), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json_test", + map[string]string{ + "tags_object_mytag": "foobar", + "tags_object_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + }, + }, + { + name: "wildcard matching with keys containing tag", + config: &Config{ + MetricName: "json_test", + TagKeys: []string{"*tag"}, + }, + input: []byte(validJSONTags), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json_test", + map[string]string{ + "mytag": "foobar", + "othertag": "baz", + "tags_object_mytag": "foobar", + "tags_object_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + }, + }, + { + name: "strings not matching tag keys are still also ignored", + config: &Config{ + MetricName: "json_test", + TagKeys: []string{"wrongtagkey", "tags_object_*"}, + }, + input: []byte(validJSONTags), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json_test", + map[string]string{ + "tags_object_mytag": "foobar", + "tags_object_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + }, + }, + { + name: "single tag key is also found and applied", + config: &Config{ + MetricName: "json_test", + TagKeys: []string{"mytag", "tags_object_*"}, + }, + input: []byte(validJSONTags), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json_test", + map[string]string{ + "mytag": "foobar", + "tags_object_mytag": "foobar", + "tags_object_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser, err := New(tt.config) + require.NoError(t, err) + + actual, err := parser.Parse(tt.input) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime()) + }) + } +} + +func TestParseLineWithWildcardTagKeys(t *testing.T) { + var tests = []struct { + name string + config *Config + input string + expected telegraf.Metric + }{ + { + name: "wildcard matching with tags nested within object", + config: &Config{ + MetricName: "json_test", + TagKeys: []string{"tags_object_*"}, + }, + input: validJSONTags, + expected: testutil.MustMetric( + "json_test", + map[string]string{ + "tags_object_mytag": "foobar", + "tags_object_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + }, + { + name: "wildcard matching with keys containing tag", + config: &Config{ + MetricName: "json_test", + TagKeys: []string{"*tag"}, + }, + input: validJSONTags, + expected: testutil.MustMetric( + "json_test", + map[string]string{ + "mytag": "foobar", + "othertag": "baz", + "tags_object_mytag": "foobar", + "tags_object_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + }, + { + name: "strings not matching tag keys are ignored", + config: &Config{ + MetricName: "json_test", + TagKeys: []string{"wrongtagkey", "tags_object_*"}, + }, + input: validJSONTags, + expected: testutil.MustMetric( + "json_test", + map[string]string{ + "tags_object_mytag": "foobar", + "tags_object_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + }, + { + name: "single tag key is also found and applied", + config: &Config{ + MetricName: "json_test", + TagKeys: []string{"mytag", "tags_object_*"}, + }, + input: validJSONTags, + expected: testutil.MustMetric( + "json_test", + map[string]string{ + "mytag": "foobar", + "tags_object_mytag": "foobar", + "tags_object_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser, err := New(tt.config) + require.NoError(t, err) + + actual, err := parser.ParseLine(tt.input) + require.NoError(t, err) + + testutil.RequireMetricEqual(t, tt.expected, actual, testutil.IgnoreTime()) + }) + } +} + +func TestParseArrayWithWildcardTagKeys(t *testing.T) { + var tests = []struct { + name string + config *Config + input []byte + expected []telegraf.Metric + }{ + { + name: "wildcard matching with keys containing tag within array works", + config: &Config{ + MetricName: "json_array_test", + TagKeys: []string{"*tag"}, + }, + input: []byte(validJSONArrayTags), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json_array_test", + map[string]string{ + "mytag": "foo", + "othertag": "baz", + "tags_array_0_mytag": "foo", + "tags_array_1_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "json_array_test", + map[string]string{ + "mytag": "bar", + "othertag": "baz", + "tags_array_0_mytag": "bar", + "tags_array_1_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(7), + "b_c": float64(8), + }, + time.Unix(0, 0), + ), + }, + }, + { + name: " wildcard matching with tags nested array within object works", + config: &Config{ + MetricName: "json_array_test", + TagKeys: []string{"tags_array_*"}, + }, + input: []byte(validJSONArrayTags), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json_array_test", + map[string]string{ + "tags_array_0_mytag": "foo", + "tags_array_1_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "json_array_test", + map[string]string{ + "tags_array_0_mytag": "bar", + "tags_array_1_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(7), + "b_c": float64(8), + }, + time.Unix(0, 0), + ), + }, + }, + { + name: "strings not matching tag keys are still also ignored", + config: &Config{ + MetricName: "json_array_test", + TagKeys: []string{"mytag", "*tag"}, + }, + input: []byte(validJSONArrayTags), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json_array_test", + map[string]string{ + "mytag": "foo", + "othertag": "baz", + "tags_array_0_mytag": "foo", + "tags_array_1_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "json_array_test", + map[string]string{ + "mytag": "bar", + "othertag": "baz", + "tags_array_0_mytag": "bar", + "tags_array_1_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(7), + "b_c": float64(8), + }, + time.Unix(0, 0), + ), + }, + }, + { + name: "single tag key is also found and applied", + config: &Config{ + MetricName: "json_array_test", + TagKeys: []string{"anothert", "*tag"}, + }, + input: []byte(validJSONArrayTags), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json_array_test", + map[string]string{ + "anothert": "foo", + "mytag": "foo", + "othertag": "baz", + "tags_array_0_mytag": "foo", + "tags_array_1_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(5), + "b_c": float64(6), + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "json_array_test", + map[string]string{ + "anothert": "bar", + "mytag": "bar", + "othertag": "baz", + "tags_array_0_mytag": "bar", + "tags_array_1_othertag": "baz", + }, + map[string]interface{}{ + "a": float64(7), + "b_c": float64(8), + }, + time.Unix(0, 0), + ), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser, err := New(tt.config) + require.NoError(t, err) + + actual, err := parser.Parse(tt.input) + require.NoError(t, err) + + testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime()) + }) + } + +}