From fb5b541b1a3c97f0b9f280b86e62b1a86f0c8e64 Mon Sep 17 00:00:00 2001 From: Sven Rebhan <36194019+srebhan@users.noreply.github.com> Date: Thu, 4 Nov 2021 17:45:52 +0100 Subject: [PATCH] feat: Extend regexp processor do allow renaming of measurements, tags and fields (#9561) --- plugins/processors/regex/README.md | 36 +- plugins/processors/regex/regex.go | 216 +++++++- plugins/processors/regex/regex_test.go | 670 ++++++++++++++++++++++--- 3 files changed, 833 insertions(+), 89 deletions(-) diff --git a/plugins/processors/regex/README.md b/plugins/processors/regex/README.md index a6cef82a0..578ed13d0 100644 --- a/plugins/processors/regex/README.md +++ b/plugins/processors/regex/README.md @@ -4,6 +4,8 @@ The `regex` plugin transforms tag and field values with regex pattern. If `resul For tags transforms, if `append` is set to `true`, it will append the transformation to the existing tag value, instead of overwriting it. +For metrics transforms, `key` denotes the element that should be transformed. Furthermore, `result_key` allows control over the behavior applied in case the resulting `tag` or `field` name already exists. + ### Configuration: ```toml @@ -38,6 +40,38 @@ For tags transforms, if `append` is set to `true`, it will append the transforma pattern = ".*category=(\\w+).*" replacement = "${1}" result_key = "search_category" + + # Rename metric fields + [[processors.regex.field_rename]] + ## Regular expression to match on a field name + pattern = "^search_(\\w+)d$" + ## Matches of the pattern will be replaced with this string. Use ${1} + ## notation to use the text of the first submatch. + replacement = "${1}" + ## If the new field name already exists, you can either "overwrite" the + ## existing one with the value of the renamed field OR you can "keep" + ## both the existing and source field. + # result_key = "keep" + + # Rename metric tags + # [[processors.regex.tag_rename]] + # ## Regular expression to match on a tag name + # pattern = "^search_(\\w+)d$" + # ## Matches of the pattern will be replaced with this string. Use ${1} + # ## notation to use the text of the first submatch. + # replacement = "${1}" + # ## If the new tag name already exists, you can either "overwrite" the + # ## existing one with the value of the renamed tag OR you can "keep" + # ## both the existing and source tag. + # # result_key = "keep" + + # Rename metrics + # [[processors.regex.metric_rename]] + # ## Regular expression to match on an metric name + # pattern = "^search_(\\w+)d$" + # ## Matches of the pattern will be replaced with this string. Use ${1} + # ## notation to use the text of the first submatch. + # replacement = "${1}" ``` ### Tags: @@ -46,5 +80,5 @@ No tags are applied by this processor. ### Example Output: ``` -nginx_requests,verb=GET,resp_code=2xx request="/api/search/?category=plugins&q=regex&sort=asc",method="/search/",search_category="plugins",referrer="-",ident="-",http_version=1.1,agent="UserAgent",client_ip="127.0.0.1",auth="-",resp_bytes=270i 1519652321000000000 +nginx_requests,verb=GET,resp_code=2xx request="/api/search/?category=plugins&q=regex&sort=asc",method="/search/",category="plugins",referrer="-",ident="-",http_version=1.1,agent="UserAgent",client_ip="127.0.0.1",auth="-",resp_bytes=270i 1519652321000000000 ``` diff --git a/plugins/processors/regex/regex.go b/plugins/processors/regex/regex.go index 47b53546f..4cf0c9856 100644 --- a/plugins/processors/regex/regex.go +++ b/plugins/processors/regex/regex.go @@ -1,24 +1,30 @@ package regex import ( + "fmt" "regexp" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal/choice" "github.com/influxdata/telegraf/plugins/processors" ) type Regex struct { - Tags []converter - Fields []converter - regexCache map[string]*regexp.Regexp + Tags []converter `toml:"tags"` + Fields []converter `toml:"fields"` + TagRename []converter `toml:"tag_rename"` + FieldRename []converter `toml:"field_rename"` + MetricRename []converter `toml:"metric_rename"` + Log telegraf.Logger `toml:"-"` + regexCache map[string]*regexp.Regexp } type converter struct { - Key string - Pattern string - Replacement string - ResultKey string - Append bool + Key string `toml:"key"` + Pattern string `toml:"pattern"` + Replacement string `toml:"replacement"` + ResultKey string `toml:"result_key"` + Append bool `toml:"append"` } const sampleConfig = ` @@ -50,12 +56,105 @@ const sampleConfig = ` # pattern = ".*category=(\\w+).*" # replacement = "${1}" # result_key = "search_category" + + ## Rename metric fields + # [[processors.regex.field_rename]] + # ## Regular expression to match on a field name + # pattern = "^search_(\\w+)d$" + # ## Matches of the pattern will be replaced with this string. Use ${1} + # ## notation to use the text of the first submatch. + # replacement = "${1}" + # ## If the new field name already exists, you can either "overwrite" the + # ## existing one with the value of the renamed field OR you can "keep" + # ## both the existing and source field. + # # result_key = "keep" + + ## Rename metric tags + # [[processors.regex.tag_rename]] + # ## Regular expression to match on a tag name + # pattern = "^search_(\\w+)d$" + # ## Matches of the pattern will be replaced with this string. Use ${1} + # ## notation to use the text of the first submatch. + # replacement = "${1}" + # ## If the new tag name already exists, you can either "overwrite" the + # ## existing one with the value of the renamed tag OR you can "keep" + # ## both the existing and source tag. + # # result_key = "keep" + + ## Rename metrics + # [[processors.regex.metric_rename]] + # ## Regular expression to match on an metric name + # pattern = "^search_(\\w+)d$" + # ## Matches of the pattern will be replaced with this string. Use ${1} + # ## notation to use the text of the first submatch. + # replacement = "${1}" ` -func NewRegex() *Regex { - return &Regex{ - regexCache: make(map[string]*regexp.Regexp), +func (r *Regex) Init() error { + r.regexCache = make(map[string]*regexp.Regexp) + + // Compile the regular expressions + for _, c := range r.Tags { + if _, compiled := r.regexCache[c.Pattern]; !compiled { + r.regexCache[c.Pattern] = regexp.MustCompile(c.Pattern) + } } + for _, c := range r.Fields { + if _, compiled := r.regexCache[c.Pattern]; !compiled { + r.regexCache[c.Pattern] = regexp.MustCompile(c.Pattern) + } + } + + resultOptions := []string{"overwrite", "keep"} + for _, c := range r.TagRename { + if c.Key != "" { + r.Log.Info("'tag_rename' section contains a key which is ignored during processing") + } + + if c.ResultKey == "" { + c.ResultKey = "keep" + } + if err := choice.Check(c.ResultKey, resultOptions); err != nil { + return fmt.Errorf("invalid metrics result_key: %v", err) + } + + if _, compiled := r.regexCache[c.Pattern]; !compiled { + r.regexCache[c.Pattern] = regexp.MustCompile(c.Pattern) + } + } + + for _, c := range r.FieldRename { + if c.Key != "" { + r.Log.Info("'field_rename' section contains a key which is ignored during processing") + } + + if c.ResultKey == "" { + c.ResultKey = "keep" + } + if err := choice.Check(c.ResultKey, resultOptions); err != nil { + return fmt.Errorf("invalid metrics result_key: %v", err) + } + + if _, compiled := r.regexCache[c.Pattern]; !compiled { + r.regexCache[c.Pattern] = regexp.MustCompile(c.Pattern) + } + } + + for _, c := range r.MetricRename { + if c.Key != "" { + r.Log.Info("'metric_rename' section contains a key which is ignored during processing") + } + + if c.ResultKey != "" { + r.Log.Info("'metric_rename' section contains a 'result_key' ignored during processing as metrics will ALWAYS the name") + } + + if _, compiled := r.regexCache[c.Pattern]; !compiled { + r.regexCache[c.Pattern] = regexp.MustCompile(c.Pattern) + } + } + + return nil } func (r *Regex) SampleConfig() string { @@ -63,7 +162,7 @@ func (r *Regex) SampleConfig() string { } func (r *Regex) Description() string { - return "Transforms tag and field values with regex pattern" + return "Transforms tag and field values as well as measurement, tag and field names with regex pattern" } func (r *Regex) Apply(in ...telegraf.Metric) []telegraf.Metric { @@ -83,27 +182,96 @@ func (r *Regex) Apply(in ...telegraf.Metric) []telegraf.Metric { for _, converter := range r.Fields { if value, ok := metric.GetField(converter.Key); ok { - switch value := value.(type) { - case string: - if key, newValue := r.convert(converter, value); newValue != "" { + if v, ok := value.(string); ok { + if key, newValue := r.convert(converter, v); newValue != "" { metric.AddField(key, newValue) } } } } + + for _, converter := range r.TagRename { + regex := r.regexCache[converter.Pattern] + replacements := make(map[string]string) + for _, tag := range metric.TagList() { + name := tag.Key + if regex.MatchString(name) { + newName := regex.ReplaceAllString(name, converter.Replacement) + + if !metric.HasTag(newName) { + // There is no colliding tag, we can just change the name. + tag.Key = newName + continue + } + + if converter.ResultKey == "overwrite" { + // We got a colliding tag, remember the replacement and do it later + replacements[name] = newName + } + } + } + // We needed to postpone the replacement as we cannot modify the tag-list + // while iterating it as this will result in invalid memory dereference panic. + for oldName, newName := range replacements { + value, ok := metric.GetTag(oldName) + if !ok { + // Just in case the tag got removed in the meantime + continue + } + metric.AddTag(newName, value) + metric.RemoveTag(oldName) + } + } + + for _, converter := range r.FieldRename { + regex := r.regexCache[converter.Pattern] + replacements := make(map[string]string) + for _, field := range metric.FieldList() { + name := field.Key + if regex.MatchString(name) { + newName := regex.ReplaceAllString(name, converter.Replacement) + + if !metric.HasField(newName) { + // There is no colliding field, we can just change the name. + field.Key = newName + continue + } + + if converter.ResultKey == "overwrite" { + // We got a colliding field, remember the replacement and do it later + replacements[name] = newName + } + } + } + // We needed to postpone the replacement as we cannot modify the field-list + // while iterating it as this will result in invalid memory dereference panic. + for oldName, newName := range replacements { + value, ok := metric.GetField(oldName) + if !ok { + // Just in case the field got removed in the meantime + continue + } + metric.AddField(newName, value) + metric.RemoveField(oldName) + } + } + + for _, converter := range r.MetricRename { + regex := r.regexCache[converter.Pattern] + value := metric.Name() + if regex.MatchString(value) { + newValue := regex.ReplaceAllString(value, converter.Replacement) + metric.SetName(newValue) + } + } } return in } -func (r *Regex) convert(c converter, src string) (string, string) { - regex, compiled := r.regexCache[c.Pattern] - if !compiled { - regex = regexp.MustCompile(c.Pattern) - r.regexCache[c.Pattern] = regex - } +func (r *Regex) convert(c converter, src string) (key string, value string) { + regex := r.regexCache[c.Pattern] - value := "" if c.ResultKey == "" || regex.MatchString(src) { value = regex.ReplaceAllString(src, c.Replacement) } @@ -116,7 +284,5 @@ func (r *Regex) convert(c converter, src string) (string, string) { } func init() { - processors.Add("regex", func() telegraf.Processor { - return NewRegex() - }) + processors.Add("regex", func() telegraf.Processor { return &Regex{} }) } diff --git a/plugins/processors/regex/regex_test.go b/plugins/processors/regex/regex_test.go index 2f8890bba..8baa0d79a 100644 --- a/plugins/processors/regex/regex_test.go +++ b/plugins/processors/regex/regex_test.go @@ -5,12 +5,14 @@ import ( "time" "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/metric" - "github.com/stretchr/testify/assert" + "github.com/influxdata/telegraf/testutil" + + "github.com/stretchr/testify/require" ) func newM1() telegraf.Metric { - m1 := metric.New("access_log", + return testutil.MustMetric( + "access_log", map[string]string{ "verb": "GET", "resp_code": "200", @@ -20,11 +22,11 @@ func newM1() telegraf.Metric { }, time.Now(), ) - return m1 } func newM2() telegraf.Metric { - m2 := metric.New("access_log", + return testutil.MustMetric( + "access_log", map[string]string{ "verb": "GET", "resp_code": "200", @@ -36,7 +38,6 @@ func newM2() telegraf.Metric { }, time.Now(), ) - return m2 } func TestFieldConversions(t *testing.T) { @@ -72,10 +73,11 @@ func TestFieldConversions(t *testing.T) { } for _, test := range tests { - regex := NewRegex() - regex.Fields = []converter{ - test.converter, + regex := Regex{ + Fields: []converter{test.converter}, + Log: testutil.Logger{}, } + require.NoError(t, regex.Init()) processed := regex.Apply(newM1()) @@ -84,9 +86,9 @@ func TestFieldConversions(t *testing.T) { "resp_code": "200", } - assert.Equal(t, test.expectedFields, processed[0].Fields(), test.message) - assert.Equal(t, expectedTags, processed[0].Tags(), "Should not change tags") - assert.Equal(t, "access_log", processed[0].Name(), "Should not change name") + require.Equal(t, test.expectedFields, processed[0].Fields(), test.message) + require.Equal(t, expectedTags, processed[0].Tags(), "Should not change tags") + require.Equal(t, "access_log", processed[0].Name(), "Should not change name") } } @@ -139,10 +141,11 @@ func TestTagConversions(t *testing.T) { } for _, test := range tests { - regex := NewRegex() - regex.Tags = []converter{ - test.converter, + regex := Regex{ + Tags: []converter{test.converter}, + Log: testutil.Logger{}, } + require.NoError(t, regex.Init()) processed := regex.Apply(newM1()) @@ -150,42 +153,579 @@ func TestTagConversions(t *testing.T) { "request": "/users/42/", } - assert.Equal(t, expectedFields, processed[0].Fields(), test.message, "Should not change fields") - assert.Equal(t, test.expectedTags, processed[0].Tags(), test.message) - assert.Equal(t, "access_log", processed[0].Name(), "Should not change name") + require.Equal(t, expectedFields, processed[0].Fields(), test.message, "Should not change fields") + require.Equal(t, test.expectedTags, processed[0].Tags(), test.message) + require.Equal(t, "access_log", processed[0].Name(), "Should not change name") + } +} + +func TestMetricNameConversions(t *testing.T) { + inputTemplate := []telegraf.Metric{ + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/users/42/", + }, + time.Unix(1627646243, 0), + ), + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(200), + "ignore_bool": true, + }, + time.Unix(1627646253, 0), + ), + testutil.MustMetric( + "error_log", + map[string]string{ + "verb": "GET", + "resp_code": "404", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(404), + "ignore_flag": true, + "error_message": "request too silly", + }, + time.Unix(1627646263, 0), + ), + } + + tests := []struct { + name string + converter converter + expected []telegraf.Metric + }{ + { + name: "Should change metric name", + converter: converter{ + Pattern: "^(\\w+)_log$", + Replacement: "${1}", + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "access", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/users/42/", + }, + time.Unix(1627646243, 0), + ), + testutil.MustMetric( + "access", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(200), + "ignore_bool": true, + }, + time.Unix(1627646253, 0), + ), + testutil.MustMetric( + "error", + map[string]string{ + "verb": "GET", + "resp_code": "404", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(404), + "ignore_flag": true, + "error_message": "request too silly", + }, + time.Unix(1627646263, 0), + ), + }, + }, + } + + for _, test := range tests { + // Copy the inputs as they will be modified by the processor + input := make([]telegraf.Metric, len(inputTemplate)) + for i, m := range inputTemplate { + input[i] = m.Copy() + } + + t.Run(test.name, func(t *testing.T) { + regex := Regex{ + MetricRename: []converter{test.converter}, + Log: testutil.Logger{}, + } + require.NoError(t, regex.Init()) + + actual := regex.Apply(input...) + testutil.RequireMetricsEqual(t, test.expected, actual) + }) + } +} + +func TestFieldRenameConversions(t *testing.T) { + inputTemplate := []telegraf.Metric{ + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/users/42/", + }, + time.Unix(1627646243, 0), + ), + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(200), + "ignore_bool": true, + }, + time.Unix(1627646253, 0), + ), + testutil.MustMetric( + "error_log", + map[string]string{ + "verb": "GET", + "resp_code": "404", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(404), + "ignore_flag": true, + "error_message": "request too silly", + }, + time.Unix(1627646263, 0), + ), + } + + tests := []struct { + name string + converter converter + expected []telegraf.Metric + }{ + { + name: "Should change field name", + converter: converter{ + Pattern: "^(?:ignore|error)_(\\w+)$", + Replacement: "result_${1}", + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/users/42/", + }, + time.Unix(1627646243, 0), + ), + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "result_number": int64(200), + "result_bool": true, + }, + time.Unix(1627646253, 0), + ), + testutil.MustMetric( + "error_log", + map[string]string{ + "verb": "GET", + "resp_code": "404", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "result_number": int64(404), + "result_flag": true, + "result_message": "request too silly", + }, + time.Unix(1627646263, 0), + ), + }, + }, + { + name: "Should keep existing field name", + converter: converter{ + Pattern: "^(?:ignore|error)_(\\w+)$", + Replacement: "request", + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/users/42/", + }, + time.Unix(1627646243, 0), + ), + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(200), + "ignore_bool": true, + }, + time.Unix(1627646253, 0), + ), + testutil.MustMetric( + "error_log", + map[string]string{ + "verb": "GET", + "resp_code": "404", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(404), + "ignore_flag": true, + "error_message": "request too silly", + }, + time.Unix(1627646263, 0), + ), + }, + }, + { + name: "Should overwrite existing field name", + converter: converter{ + Pattern: "^ignore_bool$", + Replacement: "request", + ResultKey: "overwrite", + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/users/42/", + }, + time.Unix(1627646243, 0), + ), + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "ignore_number": int64(200), + "request": true, + }, + time.Unix(1627646253, 0), + ), + testutil.MustMetric( + "error_log", + map[string]string{ + "verb": "GET", + "resp_code": "404", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(404), + "ignore_flag": true, + "error_message": "request too silly", + }, + time.Unix(1627646263, 0), + ), + }, + }, + } + + for _, test := range tests { + // Copy the inputs as they will be modified by the processor + input := make([]telegraf.Metric, len(inputTemplate)) + for i, m := range inputTemplate { + input[i] = m.Copy() + } + + t.Run(test.name, func(t *testing.T) { + regex := Regex{ + FieldRename: []converter{test.converter}, + Log: testutil.Logger{}, + } + require.NoError(t, regex.Init()) + + actual := regex.Apply(input...) + testutil.RequireMetricsEqual(t, test.expected, actual) + }) + } +} + +func TestTagRenameConversions(t *testing.T) { + inputTemplate := []telegraf.Metric{ + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/users/42/", + }, + time.Unix(1627646243, 0), + ), + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(200), + "ignore_bool": true, + }, + time.Unix(1627646253, 0), + ), + testutil.MustMetric( + "error_log", + map[string]string{ + "verb": "GET", + "resp_code": "404", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(404), + "ignore_flag": true, + "error_message": "request too silly", + }, + time.Unix(1627646263, 0), + ), + } + + tests := []struct { + name string + converter converter + expected []telegraf.Metric + }{ + { + name: "Should change tag name", + converter: converter{ + Pattern: "^resp_(\\w+)$", + Replacement: "${1}", + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "code": "200", + }, + map[string]interface{}{ + "request": "/users/42/", + }, + time.Unix(1627646243, 0), + ), + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "code": "200", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(200), + "ignore_bool": true, + }, + time.Unix(1627646253, 0), + ), + testutil.MustMetric( + "error_log", + map[string]string{ + "verb": "GET", + "code": "404", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(404), + "ignore_flag": true, + "error_message": "request too silly", + }, + time.Unix(1627646263, 0), + ), + }, + }, + { + name: "Should keep existing tag name", + converter: converter{ + Pattern: "^resp_(\\w+)$", + Replacement: "verb", + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/users/42/", + }, + time.Unix(1627646243, 0), + ), + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "GET", + "resp_code": "200", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(200), + "ignore_bool": true, + }, + time.Unix(1627646253, 0), + ), + testutil.MustMetric( + "error_log", + map[string]string{ + "verb": "GET", + "resp_code": "404", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(404), + "ignore_flag": true, + "error_message": "request too silly", + }, + time.Unix(1627646263, 0), + ), + }, + }, + { + name: "Should overwrite existing tag name", + converter: converter{ + Pattern: "^resp_(\\w+)$", + Replacement: "verb", + ResultKey: "overwrite", + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "200", + }, + map[string]interface{}{ + "request": "/users/42/", + }, + time.Unix(1627646243, 0), + ), + testutil.MustMetric( + "access_log", + map[string]string{ + "verb": "200", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(200), + "ignore_bool": true, + }, + time.Unix(1627646253, 0), + ), + testutil.MustMetric( + "error_log", + map[string]string{ + "verb": "404", + }, + map[string]interface{}{ + "request": "/api/search/?category=plugins&q=regex&sort=asc", + "ignore_number": int64(404), + "ignore_flag": true, + "error_message": "request too silly", + }, + time.Unix(1627646263, 0), + ), + }, + }, + } + + for _, test := range tests { + // Copy the inputs as they will be modified by the processor + input := make([]telegraf.Metric, len(inputTemplate)) + for i, m := range inputTemplate { + input[i] = m.Copy() + } + + t.Run(test.name, func(t *testing.T) { + regex := Regex{ + TagRename: []converter{test.converter}, + Log: testutil.Logger{}, + } + require.NoError(t, regex.Init()) + + actual := regex.Apply(input...) + testutil.RequireMetricsEqual(t, test.expected, actual) + }) } } func TestMultipleConversions(t *testing.T) { - regex := NewRegex() - regex.Tags = []converter{ - { - Key: "resp_code", - Pattern: "^(\\d)\\d\\d$", - Replacement: "${1}xx", - ResultKey: "resp_code_group", + regex := Regex{ + Tags: []converter{ + { + Key: "resp_code", + Pattern: "^(\\d)\\d\\d$", + Replacement: "${1}xx", + ResultKey: "resp_code_group", + }, + { + Key: "resp_code_group", + Pattern: "2xx", + Replacement: "OK", + ResultKey: "resp_code_text", + }, }, - { - Key: "resp_code_group", - Pattern: "2xx", - Replacement: "OK", - ResultKey: "resp_code_text", - }, - } - regex.Fields = []converter{ - { - Key: "request", - Pattern: "^/api(?P/[\\w/]+)\\S*", - Replacement: "${method}", - ResultKey: "method", - }, - { - Key: "request", - Pattern: ".*category=(\\w+).*", - Replacement: "${1}", - ResultKey: "search_category", + Fields: []converter{ + { + Key: "request", + Pattern: "^/api(?P/[\\w/]+)\\S*", + Replacement: "${method}", + ResultKey: "method", + }, + { + Key: "request", + Pattern: ".*category=(\\w+).*", + Replacement: "${1}", + ResultKey: "search_category", + }, }, + Log: testutil.Logger{}, } + require.NoError(t, regex.Init()) processed := regex.Apply(newM2()) @@ -203,8 +743,8 @@ func TestMultipleConversions(t *testing.T) { "resp_code_text": "OK", } - assert.Equal(t, expectedFields, processed[0].Fields()) - assert.Equal(t, expectedTags, processed[0].Tags()) + require.Equal(t, expectedFields, processed[0].Fields()) + require.Equal(t, expectedTags, processed[0].Tags()) } func TestNoMatches(t *testing.T) { @@ -250,34 +790,38 @@ func TestNoMatches(t *testing.T) { } for _, test := range tests { - regex := NewRegex() - regex.Fields = []converter{ - test.converter, + regex := Regex{ + Fields: []converter{test.converter}, + Log: testutil.Logger{}, } + require.NoError(t, regex.Init()) processed := regex.Apply(newM1()) - assert.Equal(t, test.expectedFields, processed[0].Fields(), test.message) + require.Equal(t, test.expectedFields, processed[0].Fields(), test.message) } } func BenchmarkConversions(b *testing.B) { - regex := NewRegex() - regex.Tags = []converter{ - { - Key: "resp_code", - Pattern: "^(\\d)\\d\\d$", - Replacement: "${1}xx", - ResultKey: "resp_code_group", + regex := Regex{ + Tags: []converter{ + { + Key: "resp_code", + Pattern: "^(\\d)\\d\\d$", + Replacement: "${1}xx", + ResultKey: "resp_code_group", + }, }, - } - regex.Fields = []converter{ - { - Key: "request", - Pattern: "^/users/\\d+/$", - Replacement: "/users/{id}/", + Fields: []converter{ + { + Key: "request", + Pattern: "^/users/\\d+/$", + Replacement: "/users/{id}/", + }, }, + Log: testutil.Logger{}, } + require.NoError(b, regex.Init()) for n := 0; n < b.N; n++ { processed := regex.Apply(newM1())