Add wildcard tags json parser support (#8579)
This commit is contained in:
Helen Weller
GitHub
parent
ee91b4856b
commit
94eb8f2e42
|
|
@ -30,10 +30,12 @@ ignored unless specified in the `tag_key` or `json_string_fields` options.
|
|||
json_query = ""
|
||||
|
||||
## Tag keys is an array of keys that should be added as tags. Matching keys
|
||||
## are no longer saved as fields.
|
||||
## are no longer saved as fields. Supports wildcard glob matching.
|
||||
tag_keys = [
|
||||
"my_tag_1",
|
||||
"my_tag_2"
|
||||
"my_tag_2",
|
||||
"tags_*",
|
||||
"tag*"
|
||||
]
|
||||
|
||||
## Array of glob pattern strings or booleans keys that should be added as string fields.
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ type Config struct {
|
|||
|
||||
type Parser struct {
|
||||
metricName string
|
||||
tagKeys []string
|
||||
tagKeys filter.Filter
|
||||
stringFields filter.Filter
|
||||
nameKey string
|
||||
query string
|
||||
|
|
@ -53,9 +53,14 @@ func New(config *Config) (*Parser, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
tagKeyFilter, err := filter.Compile(config.TagKeys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Parser{
|
||||
metricName: config.MetricName,
|
||||
tagKeys: config.TagKeys,
|
||||
tagKeys: tagKeyFilter,
|
||||
nameKey: config.NameKey,
|
||||
stringFields: stringFilter,
|
||||
query: config.Query,
|
||||
|
|
@ -150,20 +155,25 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) (
|
|||
// will delete any strings/bools that shouldn't be fields
|
||||
// assumes that any non-numeric values in TagKeys should be displayed as tags
|
||||
func (p *Parser) switchFieldToTag(tags map[string]string, fields map[string]interface{}) (map[string]string, map[string]interface{}) {
|
||||
for _, name := range p.tagKeys {
|
||||
//switch any fields in tagkeys into tags
|
||||
if fields[name] == nil {
|
||||
|
||||
for name, value := range fields {
|
||||
if p.tagKeys == nil {
|
||||
continue
|
||||
}
|
||||
switch value := fields[name].(type) {
|
||||
// skip switch statement if tagkey doesn't match fieldname
|
||||
if !p.tagKeys.Match(name) {
|
||||
continue
|
||||
}
|
||||
// switch any fields in tagkeys into tags
|
||||
switch t := value.(type) {
|
||||
case string:
|
||||
tags[name] = value
|
||||
tags[name] = t
|
||||
delete(fields, name)
|
||||
case bool:
|
||||
tags[name] = strconv.FormatBool(value)
|
||||
tags[name] = strconv.FormatBool(t)
|
||||
delete(fields, name)
|
||||
case float64:
|
||||
tags[name] = strconv.FormatFloat(value, 'f', -1, 64)
|
||||
tags[name] = strconv.FormatFloat(t, 'f', -1, 64)
|
||||
delete(fields, name)
|
||||
default:
|
||||
log.Printf("E! [parsers.json] Unrecognized type %T", value)
|
||||
|
|
|
|||
|
|
@ -26,9 +26,13 @@ const validJSONTags = `
|
|||
"b": {
|
||||
"c": 6
|
||||
},
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz",
|
||||
"tags_object": {
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz"
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
const validJSONArrayTags = `
|
||||
|
|
@ -39,7 +43,16 @@ const validJSONArrayTags = `
|
|||
"c": 6
|
||||
},
|
||||
"mytag": "foo",
|
||||
"othertag": "baz",
|
||||
"tags_array": [
|
||||
{
|
||||
"mytag": "foo"
|
||||
},
|
||||
{
|
||||
"othertag": "baz"
|
||||
}
|
||||
],
|
||||
"anothert": "foo"
|
||||
},
|
||||
{
|
||||
"a": 7,
|
||||
|
|
@ -47,8 +60,17 @@ const validJSONArrayTags = `
|
|||
"c": 8
|
||||
},
|
||||
"mytag": "bar",
|
||||
"othertag": "baz",
|
||||
"tags_array": [
|
||||
{
|
||||
"mytag": "bar"
|
||||
},
|
||||
{
|
||||
"othertag": "baz"
|
||||
}
|
||||
],
|
||||
"anothert": "bar"
|
||||
}
|
||||
]
|
||||
`
|
||||
|
||||
|
|
@ -948,3 +970,392 @@ func TestParse(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWithWildcardTagKeys(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
config *Config
|
||||
input []byte
|
||||
expected []telegraf.Metric
|
||||
}{
|
||||
{
|
||||
name: "wildcard matching with tags nested within object",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"tags_object_*"},
|
||||
},
|
||||
input: []byte(validJSONTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wildcard matching with keys containing tag",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"*tag"},
|
||||
},
|
||||
input: []byte(validJSONTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz",
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "strings not matching tag keys are still also ignored",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"wrongtagkey", "tags_object_*"},
|
||||
},
|
||||
input: []byte(validJSONTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "single tag key is also found and applied",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"mytag", "tags_object_*"},
|
||||
},
|
||||
input: []byte(validJSONTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"mytag": "foobar",
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
parser, err := New(tt.config)
|
||||
require.NoError(t, err)
|
||||
|
||||
actual, err := parser.Parse(tt.input)
|
||||
require.NoError(t, err)
|
||||
testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLineWithWildcardTagKeys(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
config *Config
|
||||
input string
|
||||
expected telegraf.Metric
|
||||
}{
|
||||
{
|
||||
name: "wildcard matching with tags nested within object",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"tags_object_*"},
|
||||
},
|
||||
input: validJSONTags,
|
||||
expected: testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "wildcard matching with keys containing tag",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"*tag"},
|
||||
},
|
||||
input: validJSONTags,
|
||||
expected: testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz",
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "strings not matching tag keys are ignored",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"wrongtagkey", "tags_object_*"},
|
||||
},
|
||||
input: validJSONTags,
|
||||
expected: testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "single tag key is also found and applied",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"mytag", "tags_object_*"},
|
||||
},
|
||||
input: validJSONTags,
|
||||
expected: testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"mytag": "foobar",
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
parser, err := New(tt.config)
|
||||
require.NoError(t, err)
|
||||
|
||||
actual, err := parser.ParseLine(tt.input)
|
||||
require.NoError(t, err)
|
||||
|
||||
testutil.RequireMetricEqual(t, tt.expected, actual, testutil.IgnoreTime())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseArrayWithWildcardTagKeys(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
config *Config
|
||||
input []byte
|
||||
expected []telegraf.Metric
|
||||
}{
|
||||
{
|
||||
name: "wildcard matching with keys containing tag within array works",
|
||||
config: &Config{
|
||||
MetricName: "json_array_test",
|
||||
TagKeys: []string{"*tag"},
|
||||
},
|
||||
input: []byte(validJSONArrayTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"mytag": "foo",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "foo",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"mytag": "bar",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "bar",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(7),
|
||||
"b_c": float64(8),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: " wildcard matching with tags nested array within object works",
|
||||
config: &Config{
|
||||
MetricName: "json_array_test",
|
||||
TagKeys: []string{"tags_array_*"},
|
||||
},
|
||||
input: []byte(validJSONArrayTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"tags_array_0_mytag": "foo",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"tags_array_0_mytag": "bar",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(7),
|
||||
"b_c": float64(8),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "strings not matching tag keys are still also ignored",
|
||||
config: &Config{
|
||||
MetricName: "json_array_test",
|
||||
TagKeys: []string{"mytag", "*tag"},
|
||||
},
|
||||
input: []byte(validJSONArrayTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"mytag": "foo",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "foo",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"mytag": "bar",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "bar",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(7),
|
||||
"b_c": float64(8),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "single tag key is also found and applied",
|
||||
config: &Config{
|
||||
MetricName: "json_array_test",
|
||||
TagKeys: []string{"anothert", "*tag"},
|
||||
},
|
||||
input: []byte(validJSONArrayTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"anothert": "foo",
|
||||
"mytag": "foo",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "foo",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"anothert": "bar",
|
||||
"mytag": "bar",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "bar",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(7),
|
||||
"b_c": float64(8),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
parser, err := New(tt.config)
|
||||
require.NoError(t, err)
|
||||
|
||||
actual, err := parser.Parse(tt.input)
|
||||
require.NoError(t, err)
|
||||
|
||||
testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime())
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue