Add wildcard tags json parser support (#8579)
This commit is contained in:
Helen Weller
GitHub
parent
ee91b4856b
commit
94eb8f2e42
|
|
@ -30,10 +30,12 @@ ignored unless specified in the `tag_key` or `json_string_fields` options.
|
|||
json_query = ""
|
||||
|
||||
## Tag keys is an array of keys that should be added as tags. Matching keys
|
||||
## are no longer saved as fields.
|
||||
## are no longer saved as fields. Supports wildcard glob matching.
|
||||
tag_keys = [
|
||||
"my_tag_1",
|
||||
"my_tag_2"
|
||||
"my_tag_2",
|
||||
"tags_*",
|
||||
"tag*"
|
||||
]
|
||||
|
||||
## Array of glob pattern strings or booleans keys that should be added as string fields.
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ type Config struct {
|
|||
|
||||
type Parser struct {
|
||||
metricName string
|
||||
tagKeys []string
|
||||
tagKeys filter.Filter
|
||||
stringFields filter.Filter
|
||||
nameKey string
|
||||
query string
|
||||
|
|
@ -53,9 +53,14 @@ func New(config *Config) (*Parser, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
tagKeyFilter, err := filter.Compile(config.TagKeys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Parser{
|
||||
metricName: config.MetricName,
|
||||
tagKeys: config.TagKeys,
|
||||
tagKeys: tagKeyFilter,
|
||||
nameKey: config.NameKey,
|
||||
stringFields: stringFilter,
|
||||
query: config.Query,
|
||||
|
|
@ -104,7 +109,7 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) (
|
|||
|
||||
name := p.metricName
|
||||
|
||||
//checks if json_name_key is set
|
||||
// checks if json_name_key is set
|
||||
if p.nameKey != "" {
|
||||
switch field := f.Fields[p.nameKey].(type) {
|
||||
case string:
|
||||
|
|
@ -112,7 +117,7 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) (
|
|||
}
|
||||
}
|
||||
|
||||
//if time key is specified, set timestamp to it
|
||||
// if time key is specified, set timestamp to it
|
||||
if p.timeKey != "" {
|
||||
if p.timeFormat == "" {
|
||||
err := fmt.Errorf("use of 'json_time_key' requires 'json_time_format'")
|
||||
|
|
@ -131,7 +136,7 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) (
|
|||
|
||||
delete(f.Fields, p.timeKey)
|
||||
|
||||
//if the year is 0, set to current year
|
||||
// if the year is 0, set to current year
|
||||
if timestamp.Year() == 0 {
|
||||
timestamp = timestamp.AddDate(time.Now().Year(), 0, 0)
|
||||
}
|
||||
|
|
@ -145,32 +150,37 @@ func (p *Parser) parseObject(data map[string]interface{}, timestamp time.Time) (
|
|||
return []telegraf.Metric{metric}, nil
|
||||
}
|
||||
|
||||
//will take in field map with strings and bools,
|
||||
//search for TagKeys that match fieldnames and add them to tags
|
||||
//will delete any strings/bools that shouldn't be fields
|
||||
//assumes that any non-numeric values in TagKeys should be displayed as tags
|
||||
// will take in field map with strings and bools,
|
||||
// search for TagKeys that match fieldnames and add them to tags
|
||||
// will delete any strings/bools that shouldn't be fields
|
||||
// assumes that any non-numeric values in TagKeys should be displayed as tags
|
||||
func (p *Parser) switchFieldToTag(tags map[string]string, fields map[string]interface{}) (map[string]string, map[string]interface{}) {
|
||||
for _, name := range p.tagKeys {
|
||||
//switch any fields in tagkeys into tags
|
||||
if fields[name] == nil {
|
||||
|
||||
for name, value := range fields {
|
||||
if p.tagKeys == nil {
|
||||
continue
|
||||
}
|
||||
switch value := fields[name].(type) {
|
||||
// skip switch statement if tagkey doesn't match fieldname
|
||||
if !p.tagKeys.Match(name) {
|
||||
continue
|
||||
}
|
||||
// switch any fields in tagkeys into tags
|
||||
switch t := value.(type) {
|
||||
case string:
|
||||
tags[name] = value
|
||||
tags[name] = t
|
||||
delete(fields, name)
|
||||
case bool:
|
||||
tags[name] = strconv.FormatBool(value)
|
||||
tags[name] = strconv.FormatBool(t)
|
||||
delete(fields, name)
|
||||
case float64:
|
||||
tags[name] = strconv.FormatFloat(value, 'f', -1, 64)
|
||||
tags[name] = strconv.FormatFloat(t, 'f', -1, 64)
|
||||
delete(fields, name)
|
||||
default:
|
||||
log.Printf("E! [parsers.json] Unrecognized type %T", value)
|
||||
}
|
||||
}
|
||||
|
||||
//remove any additional string/bool values from fields
|
||||
// remove any additional string/bool values from fields
|
||||
for fk := range fields {
|
||||
switch fields[fk].(type) {
|
||||
case string, bool:
|
||||
|
|
|
|||
|
|
@ -24,10 +24,14 @@ const validJSONTags = `
|
|||
{
|
||||
"a": 5,
|
||||
"b": {
|
||||
"c": 6
|
||||
"c": 6
|
||||
},
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz"
|
||||
"othertag": "baz",
|
||||
"tags_object": {
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz"
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
|
|
@ -39,7 +43,16 @@ const validJSONArrayTags = `
|
|||
"c": 6
|
||||
},
|
||||
"mytag": "foo",
|
||||
"othertag": "baz"
|
||||
"othertag": "baz",
|
||||
"tags_array": [
|
||||
{
|
||||
"mytag": "foo"
|
||||
},
|
||||
{
|
||||
"othertag": "baz"
|
||||
}
|
||||
],
|
||||
"anothert": "foo"
|
||||
},
|
||||
{
|
||||
"a": 7,
|
||||
|
|
@ -47,8 +60,17 @@ const validJSONArrayTags = `
|
|||
"c": 8
|
||||
},
|
||||
"mytag": "bar",
|
||||
"othertag": "baz",
|
||||
"tags_array": [
|
||||
{
|
||||
"mytag": "bar"
|
||||
},
|
||||
{
|
||||
"othertag": "baz"
|
||||
}
|
||||
}
|
||||
],
|
||||
"anothert": "bar"
|
||||
}
|
||||
]
|
||||
`
|
||||
|
||||
|
|
@ -948,3 +970,392 @@ func TestParse(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWithWildcardTagKeys(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
config *Config
|
||||
input []byte
|
||||
expected []telegraf.Metric
|
||||
}{
|
||||
{
|
||||
name: "wildcard matching with tags nested within object",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"tags_object_*"},
|
||||
},
|
||||
input: []byte(validJSONTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wildcard matching with keys containing tag",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"*tag"},
|
||||
},
|
||||
input: []byte(validJSONTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz",
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "strings not matching tag keys are still also ignored",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"wrongtagkey", "tags_object_*"},
|
||||
},
|
||||
input: []byte(validJSONTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "single tag key is also found and applied",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"mytag", "tags_object_*"},
|
||||
},
|
||||
input: []byte(validJSONTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"mytag": "foobar",
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
parser, err := New(tt.config)
|
||||
require.NoError(t, err)
|
||||
|
||||
actual, err := parser.Parse(tt.input)
|
||||
require.NoError(t, err)
|
||||
testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLineWithWildcardTagKeys(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
config *Config
|
||||
input string
|
||||
expected telegraf.Metric
|
||||
}{
|
||||
{
|
||||
name: "wildcard matching with tags nested within object",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"tags_object_*"},
|
||||
},
|
||||
input: validJSONTags,
|
||||
expected: testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "wildcard matching with keys containing tag",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"*tag"},
|
||||
},
|
||||
input: validJSONTags,
|
||||
expected: testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"mytag": "foobar",
|
||||
"othertag": "baz",
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "strings not matching tag keys are ignored",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"wrongtagkey", "tags_object_*"},
|
||||
},
|
||||
input: validJSONTags,
|
||||
expected: testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "single tag key is also found and applied",
|
||||
config: &Config{
|
||||
MetricName: "json_test",
|
||||
TagKeys: []string{"mytag", "tags_object_*"},
|
||||
},
|
||||
input: validJSONTags,
|
||||
expected: testutil.MustMetric(
|
||||
"json_test",
|
||||
map[string]string{
|
||||
"mytag": "foobar",
|
||||
"tags_object_mytag": "foobar",
|
||||
"tags_object_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
parser, err := New(tt.config)
|
||||
require.NoError(t, err)
|
||||
|
||||
actual, err := parser.ParseLine(tt.input)
|
||||
require.NoError(t, err)
|
||||
|
||||
testutil.RequireMetricEqual(t, tt.expected, actual, testutil.IgnoreTime())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseArrayWithWildcardTagKeys(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
config *Config
|
||||
input []byte
|
||||
expected []telegraf.Metric
|
||||
}{
|
||||
{
|
||||
name: "wildcard matching with keys containing tag within array works",
|
||||
config: &Config{
|
||||
MetricName: "json_array_test",
|
||||
TagKeys: []string{"*tag"},
|
||||
},
|
||||
input: []byte(validJSONArrayTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"mytag": "foo",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "foo",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"mytag": "bar",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "bar",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(7),
|
||||
"b_c": float64(8),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: " wildcard matching with tags nested array within object works",
|
||||
config: &Config{
|
||||
MetricName: "json_array_test",
|
||||
TagKeys: []string{"tags_array_*"},
|
||||
},
|
||||
input: []byte(validJSONArrayTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"tags_array_0_mytag": "foo",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"tags_array_0_mytag": "bar",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(7),
|
||||
"b_c": float64(8),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "strings not matching tag keys are still also ignored",
|
||||
config: &Config{
|
||||
MetricName: "json_array_test",
|
||||
TagKeys: []string{"mytag", "*tag"},
|
||||
},
|
||||
input: []byte(validJSONArrayTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"mytag": "foo",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "foo",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"mytag": "bar",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "bar",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(7),
|
||||
"b_c": float64(8),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "single tag key is also found and applied",
|
||||
config: &Config{
|
||||
MetricName: "json_array_test",
|
||||
TagKeys: []string{"anothert", "*tag"},
|
||||
},
|
||||
input: []byte(validJSONArrayTags),
|
||||
expected: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"anothert": "foo",
|
||||
"mytag": "foo",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "foo",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(5),
|
||||
"b_c": float64(6),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
testutil.MustMetric(
|
||||
"json_array_test",
|
||||
map[string]string{
|
||||
"anothert": "bar",
|
||||
"mytag": "bar",
|
||||
"othertag": "baz",
|
||||
"tags_array_0_mytag": "bar",
|
||||
"tags_array_1_othertag": "baz",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"a": float64(7),
|
||||
"b_c": float64(8),
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
parser, err := New(tt.config)
|
||||
require.NoError(t, err)
|
||||
|
||||
actual, err := parser.Parse(tt.input)
|
||||
require.NoError(t, err)
|
||||
|
||||
testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime())
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue