diff --git a/plugins/parsers/json_v2/parser.go b/plugins/parsers/json_v2/parser.go index ef8981dff..fa0946621 100644 --- a/plugins/parsers/json_v2/parser.go +++ b/plugins/parsers/json_v2/parser.go @@ -184,11 +184,7 @@ func (p *Parser) processMetric(data []DataSet, input []byte, tag bool) ([]telegr return nil, err } - var m []telegraf.Metric - for _, n := range nodes { - m = append(m, n.Metric) - } - metrics = append(metrics, m) + metrics = append(metrics, nodes) } for i := 1; i < len(metrics); i++ { @@ -229,8 +225,8 @@ func mergeMetric(a telegraf.Metric, m telegraf.Metric) { } // expandArray will recursively create a new MetricNode for each element in a JSON array or single value -func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) { - var results []MetricNode +func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) { + var results []telegraf.Metric if result.IsObject() { if !p.iterateObjects { @@ -262,8 +258,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) { Metric: m, Result: val, } - var r []MetricNode - r, err = p.combineObject(n) + r, err := p.combineObject(n) if err != nil { return false } @@ -274,7 +269,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) { } if len(results) != 0 { for _, newResult := range results { - mergeMetric(result.Metric, newResult.Metric) + mergeMetric(result.Metric, newResult) } } return true @@ -294,8 +289,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) { Metric: m, Result: val, } - var r []MetricNode - r, err = p.expandArray(n) + r, err := p.expandArray(n) if err != nil { return false } @@ -335,7 +329,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) { } } - results = append(results, result) + results = append(results, result.Metric) } return results, nil @@ -369,9 +363,7 @@ func (p *Parser) processObjects(objects []JSONObject, input []byte) ([]telegraf. if err != nil { return nil, err } - for _, m := range metrics { - t = append(t, m.Metric) - } + t = append(t, metrics...) } return t, nil @@ -379,12 +371,10 @@ func (p *Parser) processObjects(objects []JSONObject, input []byte) ([]telegraf. // combineObject will add all fields/tags to a single metric // If the object has multiple array's as elements it won't comine those, they will remain separate metrics -func (p *Parser) combineObject(result MetricNode) ([]MetricNode, error) { - var results []MetricNode - var combineObjectResult []MetricNode +func (p *Parser) combineObject(result MetricNode) ([]telegraf.Metric, error) { + var results []telegraf.Metric if result.IsArray() || result.IsObject() { var err error - var prevArray bool result.ForEach(func(key, val gjson.Result) bool { // Determine if field/tag set name is configured var setName string @@ -436,38 +426,18 @@ func (p *Parser) combineObject(result MetricNode) ([]MetricNode, error) { } arrayNode.Tag = tag + if val.IsObject() { - prevArray = false - combineObjectResult, err = p.combineObject(arrayNode) + results, err = p.combineObject(arrayNode) if err != nil { return false } } else { - var r []MetricNode - r, err = p.expandArray(arrayNode) + r, err := p.expandArray(arrayNode) if err != nil { return false } - if prevArray { - if !arrayNode.IsArray() { - // If another non-array element was found, merge it into all previous gathered metrics - if len(results) != 0 { - for _, newResult := range results { - mergeMetric(result.Metric, newResult.Metric) - } - } - } else { - // Multiple array's won't be merged but kept separate, add additional metrics gathered from an array - results = append(results, r...) - } - } else { - // Continue using the same metric if its an object - results = r - } - - if val.IsArray() { - prevArray = true - } + results = cartesianProduct(results, r) } return true @@ -477,13 +447,6 @@ func (p *Parser) combineObject(result MetricNode) ([]MetricNode, error) { return nil, err } } - - if len(results) == 0 { - // If the results are empty, use the results of the call to combine object - // This happens with nested objects in array's, see the test array_of_objects - results = combineObjectResult - } - return results, nil } diff --git a/plugins/parsers/json_v2/parser_test.go b/plugins/parsers/json_v2/parser_test.go index 9321d7256..f0f018034 100644 --- a/plugins/parsers/json_v2/parser_test.go +++ b/plugins/parsers/json_v2/parser_test.go @@ -21,6 +21,10 @@ func TestData(t *testing.T) { name string test string }{ + { + name: "Test complex nesting", + test: "complex_nesting", + }, { name: "Test having an array of objects", test: "array_of_objects", diff --git a/plugins/parsers/json_v2/testdata/complex_nesting/expected.out b/plugins/parsers/json_v2/testdata/complex_nesting/expected.out new file mode 100644 index 000000000..265549c57 --- /dev/null +++ b/plugins/parsers/json_v2/testdata/complex_nesting/expected.out @@ -0,0 +1,3 @@ +file,properties_place=Antelope\ Valley\,\ CA geometry_coordinates=-119.4998333,geometry_type="Point",id="nc73584926",properties_mag=6,properties_updated=1.626277167263e+12,type="Feature" +file,properties_place=Antelope\ Valley\,\ CA geometry_coordinates=38.5075,geometry_type="Point",id="nc73584926",properties_mag=6,properties_updated=1.626277167263e+12,type="Feature" +file,properties_place=Antelope\ Valley\,\ CA geometry_coordinates=7.45,geometry_type="Point",id="nc73584926",properties_mag=6,properties_updated=1.626277167263e+12,type="Feature" diff --git a/plugins/parsers/json_v2/testdata/complex_nesting/input.json b/plugins/parsers/json_v2/testdata/complex_nesting/input.json new file mode 100644 index 000000000..69bff40a4 --- /dev/null +++ b/plugins/parsers/json_v2/testdata/complex_nesting/input.json @@ -0,0 +1,31 @@ +{ + "type": "FeatureCollection", + "metadata": { + "generated": 1626285886000, + "url": "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/significant_week.geojson", + "title": "USGS Significant Earthquakes, Past Week", + "status": 200, + "api": "1.10.3", + "count": 1 + }, + "features": [ + { + "type": "Feature", + "properties": { + "mag": 6, + "place": "Antelope Valley, CA", + "time": 1625784588110, + "updated": 1626277167263 + }, + "geometry": { + "type": "Point", + "coordinates": [ + -119.4998333, + 38.5075, + 7.45 + ] + }, + "id": "nc73584926" + } + ] +} diff --git a/plugins/parsers/json_v2/testdata/complex_nesting/telegraf.conf b/plugins/parsers/json_v2/testdata/complex_nesting/telegraf.conf new file mode 100644 index 000000000..66347da84 --- /dev/null +++ b/plugins/parsers/json_v2/testdata/complex_nesting/telegraf.conf @@ -0,0 +1,9 @@ +[[inputs.file]] + files = ["./testdata/complex_nesting/input.json"] + data_format = "json_v2" + [[inputs.file.json_v2]] + [[inputs.file.json_v2.object]] + path = "features" + timestamp_key = "properties_time" + timestamp_format = "unix_ms" + tags = ["properties_place"] diff --git a/plugins/parsers/json_v2/testdata/multiple_arrays_in_object/expected.out b/plugins/parsers/json_v2/testdata/multiple_arrays_in_object/expected.out index 814d044ce..2948da172 100644 --- a/plugins/parsers/json_v2/testdata/multiple_arrays_in_object/expected.out +++ b/plugins/parsers/json_v2/testdata/multiple_arrays_in_object/expected.out @@ -1,6 +1,9 @@ -file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party" -file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past" -file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",name="Bilbo",species="hobbit" -file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",name="Frodo",species="hobbit" -file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",random=1 -file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",random=2 +file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Bilbo",species="hobbit",random=1 +file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Bilbo",species="hobbit",random=2 +file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Frodo",species="hobbit",random=1 +file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Frodo",species="hobbit",random=2 +file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Bilbo",species="hobbit",random=1 +file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Bilbo",species="hobbit",random=2 +file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Frodo",species="hobbit",random=1 +file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Frodo",species="hobbit",random=2 +