CL-Softwares
/
telegraf
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Simplify how nesting is handled (#9504)

This commit is contained in:
Sebastian Spaink 2021-07-22 19:09:01 -05:00 committed by GitHub
parent cae338814b
commit d6b7d4da2c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 70 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
plugins/parsers/json_v2/parser.go
View File

@ -184,11 +184,7 @@ func (p *Parser) processMetric(data []DataSet, input []byte, tag bool) ([]telegr
return nil, err
}
var m []telegraf.Metric
for _, n := range nodes {
m = append(m, n.Metric)
}
metrics = append(metrics, m)
metrics = append(metrics, nodes)
}
for i := 1; i < len(metrics); i++ {
@ -229,8 +225,8 @@ func mergeMetric(a telegraf.Metric, m telegraf.Metric) {
}
// expandArray will recursively create a new MetricNode for each element in a JSON array or single value
func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) {
var results []MetricNode
func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) {
var results []telegraf.Metric
if result.IsObject() {
if !p.iterateObjects {
@ -262,8 +258,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) {
Metric: m,
Result: val,
}
var r []MetricNode
r, err = p.combineObject(n)
r, err := p.combineObject(n)
if err != nil {
return false
}
@ -274,7 +269,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) {
}
if len(results) != 0 {
for _, newResult := range results {
mergeMetric(result.Metric, newResult.Metric)
mergeMetric(result.Metric, newResult)
}
}
return true
@ -294,8 +289,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) {
Metric: m,
Result: val,
}
var r []MetricNode
r, err = p.expandArray(n)
r, err := p.expandArray(n)
if err != nil {
return false
}
@ -335,7 +329,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) {
}
}
results = append(results, result)
results = append(results, result.Metric)
}
return results, nil
@ -369,9 +363,7 @@ func (p *Parser) processObjects(objects []JSONObject, input []byte) ([]telegraf.
if err != nil {
return nil, err
}
for _, m := range metrics {
t = append(t, m.Metric)
}
t = append(t, metrics...)
}
return t, nil
@ -379,12 +371,10 @@ func (p *Parser) processObjects(objects []JSONObject, input []byte) ([]telegraf.
// combineObject will add all fields/tags to a single metric
// If the object has multiple array's as elements it won't comine those, they will remain separate metrics
func (p *Parser) combineObject(result MetricNode) ([]MetricNode, error) {
var results []MetricNode
var combineObjectResult []MetricNode
func (p *Parser) combineObject(result MetricNode) ([]telegraf.Metric, error) {
var results []telegraf.Metric
if result.IsArray() || result.IsObject() {
var err error
var prevArray bool
result.ForEach(func(key, val gjson.Result) bool {
// Determine if field/tag set name is configured
var setName string
@ -436,38 +426,18 @@ func (p *Parser) combineObject(result MetricNode) ([]MetricNode, error) {
}
arrayNode.Tag = tag
if val.IsObject() {
prevArray = false
combineObjectResult, err = p.combineObject(arrayNode)
results, err = p.combineObject(arrayNode)
if err != nil {
return false
}
} else {
var r []MetricNode
r, err = p.expandArray(arrayNode)
r, err := p.expandArray(arrayNode)
if err != nil {
return false
}
if prevArray {
if !arrayNode.IsArray() {
// If another non-array element was found, merge it into all previous gathered metrics
if len(results) != 0 {
for _, newResult := range results {
mergeMetric(result.Metric, newResult.Metric)
}
}
} else {
// Multiple array's won't be merged but kept separate, add additional metrics gathered from an array
results = append(results, r...)
}
} else {
// Continue using the same metric if its an object
results = r
}
if val.IsArray() {
prevArray = true
}
results = cartesianProduct(results, r)
}
return true
@ -477,13 +447,6 @@ func (p *Parser) combineObject(result MetricNode) ([]MetricNode, error) {
return nil, err
}
}
if len(results) == 0 {
// If the results are empty, use the results of the call to combine object
// This happens with nested objects in array's, see the test array_of_objects
results = combineObjectResult
}
return results, nil
}

4
plugins/parsers/json_v2/parser_test.go
View File

@ -21,6 +21,10 @@ func TestData(t *testing.T) {
name string
test string
}{
{
name: "Test complex nesting",
test: "complex_nesting",
},
{
name: "Test having an array of objects",
test: "array_of_objects",

3
plugins/parsers/json_v2/testdata/complex_nesting/expected.out vendored Normal file
View File

@ -0,0 +1,3 @@
file,properties_place=Antelope\ Valley\,\ CA geometry_coordinates=-119.4998333,geometry_type="Point",id="nc73584926",properties_mag=6,properties_updated=1.626277167263e+12,type="Feature"
file,properties_place=Antelope\ Valley\,\ CA geometry_coordinates=38.5075,geometry_type="Point",id="nc73584926",properties_mag=6,properties_updated=1.626277167263e+12,type="Feature"
file,properties_place=Antelope\ Valley\,\ CA geometry_coordinates=7.45,geometry_type="Point",id="nc73584926",properties_mag=6,properties_updated=1.626277167263e+12,type="Feature"

31
plugins/parsers/json_v2/testdata/complex_nesting/input.json vendored Normal file
View File

@ -0,0 +1,31 @@
{
"type": "FeatureCollection",
"metadata": {
"generated": 1626285886000,
"url": "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/significant_week.geojson",
"title": "USGS Significant Earthquakes, Past Week",
"status": 200,
"api": "1.10.3",
"count": 1
},
"features": [
{
"type": "Feature",
"properties": {
"mag": 6,
"place": "Antelope Valley, CA",
"time": 1625784588110,
"updated": 1626277167263
},
"geometry": {
"type": "Point",
"coordinates": [
-119.4998333,
38.5075,
7.45
]
},
"id": "nc73584926"
}
]
}

9
plugins/parsers/json_v2/testdata/complex_nesting/telegraf.conf vendored Normal file
View File

@ -0,0 +1,9 @@
[[inputs.file]]
files = ["./testdata/complex_nesting/input.json"]
data_format = "json_v2"
[[inputs.file.json_v2]]
[[inputs.file.json_v2.object]]
path = "features"
timestamp_key = "properties_time"
timestamp_format = "unix_ms"
tags = ["properties_place"]

15
plugins/parsers/json_v2/testdata/multiple_arrays_in_object/expected.out vendored
View File

@ -1,6 +1,9 @@
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party"
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past"
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",name="Bilbo",species="hobbit"
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",name="Frodo",species="hobbit"
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",random=1
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",random=2
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Bilbo",species="hobbit",random=1
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Bilbo",species="hobbit",random=2
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Frodo",species="hobbit",random=1
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Frodo",species="hobbit",random=2
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Bilbo",species="hobbit",random=1
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Bilbo",species="hobbit",random=2
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Frodo",species="hobbit",random=1
file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Frodo",species="hobbit",random=2