Simplify how nesting is handled (#9504)

plugins/parsers/json_v2/parser.go

@@ -184,11 +184,7 @@ func (p *Parser) processMetric(data []DataSet, input []byte, tag bool) ([]telegr
 			return nil, err
 		}
 
-		var m []telegraf.Metric
+		metrics = append(metrics, nodes)
-		for _, n := range nodes {
-			m = append(m, n.Metric)
-		}
-		metrics = append(metrics, m)
 	}
 
 	for i := 1; i < len(metrics); i++ {
@@ -229,8 +225,8 @@ func mergeMetric(a telegraf.Metric, m telegraf.Metric) {
 }
 
 // expandArray will recursively create a new MetricNode for each element in a JSON array or single value
-func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) {
+func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) {
-	var results []MetricNode
+	var results []telegraf.Metric
 
 	if result.IsObject() {
 		if !p.iterateObjects {
@@ -262,8 +258,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) {
 						Metric:  m,
 						Result:  val,
 					}
-					var r []MetricNode
+					r, err := p.combineObject(n)
-					r, err = p.combineObject(n)
 					if err != nil {
 						return false
 					}
@@ -274,7 +269,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) {
 				}
 				if len(results) != 0 {
 					for _, newResult := range results {
-						mergeMetric(result.Metric, newResult.Metric)
+						mergeMetric(result.Metric, newResult)
 					}
 				}
 				return true
@@ -294,8 +289,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) {
 				Metric:      m,
 				Result:      val,
 			}
-			var r []MetricNode
+			r, err := p.expandArray(n)
-			r, err = p.expandArray(n)
 			if err != nil {
 				return false
 			}
@@ -335,7 +329,7 @@ func (p *Parser) expandArray(result MetricNode) ([]MetricNode, error) {
 			}
 		}
 
-		results = append(results, result)
+		results = append(results, result.Metric)
 	}
 
 	return results, nil
@@ -369,9 +363,7 @@ func (p *Parser) processObjects(objects []JSONObject, input []byte) ([]telegraf.
 		if err != nil {
 			return nil, err
 		}
-		for _, m := range metrics {
+		t = append(t, metrics...)
-			t = append(t, m.Metric)
-		}
 	}
 
 	return t, nil
@@ -379,12 +371,10 @@ func (p *Parser) processObjects(objects []JSONObject, input []byte) ([]telegraf.
 
 // combineObject will add all fields/tags to a single metric
 // If the object has multiple array's as elements it won't comine those, they will remain separate metrics
-func (p *Parser) combineObject(result MetricNode) ([]MetricNode, error) {
+func (p *Parser) combineObject(result MetricNode) ([]telegraf.Metric, error) {
-	var results []MetricNode
+	var results []telegraf.Metric
-	var combineObjectResult []MetricNode
 	if result.IsArray() || result.IsObject() {
 		var err error
-		var prevArray bool
 		result.ForEach(func(key, val gjson.Result) bool {
 			// Determine if field/tag set name is configured
 			var setName string
@@ -436,38 +426,18 @@ func (p *Parser) combineObject(result MetricNode) ([]MetricNode, error) {
 			}
 
 			arrayNode.Tag = tag
-			if val.IsObject() {
-				prevArray = false
-				combineObjectResult, err = p.combineObject(arrayNode)
-				if err != nil {
-					return false
-				}
-			} else {
-				var r []MetricNode
-				r, err = p.expandArray(arrayNode)
-				if err != nil {
-					return false
-				}
-				if prevArray {
-					if !arrayNode.IsArray() {
-						// If another non-array element was found, merge it into all previous gathered metrics
-						if len(results) != 0 {
-							for _, newResult := range results {
-								mergeMetric(result.Metric, newResult.Metric)
-							}
-						}
-					} else {
-						// Multiple array's won't be merged but kept separate, add additional metrics gathered from an array
-						results = append(results, r...)
-					}
-				} else {
-					// Continue using the same metric if its an object
-					results = r
-				}
 
-				if val.IsArray() {
+			if val.IsObject() {
-					prevArray = true
+				results, err = p.combineObject(arrayNode)
+				if err != nil {
+					return false
 				}
+			} else {
+				r, err := p.expandArray(arrayNode)
+				if err != nil {
+					return false
+				}
+				results = cartesianProduct(results, r)
 			}
 
 			return true
@@ -477,13 +447,6 @@ func (p *Parser) combineObject(result MetricNode) ([]MetricNode, error) {
 			return nil, err
 		}
 	}
-
-	if len(results) == 0 {
-		// If the results are empty, use the results of the call to combine object
-		// This happens with nested objects in array's, see the test array_of_objects
-		results = combineObjectResult
-	}
-
 	return results, nil
 }
 

plugins/parsers/json_v2/parser_test.go

@@ -21,6 +21,10 @@ func TestData(t *testing.T) {
 		name string
 		test string
 	}{
+		{
+			name: "Test complex nesting",
+			test: "complex_nesting",
+		},
 		{
 			name: "Test having an array of objects",
 			test: "array_of_objects",

plugins/parsers/json_v2/testdata/complex_nesting/expected.out

@@ -0,0 +1,3 @@
+file,properties_place=Antelope\ Valley\,\ CA geometry_coordinates=-119.4998333,geometry_type="Point",id="nc73584926",properties_mag=6,properties_updated=1.626277167263e+12,type="Feature"
+file,properties_place=Antelope\ Valley\,\ CA geometry_coordinates=38.5075,geometry_type="Point",id="nc73584926",properties_mag=6,properties_updated=1.626277167263e+12,type="Feature"
+file,properties_place=Antelope\ Valley\,\ CA geometry_coordinates=7.45,geometry_type="Point",id="nc73584926",properties_mag=6,properties_updated=1.626277167263e+12,type="Feature"

plugins/parsers/json_v2/testdata/complex_nesting/input.json

@@ -0,0 +1,31 @@
+{
+    "type": "FeatureCollection",
+    "metadata": {
+        "generated": 1626285886000,
+        "url": "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/significant_week.geojson",
+        "title": "USGS Significant Earthquakes, Past Week",
+        "status": 200,
+        "api": "1.10.3",
+        "count": 1
+    },
+    "features": [
+        {
+            "type": "Feature",
+            "properties": {
+                "mag": 6,
+                "place": "Antelope Valley, CA",
+                "time": 1625784588110,
+                "updated": 1626277167263
+            },
+            "geometry": {
+                "type": "Point",
+                "coordinates": [
+                    -119.4998333,
+                    38.5075,
+                    7.45
+                ]
+            },
+            "id": "nc73584926"
+        }
+    ]
+}

plugins/parsers/json_v2/testdata/complex_nesting/telegraf.conf

@@ -0,0 +1,9 @@
+[[inputs.file]]
+    files = ["./testdata/complex_nesting/input.json"]
+    data_format = "json_v2"
+    [[inputs.file.json_v2]]
+        [[inputs.file.json_v2.object]]
+            path = "features"
+            timestamp_key = "properties_time"
+            timestamp_format = "unix_ms"
+            tags = ["properties_place"]

plugins/parsers/json_v2/testdata/multiple_arrays_in_object/expected.out

@@ -1,6 +1,9 @@
-file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party"
+file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Bilbo",species="hobbit",random=1
-file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past"
+file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Bilbo",species="hobbit",random=2
-file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",name="Bilbo",species="hobbit"
+file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Frodo",species="hobbit",random=1
-file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",name="Frodo",species="hobbit"
+file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="A Long-expected Party",name="Frodo",species="hobbit",random=2
-file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",random=1
+file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Bilbo",species="hobbit",random=1
-file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",random=2
+file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Bilbo",species="hobbit",random=2
+file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Frodo",species="hobbit",random=1
+file,title=The\ Lord\ Of\ The\ Rings author="Tolkien",chapters="The Shadow of the Past",name="Frodo",species="hobbit",random=2
+