diff --git a/plugins/parsers/json/parser_test.go b/plugins/parsers/json/parser_test.go index 3f6b1ecbb..75d37d73a 100644 --- a/plugins/parsers/json/parser_test.go +++ b/plugins/parsers/json/parser_test.go @@ -9,6 +9,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal/fuzz" + "github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/testutil" ) @@ -76,52 +77,86 @@ const validJSONArrayTags = ` ] ` +const benchmarkData = ` +[ + { + "name": "impression", + "timestamp": 1653643420, + "fields": { + "count_sum": 5 + }, + "tags": { + "key": "12345", + "flagname": "F5", + "host": "1cbbb3796fc2", + "platform": "Java", + "sdkver": "4.9.1", + "value": "false" + } + }, + { + "name": "expression", + "timestamp": 1653646789, + "fields": { + "count_sum": 42 + }, + "tags": { + "key": "67890", + "flagname": "E42", + "host": "klaus", + "platform": "Golang", + "sdkver": "1.18.3", + "value": "true" + } + } +]` + func TestParseValidJSON(t *testing.T) { parser := &Parser{MetricName: "json_test"} require.NoError(t, parser.Init()) // Most basic vanilla test - metrics, err := parser.Parse([]byte(validJSON)) + actual, err := parser.Parse([]byte(validJSON)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) - require.Equal(t, map[string]string{}, metrics[0].Tags()) + }, actual[0].Fields()) + require.Equal(t, map[string]string{}, actual[0].Tags()) // Test that newlines are fine - metrics, err = parser.Parse([]byte(validJSONNewline)) + actual, err = parser.Parse([]byte(validJSONNewline)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "d": float64(7), "b_d": float64(8), - }, metrics[0].Fields()) - require.Equal(t, map[string]string{}, metrics[0].Tags()) + }, actual[0].Fields()) + require.Equal(t, map[string]string{}, actual[0].Tags()) // Test that strings without TagKeys defined are ignored - metrics, err = parser.Parse([]byte(validJSONTags)) + actual, err = parser.Parse([]byte(validJSONTags)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) - require.Equal(t, map[string]string{}, metrics[0].Tags()) + }, actual[0].Fields()) + require.Equal(t, map[string]string{}, actual[0].Tags()) - // Test that whitespace only will parse as an empty list of metrics - metrics, err = parser.Parse([]byte("\n\t")) + // Test that whitespace only will parse as an empty list of actual + actual, err = parser.Parse([]byte("\n\t")) require.NoError(t, err) - require.Empty(t, metrics) + require.Empty(t, actual) - // Test that an empty string will parse as an empty list of metrics - metrics, err = parser.Parse([]byte("")) + // Test that an empty string will parse as an empty list of actual + actual, err = parser.Parse([]byte("")) require.NoError(t, err) - require.Empty(t, metrics) + require.Empty(t, actual) } func TestParseLineValidJSON(t *testing.T) { @@ -129,34 +164,34 @@ func TestParseLineValidJSON(t *testing.T) { require.NoError(t, parser.Init()) // Most basic vanilla test - metric, err := parser.ParseLine(validJSON) + actual, err := parser.ParseLine(validJSON) require.NoError(t, err) - require.Equal(t, "json_test", metric.Name()) + require.Equal(t, "json_test", actual.Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metric.Fields()) - require.Equal(t, map[string]string{}, metric.Tags()) + }, actual.Fields()) + require.Equal(t, map[string]string{}, actual.Tags()) // Test that newlines are fine - metric, err = parser.ParseLine(validJSONNewline) + actual, err = parser.ParseLine(validJSONNewline) require.NoError(t, err) - require.Equal(t, "json_test", metric.Name()) + require.Equal(t, "json_test", actual.Name()) require.Equal(t, map[string]interface{}{ "d": float64(7), "b_d": float64(8), - }, metric.Fields()) - require.Equal(t, map[string]string{}, metric.Tags()) + }, actual.Fields()) + require.Equal(t, map[string]string{}, actual.Tags()) // Test that strings without TagKeys defined are ignored - metric, err = parser.ParseLine(validJSONTags) + actual, err = parser.ParseLine(validJSONTags) require.NoError(t, err) - require.Equal(t, "json_test", metric.Name()) + require.Equal(t, "json_test", actual.Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metric.Fields()) - require.Equal(t, map[string]string{}, metric.Tags()) + }, actual.Fields()) + require.Equal(t, map[string]string{}, actual.Tags()) } func TestParseInvalidJSON(t *testing.T) { @@ -214,15 +249,15 @@ func TestParseWithTagKeys(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(validJSONTags)) + actual, err := parser.Parse([]byte(validJSONTags)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) - require.Equal(t, map[string]string{}, metrics[0].Tags()) + }, actual[0].Fields()) + require.Equal(t, map[string]string{}, actual[0].Tags()) // Test that single tag key is found and applied parser = &Parser{ @@ -231,17 +266,17 @@ func TestParseWithTagKeys(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err = parser.Parse([]byte(validJSONTags)) + actual, err = parser.Parse([]byte(validJSONTags)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) + }, actual[0].Fields()) require.Equal(t, map[string]string{ "mytag": "foobar", - }, metrics[0].Tags()) + }, actual[0].Tags()) // Test that both tag keys are found and applied parser = &Parser{ @@ -250,18 +285,18 @@ func TestParseWithTagKeys(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err = parser.Parse([]byte(validJSONTags)) + actual, err = parser.Parse([]byte(validJSONTags)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) + }, actual[0].Fields()) require.Equal(t, map[string]string{ "mytag": "foobar", "othertag": "baz", - }, metrics[0].Tags()) + }, actual[0].Tags()) } func TestParseLineWithTagKeys(t *testing.T) { @@ -272,14 +307,14 @@ func TestParseLineWithTagKeys(t *testing.T) { } require.NoError(t, parser.Init()) - metric, err := parser.ParseLine(validJSONTags) + actual, err := parser.ParseLine(validJSONTags) require.NoError(t, err) - require.Equal(t, "json_test", metric.Name()) + require.Equal(t, "json_test", actual.Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metric.Fields()) - require.Equal(t, map[string]string{}, metric.Tags()) + }, actual.Fields()) + require.Equal(t, map[string]string{}, actual.Tags()) // Test that single tag key is found and applied parser = &Parser{ @@ -288,16 +323,16 @@ func TestParseLineWithTagKeys(t *testing.T) { } require.NoError(t, parser.Init()) - metric, err = parser.ParseLine(validJSONTags) + actual, err = parser.ParseLine(validJSONTags) require.NoError(t, err) - require.Equal(t, "json_test", metric.Name()) + require.Equal(t, "json_test", actual.Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metric.Fields()) + }, actual.Fields()) require.Equal(t, map[string]string{ "mytag": "foobar", - }, metric.Tags()) + }, actual.Tags()) // Test that both tag keys are found and applied parser = &Parser{ @@ -306,17 +341,17 @@ func TestParseLineWithTagKeys(t *testing.T) { } require.NoError(t, parser.Init()) - metric, err = parser.ParseLine(validJSONTags) + actual, err = parser.ParseLine(validJSONTags) require.NoError(t, err) - require.Equal(t, "json_test", metric.Name()) + require.Equal(t, "json_test", actual.Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metric.Fields()) + }, actual.Fields()) require.Equal(t, map[string]string{ "mytag": "foobar", "othertag": "baz", - }, metric.Tags()) + }, actual.Tags()) } func TestParseValidJSONDefaultTags(t *testing.T) { @@ -328,29 +363,29 @@ func TestParseValidJSONDefaultTags(t *testing.T) { require.NoError(t, parser.Init()) // Most basic vanilla test - metrics, err := parser.Parse([]byte(validJSON)) + actual, err := parser.Parse([]byte(validJSON)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) - require.Equal(t, map[string]string{"t4g": "default"}, metrics[0].Tags()) + }, actual[0].Fields()) + require.Equal(t, map[string]string{"t4g": "default"}, actual[0].Tags()) // Test that tagkeys and default tags are applied - metrics, err = parser.Parse([]byte(validJSONTags)) + actual, err = parser.Parse([]byte(validJSONTags)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) + }, actual[0].Fields()) require.Equal(t, map[string]string{ "t4g": "default", "mytag": "foobar", - }, metrics[0].Tags()) + }, actual[0].Tags()) } // Test that default tags are overridden by tag keys @@ -363,28 +398,28 @@ func TestParseValidJSONDefaultTagsOverride(t *testing.T) { require.NoError(t, parser.Init()) // Most basic vanilla test - metrics, err := parser.Parse([]byte(validJSON)) + actual, err := parser.Parse([]byte(validJSON)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) - require.Equal(t, map[string]string{"mytag": "default"}, metrics[0].Tags()) + }, actual[0].Fields()) + require.Equal(t, map[string]string{"mytag": "default"}, actual[0].Tags()) // Test that tagkeys override default tags - metrics, err = parser.Parse([]byte(validJSONTags)) + actual, err = parser.Parse([]byte(validJSONTags)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) + }, actual[0].Fields()) require.Equal(t, map[string]string{ "mytag": "foobar", - }, metrics[0].Tags()) + }, actual[0].Tags()) } // Test that json arrays can be parsed @@ -393,32 +428,32 @@ func TestParseValidJSONArray(t *testing.T) { require.NoError(t, parser.Init()) // Most basic vanilla test - metrics, err := parser.Parse([]byte(validJSONArray)) + actual, err := parser.Parse([]byte(validJSONArray)) require.NoError(t, err) - require.Len(t, metrics, 1) - require.Equal(t, "json_array_test", metrics[0].Name()) + require.Len(t, actual, 1) + require.Equal(t, "json_array_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) - require.Equal(t, map[string]string{}, metrics[0].Tags()) + }, actual[0].Fields()) + require.Equal(t, map[string]string{}, actual[0].Tags()) // Basic multiple datapoints - metrics, err = parser.Parse([]byte(validJSONArrayMultiple)) + actual, err = parser.Parse([]byte(validJSONArrayMultiple)) require.NoError(t, err) - require.Len(t, metrics, 2) - require.Equal(t, "json_array_test", metrics[0].Name()) + require.Len(t, actual, 2) + require.Equal(t, "json_array_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) - require.Equal(t, map[string]string{}, metrics[1].Tags()) - require.Equal(t, "json_array_test", metrics[1].Name()) + }, actual[0].Fields()) + require.Equal(t, map[string]string{}, actual[1].Tags()) + require.Equal(t, "json_array_test", actual[1].Name()) require.Equal(t, map[string]interface{}{ "a": float64(7), "b_c": float64(8), - }, metrics[1].Fields()) - require.Equal(t, map[string]string{}, metrics[1].Tags()) + }, actual[1].Fields()) + require.Equal(t, map[string]string{}, actual[1].Tags()) } func TestParseArrayWithTagKeys(t *testing.T) { @@ -429,22 +464,22 @@ func TestParseArrayWithTagKeys(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(validJSONArrayTags)) + actual, err := parser.Parse([]byte(validJSONArrayTags)) require.NoError(t, err) - require.Len(t, metrics, 2) - require.Equal(t, "json_array_test", metrics[0].Name()) + require.Len(t, actual, 2) + require.Equal(t, "json_array_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) - require.Equal(t, map[string]string{}, metrics[0].Tags()) + }, actual[0].Fields()) + require.Equal(t, map[string]string{}, actual[0].Tags()) - require.Equal(t, "json_array_test", metrics[1].Name()) + require.Equal(t, "json_array_test", actual[1].Name()) require.Equal(t, map[string]interface{}{ "a": float64(7), "b_c": float64(8), - }, metrics[1].Fields()) - require.Equal(t, map[string]string{}, metrics[1].Tags()) + }, actual[1].Fields()) + require.Equal(t, map[string]string{}, actual[1].Tags()) // Test that single tag key is found and applied parser = &Parser{ @@ -453,26 +488,26 @@ func TestParseArrayWithTagKeys(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err = parser.Parse([]byte(validJSONArrayTags)) + actual, err = parser.Parse([]byte(validJSONArrayTags)) require.NoError(t, err) - require.Len(t, metrics, 2) - require.Equal(t, "json_array_test", metrics[0].Name()) + require.Len(t, actual, 2) + require.Equal(t, "json_array_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) + }, actual[0].Fields()) require.Equal(t, map[string]string{ "mytag": "foo", - }, metrics[0].Tags()) + }, actual[0].Tags()) - require.Equal(t, "json_array_test", metrics[1].Name()) + require.Equal(t, "json_array_test", actual[1].Name()) require.Equal(t, map[string]interface{}{ "a": float64(7), "b_c": float64(8), - }, metrics[1].Fields()) + }, actual[1].Fields()) require.Equal(t, map[string]string{ "mytag": "bar", - }, metrics[1].Tags()) + }, actual[1].Tags()) // Test that both tag keys are found and applied parser = &Parser{ @@ -481,28 +516,28 @@ func TestParseArrayWithTagKeys(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err = parser.Parse([]byte(validJSONArrayTags)) + actual, err = parser.Parse([]byte(validJSONArrayTags)) require.NoError(t, err) - require.Len(t, metrics, 2) - require.Equal(t, "json_array_test", metrics[0].Name()) + require.Len(t, actual, 2) + require.Equal(t, "json_array_test", actual[0].Name()) require.Equal(t, map[string]interface{}{ "a": float64(5), "b_c": float64(6), - }, metrics[0].Fields()) + }, actual[0].Fields()) require.Equal(t, map[string]string{ "mytag": "foo", "othertag": "baz", - }, metrics[0].Tags()) + }, actual[0].Tags()) - require.Equal(t, "json_array_test", metrics[1].Name()) + require.Equal(t, "json_array_test", actual[1].Name()) require.Equal(t, map[string]interface{}{ "a": float64(7), "b_c": float64(8), - }, metrics[1].Fields()) + }, actual[1].Fields()) require.Equal(t, map[string]string{ "mytag": "bar", "othertag": "baz", - }, metrics[1].Tags()) + }, actual[1].Tags()) } var jsonBOM = []byte("\xef\xbb\xbf[{\"value\":17}]") @@ -542,10 +577,10 @@ func TestJSONParseNestedArray(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(testString)) - require.Len(t, metrics, 1) + actual, err := parser.Parse([]byte(testString)) + require.Len(t, actual, 1) require.NoError(t, err) - require.Equal(t, 3, len(metrics[0].Tags())) + require.Equal(t, 3, len(actual[0].Tags())) } func TestJSONQueryErrorOnArray(t *testing.T) { @@ -606,9 +641,9 @@ func TestArrayOfObjects(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(testString)) + actual, err := parser.Parse([]byte(testString)) require.NoError(t, err) - require.Equal(t, 3, len(metrics)) + require.Equal(t, 3, len(actual)) } func TestUseCaseJSONQuery(t *testing.T) { @@ -634,10 +669,10 @@ func TestUseCaseJSONQuery(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(testString)) + actual, err := parser.Parse([]byte(testString)) require.NoError(t, err) - require.Equal(t, 3, len(metrics)) - require.Equal(t, metrics[0].Fields()["last"], "Murphy") + require.Equal(t, 3, len(actual)) + require.Equal(t, actual[0].Fields()["last"], "Murphy") } func TestTimeParser(t *testing.T) { @@ -669,10 +704,10 @@ func TestTimeParser(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(testString)) + actual, err := parser.Parse([]byte(testString)) require.NoError(t, err) - require.Equal(t, 2, len(metrics)) - require.NotEqual(t, metrics[0].Time(), metrics[1].Time()) + require.Equal(t, 2, len(actual)) + require.NotEqual(t, actual[0].Time(), actual[1].Time()) } func TestTimeParserWithTimezone(t *testing.T) { @@ -688,10 +723,10 @@ func TestTimeParserWithTimezone(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(testString)) + actual, err := parser.Parse([]byte(testString)) require.NoError(t, err) - require.Equal(t, 1, len(metrics)) - require.EqualValues(t, int64(1136405040000000000), metrics[0].Time().UnixNano()) + require.Equal(t, 1, len(actual)) + require.EqualValues(t, int64(1136405040000000000), actual[0].Time().UnixNano()) } func TestUnixTimeParser(t *testing.T) { @@ -723,10 +758,10 @@ func TestUnixTimeParser(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(testString)) + actual, err := parser.Parse([]byte(testString)) require.NoError(t, err) - require.Equal(t, 2, len(metrics)) - require.NotEqual(t, metrics[0].Time(), metrics[1].Time()) + require.Equal(t, 2, len(actual)) + require.NotEqual(t, actual[0].Time(), actual[1].Time()) } func TestUnixMsTimeParser(t *testing.T) { @@ -758,10 +793,10 @@ func TestUnixMsTimeParser(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(testString)) + actual, err := parser.Parse([]byte(testString)) require.NoError(t, err) - require.Equal(t, 2, len(metrics)) - require.NotEqual(t, metrics[0].Time(), metrics[1].Time()) + require.Equal(t, 2, len(actual)) + require.NotEqual(t, actual[0].Time(), actual[1].Time()) } func TestTimeErrors(t *testing.T) { @@ -782,9 +817,9 @@ func TestTimeErrors(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(testString)) + actual, err := parser.Parse([]byte(testString)) require.Error(t, err) - require.Empty(t, metrics) + require.Empty(t, actual) testString2 := `{ "a": 5, @@ -802,9 +837,9 @@ func TestTimeErrors(t *testing.T) { } require.NoError(t, parser.Init()) - metrics, err = parser.Parse([]byte(testString2)) + actual, err = parser.Parse([]byte(testString2)) require.Error(t, err) - require.Empty(t, metrics) + require.Empty(t, actual) require.Equal(t, fmt.Errorf("JSON time key could not be found"), err) } @@ -812,10 +847,10 @@ func TestShareTimestamp(t *testing.T) { parser := &Parser{MetricName: "json_test"} require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(validJSONArrayMultiple)) + actual, err := parser.Parse([]byte(validJSONArrayMultiple)) require.NoError(t, err) - require.Equal(t, 2, len(metrics)) - require.Equal(t, metrics[0].Time(), metrics[1].Time()) + require.Equal(t, 2, len(actual)) + require.Equal(t, actual[0].Time(), actual[1].Time()) } func TestNameKey(t *testing.T) { @@ -832,9 +867,9 @@ func TestNameKey(t *testing.T) { parser := &Parser{NameKey: "b_c"} require.NoError(t, parser.Init()) - metrics, err := parser.Parse([]byte(testString)) + actual, err := parser.Parse([]byte(testString)) require.NoError(t, err) - require.Equal(t, "this is my name", metrics[0].Name()) + require.Equal(t, "this is my name", actual[0].Name()) } func TestParseArrayWithWrongType(t *testing.T) { @@ -1359,6 +1394,85 @@ func TestParseArrayWithWildcardTagKeys(t *testing.T) { } } +func TestBenchmarkData(t *testing.T) { + // Setup the plugin + plugin := &Parser{ + MetricName: "benchmark", + TagKeys: []string{"tags_*"}, + } + require.NoError(t, plugin.Init()) + + expected := []telegraf.Metric{ + metric.New( + "benchmark", + map[string]string{ + "tags_flagname": "F5", + "tags_host": "1cbbb3796fc2", + "tags_key": "12345", + "tags_platform": "Java", + "tags_sdkver": "4.9.1", + "tags_value": "false", + }, + map[string]interface{}{ + "fields_count_sum": float64(5), + "timestamp": float64(1653643420), + }, + time.Unix(0, 0), + ), + metric.New( + "benchmark", + map[string]string{ + "tags_flagname": "E42", + "tags_host": "klaus", + "tags_key": "67890", + "tags_platform": "Golang", + "tags_sdkver": "1.18.3", + "tags_value": "true", + }, + map[string]interface{}{ + "fields_count_sum": float64(42), + "timestamp": float64(1653646789), + }, + time.Unix(0, 0), + ), + } + + // Do the parsing + actual, err := plugin.Parse([]byte(benchmarkData)) + require.NoError(t, err) + testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime()) +} + +func BenchmarkParsingSequential(b *testing.B) { + // Configure the plugin + plugin := &Parser{ + MetricName: "benchmark", + TagKeys: []string{"tags_*"}, + } + require.NoError(b, plugin.Init()) + + // Do the benchmarking + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse([]byte(benchmarkData)) + } +} + +func BenchmarkParsingParallel(b *testing.B) { + // Configure the plugin + plugin := &Parser{ + MetricName: "benchmark", + TagKeys: []string{"tags_*"}, + } + require.NoError(b, plugin.Init()) + + // Do the benchmarking + b.RunParallel(func(p *testing.PB) { + for p.Next() { + _, _ = plugin.Parse([]byte(benchmarkData)) + } + }) +} + func FuzzParserJSON(f *testing.F) { for _, value := range fuzz.JSONDictionary { f.Add([]byte(value)) diff --git a/plugins/parsers/json_v2/parser_test.go b/plugins/parsers/json_v2/parser_test.go index b0749d992..498afd4e5 100644 --- a/plugins/parsers/json_v2/parser_test.go +++ b/plugins/parsers/json_v2/parser_test.go @@ -15,6 +15,7 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/inputs/file" "github.com/influxdata/telegraf/plugins/parsers/influx" + "github.com/influxdata/telegraf/plugins/parsers/json_v2" "github.com/influxdata/telegraf/testutil" ) @@ -34,6 +35,10 @@ func TestMultipleConfigs(t *testing.T) { }) for _, f := range folders { + // Only use directories as those contain test-cases + if !f.IsDir() { + continue + } testdataPath := filepath.Join("testdata", f.Name()) configFilename := filepath.Join(testdataPath, "telegraf.conf") expectedFilename := filepath.Join(testdataPath, "expected.out") @@ -94,3 +99,63 @@ func TestMultipleConfigs(t *testing.T) { }) } } + +func BenchmarkParsingSequential(b *testing.B) { + inputFilename := filepath.Join("testdata", "benchmark", "input.json") + + // Configure the plugin + plugin := &json_v2.Parser{ + Configs: []json_v2.Config{ + { + MeasurementName: "benchmark", + JSONObjects: []json_v2.Object{ + { + Path: "metrics", + DisablePrependKeys: true, + }, + }, + }, + }, + } + require.NoError(b, plugin.Init()) + + // Read the input data + input, err := os.ReadFile(inputFilename) + require.NoError(b, err) + + // Do the benchmarking + for n := 0; n < b.N; n++ { + _, _ = plugin.Parse(input) + } +} + +func BenchmarkParsingParallel(b *testing.B) { + inputFilename := filepath.Join("testdata", "benchmark", "input.json") + + // Configure the plugin + plugin := &json_v2.Parser{ + Configs: []json_v2.Config{ + { + MeasurementName: "benchmark", + JSONObjects: []json_v2.Object{ + { + Path: "metrics", + DisablePrependKeys: true, + }, + }, + }, + }, + } + require.NoError(b, plugin.Init()) + + // Read the input data + input, err := os.ReadFile(inputFilename) + require.NoError(b, err) + + // Do the benchmarking + b.RunParallel(func(p *testing.PB) { + for p.Next() { + _, _ = plugin.Parse(input) + } + }) +} diff --git a/plugins/parsers/json_v2/testdata/benchmark/benchmark.json b/plugins/parsers/json_v2/testdata/benchmark/benchmark.json new file mode 100644 index 000000000..4f17cb096 --- /dev/null +++ b/plugins/parsers/json_v2/testdata/benchmark/benchmark.json @@ -0,0 +1,34 @@ +{ + "metrics": [ + { + "name": "impression", + "timestamp": 1653643420, + "fields": { + "count_sum": 5 + }, + "tags": { + "key": "12345", + "flagname": "F5", + "host": "1cbbb3796fc2", + "platform": "Java", + "sdkver": "4.9.1", + "value": "false" + } + }, + { + "name": "expression", + "timestamp": 1653646789, + "fields": { + "count_sum": 42 + }, + "tags": { + "key": "67890", + "flagname": "E42", + "host": "klaus", + "platform": "Golang", + "sdkver": "1.18.3", + "value": "true" + } + } + ] +} \ No newline at end of file diff --git a/plugins/parsers/json_v2/testdata/benchmark/expected.out b/plugins/parsers/json_v2/testdata/benchmark/expected.out new file mode 100644 index 000000000..3bd3868db --- /dev/null +++ b/plugins/parsers/json_v2/testdata/benchmark/expected.out @@ -0,0 +1,2 @@ +benchmark count_sum=5,flagname="F5",host="1cbbb3796fc2",key="12345",name="impression",platform="Java",sdkver="4.9.1",timestamp=1653643420,value="false" 1697730835574029794 +benchmark count_sum=42,flagname="E42",host="klaus",key="67890",name="expression",platform="Golang",sdkver="1.18.3",timestamp=1653646789,value="true" 1697730835574029794 diff --git a/plugins/parsers/json_v2/testdata/benchmark/input.json b/plugins/parsers/json_v2/testdata/benchmark/input.json new file mode 100644 index 000000000..915ff465c --- /dev/null +++ b/plugins/parsers/json_v2/testdata/benchmark/input.json @@ -0,0 +1,34 @@ +{ + "metrics": [ + { + "name": "impression", + "timestamp": 1653643420, + "fields": { + "count_sum": 5 + }, + "tags": { + "key": "12345", + "flagname": "F5", + "host": "1cbbb3796fc2", + "platform": "Java", + "sdkver": "4.9.1", + "value": "false" + } + }, + { + "name": "expression", + "timestamp": 1653646789, + "fields": { + "count_sum": 42 + }, + "tags": { + "key": "67890", + "flagname": "E42", + "host": "klaus", + "platform": "Golang", + "sdkver": "1.18.3", + "value": "true" + } + } + ] + } \ No newline at end of file diff --git a/plugins/parsers/json_v2/testdata/benchmark/telegraf.conf b/plugins/parsers/json_v2/testdata/benchmark/telegraf.conf new file mode 100644 index 000000000..6c498fbbb --- /dev/null +++ b/plugins/parsers/json_v2/testdata/benchmark/telegraf.conf @@ -0,0 +1,9 @@ +[[inputs.file]] + files = ["./testdata/benchmark/input.json"] + data_format = "json_v2" + + [[inputs.file.json_v2]] + measurement_name = "benchmark" + [[inputs.file.json_v2.object]] + path = 'metrics' + disable_prepend_keys = true