test(parsers): Add benchmarks (#14148)

This commit is contained in:
Sven Rebhan 2023-10-26 01:05:25 +02:00 committed by GitHub
parent fe3100fb39
commit 321c5a4070
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 404 additions and 146 deletions

View File

@ -9,6 +9,7 @@ import (
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal/fuzz" "github.com/influxdata/telegraf/internal/fuzz"
"github.com/influxdata/telegraf/metric"
"github.com/influxdata/telegraf/testutil" "github.com/influxdata/telegraf/testutil"
) )
@ -76,52 +77,86 @@ const validJSONArrayTags = `
] ]
` `
const benchmarkData = `
[
{
"name": "impression",
"timestamp": 1653643420,
"fields": {
"count_sum": 5
},
"tags": {
"key": "12345",
"flagname": "F5",
"host": "1cbbb3796fc2",
"platform": "Java",
"sdkver": "4.9.1",
"value": "false"
}
},
{
"name": "expression",
"timestamp": 1653646789,
"fields": {
"count_sum": 42
},
"tags": {
"key": "67890",
"flagname": "E42",
"host": "klaus",
"platform": "Golang",
"sdkver": "1.18.3",
"value": "true"
}
}
]`
func TestParseValidJSON(t *testing.T) { func TestParseValidJSON(t *testing.T) {
parser := &Parser{MetricName: "json_test"} parser := &Parser{MetricName: "json_test"}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
// Most basic vanilla test // Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSON)) actual, err := parser.Parse([]byte(validJSON))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_test", metrics[0].Name()) require.Equal(t, "json_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{}, metrics[0].Tags()) require.Equal(t, map[string]string{}, actual[0].Tags())
// Test that newlines are fine // Test that newlines are fine
metrics, err = parser.Parse([]byte(validJSONNewline)) actual, err = parser.Parse([]byte(validJSONNewline))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_test", metrics[0].Name()) require.Equal(t, "json_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"d": float64(7), "d": float64(7),
"b_d": float64(8), "b_d": float64(8),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{}, metrics[0].Tags()) require.Equal(t, map[string]string{}, actual[0].Tags())
// Test that strings without TagKeys defined are ignored // Test that strings without TagKeys defined are ignored
metrics, err = parser.Parse([]byte(validJSONTags)) actual, err = parser.Parse([]byte(validJSONTags))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_test", metrics[0].Name()) require.Equal(t, "json_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{}, metrics[0].Tags()) require.Equal(t, map[string]string{}, actual[0].Tags())
// Test that whitespace only will parse as an empty list of metrics // Test that whitespace only will parse as an empty list of actual
metrics, err = parser.Parse([]byte("\n\t")) actual, err = parser.Parse([]byte("\n\t"))
require.NoError(t, err) require.NoError(t, err)
require.Empty(t, metrics) require.Empty(t, actual)
// Test that an empty string will parse as an empty list of metrics // Test that an empty string will parse as an empty list of actual
metrics, err = parser.Parse([]byte("")) actual, err = parser.Parse([]byte(""))
require.NoError(t, err) require.NoError(t, err)
require.Empty(t, metrics) require.Empty(t, actual)
} }
func TestParseLineValidJSON(t *testing.T) { func TestParseLineValidJSON(t *testing.T) {
@ -129,34 +164,34 @@ func TestParseLineValidJSON(t *testing.T) {
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
// Most basic vanilla test // Most basic vanilla test
metric, err := parser.ParseLine(validJSON) actual, err := parser.ParseLine(validJSON)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "json_test", metric.Name()) require.Equal(t, "json_test", actual.Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metric.Fields()) }, actual.Fields())
require.Equal(t, map[string]string{}, metric.Tags()) require.Equal(t, map[string]string{}, actual.Tags())
// Test that newlines are fine // Test that newlines are fine
metric, err = parser.ParseLine(validJSONNewline) actual, err = parser.ParseLine(validJSONNewline)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "json_test", metric.Name()) require.Equal(t, "json_test", actual.Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"d": float64(7), "d": float64(7),
"b_d": float64(8), "b_d": float64(8),
}, metric.Fields()) }, actual.Fields())
require.Equal(t, map[string]string{}, metric.Tags()) require.Equal(t, map[string]string{}, actual.Tags())
// Test that strings without TagKeys defined are ignored // Test that strings without TagKeys defined are ignored
metric, err = parser.ParseLine(validJSONTags) actual, err = parser.ParseLine(validJSONTags)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "json_test", metric.Name()) require.Equal(t, "json_test", actual.Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metric.Fields()) }, actual.Fields())
require.Equal(t, map[string]string{}, metric.Tags()) require.Equal(t, map[string]string{}, actual.Tags())
} }
func TestParseInvalidJSON(t *testing.T) { func TestParseInvalidJSON(t *testing.T) {
@ -214,15 +249,15 @@ func TestParseWithTagKeys(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(validJSONTags)) actual, err := parser.Parse([]byte(validJSONTags))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_test", metrics[0].Name()) require.Equal(t, "json_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{}, metrics[0].Tags()) require.Equal(t, map[string]string{}, actual[0].Tags())
// Test that single tag key is found and applied // Test that single tag key is found and applied
parser = &Parser{ parser = &Parser{
@ -231,17 +266,17 @@ func TestParseWithTagKeys(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err = parser.Parse([]byte(validJSONTags)) actual, err = parser.Parse([]byte(validJSONTags))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_test", metrics[0].Name()) require.Equal(t, "json_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{ require.Equal(t, map[string]string{
"mytag": "foobar", "mytag": "foobar",
}, metrics[0].Tags()) }, actual[0].Tags())
// Test that both tag keys are found and applied // Test that both tag keys are found and applied
parser = &Parser{ parser = &Parser{
@ -250,18 +285,18 @@ func TestParseWithTagKeys(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err = parser.Parse([]byte(validJSONTags)) actual, err = parser.Parse([]byte(validJSONTags))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_test", metrics[0].Name()) require.Equal(t, "json_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{ require.Equal(t, map[string]string{
"mytag": "foobar", "mytag": "foobar",
"othertag": "baz", "othertag": "baz",
}, metrics[0].Tags()) }, actual[0].Tags())
} }
func TestParseLineWithTagKeys(t *testing.T) { func TestParseLineWithTagKeys(t *testing.T) {
@ -272,14 +307,14 @@ func TestParseLineWithTagKeys(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metric, err := parser.ParseLine(validJSONTags) actual, err := parser.ParseLine(validJSONTags)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "json_test", metric.Name()) require.Equal(t, "json_test", actual.Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metric.Fields()) }, actual.Fields())
require.Equal(t, map[string]string{}, metric.Tags()) require.Equal(t, map[string]string{}, actual.Tags())
// Test that single tag key is found and applied // Test that single tag key is found and applied
parser = &Parser{ parser = &Parser{
@ -288,16 +323,16 @@ func TestParseLineWithTagKeys(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metric, err = parser.ParseLine(validJSONTags) actual, err = parser.ParseLine(validJSONTags)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "json_test", metric.Name()) require.Equal(t, "json_test", actual.Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metric.Fields()) }, actual.Fields())
require.Equal(t, map[string]string{ require.Equal(t, map[string]string{
"mytag": "foobar", "mytag": "foobar",
}, metric.Tags()) }, actual.Tags())
// Test that both tag keys are found and applied // Test that both tag keys are found and applied
parser = &Parser{ parser = &Parser{
@ -306,17 +341,17 @@ func TestParseLineWithTagKeys(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metric, err = parser.ParseLine(validJSONTags) actual, err = parser.ParseLine(validJSONTags)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "json_test", metric.Name()) require.Equal(t, "json_test", actual.Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metric.Fields()) }, actual.Fields())
require.Equal(t, map[string]string{ require.Equal(t, map[string]string{
"mytag": "foobar", "mytag": "foobar",
"othertag": "baz", "othertag": "baz",
}, metric.Tags()) }, actual.Tags())
} }
func TestParseValidJSONDefaultTags(t *testing.T) { func TestParseValidJSONDefaultTags(t *testing.T) {
@ -328,29 +363,29 @@ func TestParseValidJSONDefaultTags(t *testing.T) {
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
// Most basic vanilla test // Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSON)) actual, err := parser.Parse([]byte(validJSON))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_test", metrics[0].Name()) require.Equal(t, "json_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{"t4g": "default"}, metrics[0].Tags()) require.Equal(t, map[string]string{"t4g": "default"}, actual[0].Tags())
// Test that tagkeys and default tags are applied // Test that tagkeys and default tags are applied
metrics, err = parser.Parse([]byte(validJSONTags)) actual, err = parser.Parse([]byte(validJSONTags))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_test", metrics[0].Name()) require.Equal(t, "json_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{ require.Equal(t, map[string]string{
"t4g": "default", "t4g": "default",
"mytag": "foobar", "mytag": "foobar",
}, metrics[0].Tags()) }, actual[0].Tags())
} }
// Test that default tags are overridden by tag keys // Test that default tags are overridden by tag keys
@ -363,28 +398,28 @@ func TestParseValidJSONDefaultTagsOverride(t *testing.T) {
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
// Most basic vanilla test // Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSON)) actual, err := parser.Parse([]byte(validJSON))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_test", metrics[0].Name()) require.Equal(t, "json_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{"mytag": "default"}, metrics[0].Tags()) require.Equal(t, map[string]string{"mytag": "default"}, actual[0].Tags())
// Test that tagkeys override default tags // Test that tagkeys override default tags
metrics, err = parser.Parse([]byte(validJSONTags)) actual, err = parser.Parse([]byte(validJSONTags))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_test", metrics[0].Name()) require.Equal(t, "json_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{ require.Equal(t, map[string]string{
"mytag": "foobar", "mytag": "foobar",
}, metrics[0].Tags()) }, actual[0].Tags())
} }
// Test that json arrays can be parsed // Test that json arrays can be parsed
@ -393,32 +428,32 @@ func TestParseValidJSONArray(t *testing.T) {
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
// Most basic vanilla test // Most basic vanilla test
metrics, err := parser.Parse([]byte(validJSONArray)) actual, err := parser.Parse([]byte(validJSONArray))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.Equal(t, "json_array_test", metrics[0].Name()) require.Equal(t, "json_array_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{}, metrics[0].Tags()) require.Equal(t, map[string]string{}, actual[0].Tags())
// Basic multiple datapoints // Basic multiple datapoints
metrics, err = parser.Parse([]byte(validJSONArrayMultiple)) actual, err = parser.Parse([]byte(validJSONArrayMultiple))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 2) require.Len(t, actual, 2)
require.Equal(t, "json_array_test", metrics[0].Name()) require.Equal(t, "json_array_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{}, metrics[1].Tags()) require.Equal(t, map[string]string{}, actual[1].Tags())
require.Equal(t, "json_array_test", metrics[1].Name()) require.Equal(t, "json_array_test", actual[1].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(7), "a": float64(7),
"b_c": float64(8), "b_c": float64(8),
}, metrics[1].Fields()) }, actual[1].Fields())
require.Equal(t, map[string]string{}, metrics[1].Tags()) require.Equal(t, map[string]string{}, actual[1].Tags())
} }
func TestParseArrayWithTagKeys(t *testing.T) { func TestParseArrayWithTagKeys(t *testing.T) {
@ -429,22 +464,22 @@ func TestParseArrayWithTagKeys(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(validJSONArrayTags)) actual, err := parser.Parse([]byte(validJSONArrayTags))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 2) require.Len(t, actual, 2)
require.Equal(t, "json_array_test", metrics[0].Name()) require.Equal(t, "json_array_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{}, metrics[0].Tags()) require.Equal(t, map[string]string{}, actual[0].Tags())
require.Equal(t, "json_array_test", metrics[1].Name()) require.Equal(t, "json_array_test", actual[1].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(7), "a": float64(7),
"b_c": float64(8), "b_c": float64(8),
}, metrics[1].Fields()) }, actual[1].Fields())
require.Equal(t, map[string]string{}, metrics[1].Tags()) require.Equal(t, map[string]string{}, actual[1].Tags())
// Test that single tag key is found and applied // Test that single tag key is found and applied
parser = &Parser{ parser = &Parser{
@ -453,26 +488,26 @@ func TestParseArrayWithTagKeys(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err = parser.Parse([]byte(validJSONArrayTags)) actual, err = parser.Parse([]byte(validJSONArrayTags))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 2) require.Len(t, actual, 2)
require.Equal(t, "json_array_test", metrics[0].Name()) require.Equal(t, "json_array_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{ require.Equal(t, map[string]string{
"mytag": "foo", "mytag": "foo",
}, metrics[0].Tags()) }, actual[0].Tags())
require.Equal(t, "json_array_test", metrics[1].Name()) require.Equal(t, "json_array_test", actual[1].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(7), "a": float64(7),
"b_c": float64(8), "b_c": float64(8),
}, metrics[1].Fields()) }, actual[1].Fields())
require.Equal(t, map[string]string{ require.Equal(t, map[string]string{
"mytag": "bar", "mytag": "bar",
}, metrics[1].Tags()) }, actual[1].Tags())
// Test that both tag keys are found and applied // Test that both tag keys are found and applied
parser = &Parser{ parser = &Parser{
@ -481,28 +516,28 @@ func TestParseArrayWithTagKeys(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err = parser.Parse([]byte(validJSONArrayTags)) actual, err = parser.Parse([]byte(validJSONArrayTags))
require.NoError(t, err) require.NoError(t, err)
require.Len(t, metrics, 2) require.Len(t, actual, 2)
require.Equal(t, "json_array_test", metrics[0].Name()) require.Equal(t, "json_array_test", actual[0].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(5), "a": float64(5),
"b_c": float64(6), "b_c": float64(6),
}, metrics[0].Fields()) }, actual[0].Fields())
require.Equal(t, map[string]string{ require.Equal(t, map[string]string{
"mytag": "foo", "mytag": "foo",
"othertag": "baz", "othertag": "baz",
}, metrics[0].Tags()) }, actual[0].Tags())
require.Equal(t, "json_array_test", metrics[1].Name()) require.Equal(t, "json_array_test", actual[1].Name())
require.Equal(t, map[string]interface{}{ require.Equal(t, map[string]interface{}{
"a": float64(7), "a": float64(7),
"b_c": float64(8), "b_c": float64(8),
}, metrics[1].Fields()) }, actual[1].Fields())
require.Equal(t, map[string]string{ require.Equal(t, map[string]string{
"mytag": "bar", "mytag": "bar",
"othertag": "baz", "othertag": "baz",
}, metrics[1].Tags()) }, actual[1].Tags())
} }
var jsonBOM = []byte("\xef\xbb\xbf[{\"value\":17}]") var jsonBOM = []byte("\xef\xbb\xbf[{\"value\":17}]")
@ -542,10 +577,10 @@ func TestJSONParseNestedArray(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(testString)) actual, err := parser.Parse([]byte(testString))
require.Len(t, metrics, 1) require.Len(t, actual, 1)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 3, len(metrics[0].Tags())) require.Equal(t, 3, len(actual[0].Tags()))
} }
func TestJSONQueryErrorOnArray(t *testing.T) { func TestJSONQueryErrorOnArray(t *testing.T) {
@ -606,9 +641,9 @@ func TestArrayOfObjects(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(testString)) actual, err := parser.Parse([]byte(testString))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 3, len(metrics)) require.Equal(t, 3, len(actual))
} }
func TestUseCaseJSONQuery(t *testing.T) { func TestUseCaseJSONQuery(t *testing.T) {
@ -634,10 +669,10 @@ func TestUseCaseJSONQuery(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(testString)) actual, err := parser.Parse([]byte(testString))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 3, len(metrics)) require.Equal(t, 3, len(actual))
require.Equal(t, metrics[0].Fields()["last"], "Murphy") require.Equal(t, actual[0].Fields()["last"], "Murphy")
} }
func TestTimeParser(t *testing.T) { func TestTimeParser(t *testing.T) {
@ -669,10 +704,10 @@ func TestTimeParser(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(testString)) actual, err := parser.Parse([]byte(testString))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 2, len(metrics)) require.Equal(t, 2, len(actual))
require.NotEqual(t, metrics[0].Time(), metrics[1].Time()) require.NotEqual(t, actual[0].Time(), actual[1].Time())
} }
func TestTimeParserWithTimezone(t *testing.T) { func TestTimeParserWithTimezone(t *testing.T) {
@ -688,10 +723,10 @@ func TestTimeParserWithTimezone(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(testString)) actual, err := parser.Parse([]byte(testString))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 1, len(metrics)) require.Equal(t, 1, len(actual))
require.EqualValues(t, int64(1136405040000000000), metrics[0].Time().UnixNano()) require.EqualValues(t, int64(1136405040000000000), actual[0].Time().UnixNano())
} }
func TestUnixTimeParser(t *testing.T) { func TestUnixTimeParser(t *testing.T) {
@ -723,10 +758,10 @@ func TestUnixTimeParser(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(testString)) actual, err := parser.Parse([]byte(testString))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 2, len(metrics)) require.Equal(t, 2, len(actual))
require.NotEqual(t, metrics[0].Time(), metrics[1].Time()) require.NotEqual(t, actual[0].Time(), actual[1].Time())
} }
func TestUnixMsTimeParser(t *testing.T) { func TestUnixMsTimeParser(t *testing.T) {
@ -758,10 +793,10 @@ func TestUnixMsTimeParser(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(testString)) actual, err := parser.Parse([]byte(testString))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 2, len(metrics)) require.Equal(t, 2, len(actual))
require.NotEqual(t, metrics[0].Time(), metrics[1].Time()) require.NotEqual(t, actual[0].Time(), actual[1].Time())
} }
func TestTimeErrors(t *testing.T) { func TestTimeErrors(t *testing.T) {
@ -782,9 +817,9 @@ func TestTimeErrors(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(testString)) actual, err := parser.Parse([]byte(testString))
require.Error(t, err) require.Error(t, err)
require.Empty(t, metrics) require.Empty(t, actual)
testString2 := `{ testString2 := `{
"a": 5, "a": 5,
@ -802,9 +837,9 @@ func TestTimeErrors(t *testing.T) {
} }
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err = parser.Parse([]byte(testString2)) actual, err = parser.Parse([]byte(testString2))
require.Error(t, err) require.Error(t, err)
require.Empty(t, metrics) require.Empty(t, actual)
require.Equal(t, fmt.Errorf("JSON time key could not be found"), err) require.Equal(t, fmt.Errorf("JSON time key could not be found"), err)
} }
@ -812,10 +847,10 @@ func TestShareTimestamp(t *testing.T) {
parser := &Parser{MetricName: "json_test"} parser := &Parser{MetricName: "json_test"}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(validJSONArrayMultiple)) actual, err := parser.Parse([]byte(validJSONArrayMultiple))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 2, len(metrics)) require.Equal(t, 2, len(actual))
require.Equal(t, metrics[0].Time(), metrics[1].Time()) require.Equal(t, actual[0].Time(), actual[1].Time())
} }
func TestNameKey(t *testing.T) { func TestNameKey(t *testing.T) {
@ -832,9 +867,9 @@ func TestNameKey(t *testing.T) {
parser := &Parser{NameKey: "b_c"} parser := &Parser{NameKey: "b_c"}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
metrics, err := parser.Parse([]byte(testString)) actual, err := parser.Parse([]byte(testString))
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "this is my name", metrics[0].Name()) require.Equal(t, "this is my name", actual[0].Name())
} }
func TestParseArrayWithWrongType(t *testing.T) { func TestParseArrayWithWrongType(t *testing.T) {
@ -1359,6 +1394,85 @@ func TestParseArrayWithWildcardTagKeys(t *testing.T) {
} }
} }
func TestBenchmarkData(t *testing.T) {
// Setup the plugin
plugin := &Parser{
MetricName: "benchmark",
TagKeys: []string{"tags_*"},
}
require.NoError(t, plugin.Init())
expected := []telegraf.Metric{
metric.New(
"benchmark",
map[string]string{
"tags_flagname": "F5",
"tags_host": "1cbbb3796fc2",
"tags_key": "12345",
"tags_platform": "Java",
"tags_sdkver": "4.9.1",
"tags_value": "false",
},
map[string]interface{}{
"fields_count_sum": float64(5),
"timestamp": float64(1653643420),
},
time.Unix(0, 0),
),
metric.New(
"benchmark",
map[string]string{
"tags_flagname": "E42",
"tags_host": "klaus",
"tags_key": "67890",
"tags_platform": "Golang",
"tags_sdkver": "1.18.3",
"tags_value": "true",
},
map[string]interface{}{
"fields_count_sum": float64(42),
"timestamp": float64(1653646789),
},
time.Unix(0, 0),
),
}
// Do the parsing
actual, err := plugin.Parse([]byte(benchmarkData))
require.NoError(t, err)
testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime())
}
func BenchmarkParsingSequential(b *testing.B) {
// Configure the plugin
plugin := &Parser{
MetricName: "benchmark",
TagKeys: []string{"tags_*"},
}
require.NoError(b, plugin.Init())
// Do the benchmarking
for n := 0; n < b.N; n++ {
_, _ = plugin.Parse([]byte(benchmarkData))
}
}
func BenchmarkParsingParallel(b *testing.B) {
// Configure the plugin
plugin := &Parser{
MetricName: "benchmark",
TagKeys: []string{"tags_*"},
}
require.NoError(b, plugin.Init())
// Do the benchmarking
b.RunParallel(func(p *testing.PB) {
for p.Next() {
_, _ = plugin.Parse([]byte(benchmarkData))
}
})
}
func FuzzParserJSON(f *testing.F) { func FuzzParserJSON(f *testing.F) {
for _, value := range fuzz.JSONDictionary { for _, value := range fuzz.JSONDictionary {
f.Add([]byte(value)) f.Add([]byte(value))

View File

@ -15,6 +15,7 @@ import (
"github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/plugins/inputs/file" "github.com/influxdata/telegraf/plugins/inputs/file"
"github.com/influxdata/telegraf/plugins/parsers/influx" "github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/plugins/parsers/json_v2"
"github.com/influxdata/telegraf/testutil" "github.com/influxdata/telegraf/testutil"
) )
@ -34,6 +35,10 @@ func TestMultipleConfigs(t *testing.T) {
}) })
for _, f := range folders { for _, f := range folders {
// Only use directories as those contain test-cases
if !f.IsDir() {
continue
}
testdataPath := filepath.Join("testdata", f.Name()) testdataPath := filepath.Join("testdata", f.Name())
configFilename := filepath.Join(testdataPath, "telegraf.conf") configFilename := filepath.Join(testdataPath, "telegraf.conf")
expectedFilename := filepath.Join(testdataPath, "expected.out") expectedFilename := filepath.Join(testdataPath, "expected.out")
@ -94,3 +99,63 @@ func TestMultipleConfigs(t *testing.T) {
}) })
} }
} }
func BenchmarkParsingSequential(b *testing.B) {
inputFilename := filepath.Join("testdata", "benchmark", "input.json")
// Configure the plugin
plugin := &json_v2.Parser{
Configs: []json_v2.Config{
{
MeasurementName: "benchmark",
JSONObjects: []json_v2.Object{
{
Path: "metrics",
DisablePrependKeys: true,
},
},
},
},
}
require.NoError(b, plugin.Init())
// Read the input data
input, err := os.ReadFile(inputFilename)
require.NoError(b, err)
// Do the benchmarking
for n := 0; n < b.N; n++ {
_, _ = plugin.Parse(input)
}
}
func BenchmarkParsingParallel(b *testing.B) {
inputFilename := filepath.Join("testdata", "benchmark", "input.json")
// Configure the plugin
plugin := &json_v2.Parser{
Configs: []json_v2.Config{
{
MeasurementName: "benchmark",
JSONObjects: []json_v2.Object{
{
Path: "metrics",
DisablePrependKeys: true,
},
},
},
},
}
require.NoError(b, plugin.Init())
// Read the input data
input, err := os.ReadFile(inputFilename)
require.NoError(b, err)
// Do the benchmarking
b.RunParallel(func(p *testing.PB) {
for p.Next() {
_, _ = plugin.Parse(input)
}
})
}

View File

@ -0,0 +1,34 @@
{
"metrics": [
{
"name": "impression",
"timestamp": 1653643420,
"fields": {
"count_sum": 5
},
"tags": {
"key": "12345",
"flagname": "F5",
"host": "1cbbb3796fc2",
"platform": "Java",
"sdkver": "4.9.1",
"value": "false"
}
},
{
"name": "expression",
"timestamp": 1653646789,
"fields": {
"count_sum": 42
},
"tags": {
"key": "67890",
"flagname": "E42",
"host": "klaus",
"platform": "Golang",
"sdkver": "1.18.3",
"value": "true"
}
}
]
}

View File

@ -0,0 +1,2 @@
benchmark count_sum=5,flagname="F5",host="1cbbb3796fc2",key="12345",name="impression",platform="Java",sdkver="4.9.1",timestamp=1653643420,value="false" 1697730835574029794
benchmark count_sum=42,flagname="E42",host="klaus",key="67890",name="expression",platform="Golang",sdkver="1.18.3",timestamp=1653646789,value="true" 1697730835574029794

View File

@ -0,0 +1,34 @@
{
"metrics": [
{
"name": "impression",
"timestamp": 1653643420,
"fields": {
"count_sum": 5
},
"tags": {
"key": "12345",
"flagname": "F5",
"host": "1cbbb3796fc2",
"platform": "Java",
"sdkver": "4.9.1",
"value": "false"
}
},
{
"name": "expression",
"timestamp": 1653646789,
"fields": {
"count_sum": 42
},
"tags": {
"key": "67890",
"flagname": "E42",
"host": "klaus",
"platform": "Golang",
"sdkver": "1.18.3",
"value": "true"
}
}
]
}

View File

@ -0,0 +1,9 @@
[[inputs.file]]
files = ["./testdata/benchmark/input.json"]
data_format = "json_v2"
[[inputs.file.json_v2]]
measurement_name = "benchmark"
[[inputs.file.json_v2.object]]
path = 'metrics'
disable_prepend_keys = true