From 64f6c97d130bf6523f3ca40f6c5246098df0621a Mon Sep 17 00:00:00 2001 From: Sven Rebhan <36194019+srebhan@users.noreply.github.com> Date: Wed, 26 Jul 2023 15:11:25 +0200 Subject: [PATCH] fix(parsers.xpath): Improve handling of complex-type nodes (#13660) --- go.mod | 2 +- go.sum | 4 ++-- plugins/parsers/xpath/parser.go | 19 +++++++++++++++---- .../json_native_nonnested/expected.out | 1 + .../json_native_nonnested/telegraf.conf | 11 +++++++++++ .../testcases/json_native_nonnested/test.json | 13 +++++++++++++ .../json_string_representation/expected.out | 1 + .../json_string_representation/telegraf.conf | 9 +++++++++ .../json_string_representation/test.json | 13 +++++++++++++ .../testcases/name_expansion/telegraf.conf | 2 +- 10 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 plugins/parsers/xpath/testcases/json_native_nonnested/expected.out create mode 100644 plugins/parsers/xpath/testcases/json_native_nonnested/telegraf.conf create mode 100644 plugins/parsers/xpath/testcases/json_native_nonnested/test.json create mode 100644 plugins/parsers/xpath/testcases/json_string_representation/expected.out create mode 100644 plugins/parsers/xpath/testcases/json_string_representation/telegraf.conf create mode 100644 plugins/parsers/xpath/testcases/json_string_representation/test.json diff --git a/go.mod b/go.mod index e505b2e80..6953ae525 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 github.com/aliyun/alibaba-cloud-sdk-go v1.62.470 github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 - github.com/antchfx/jsonquery v1.3.2 + github.com/antchfx/jsonquery v1.3.3 github.com/antchfx/xmlquery v1.3.17 github.com/antchfx/xpath v1.2.4 github.com/apache/arrow/go/v13 v13.0.0-20230630125530-5a06b2ec2a8e diff --git a/go.sum b/go.sum index 62bf31282..60cc084f2 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,8 @@ github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antchfx/jsonquery v1.3.2 h1:/BgHv1le9CCkqDe7t1x5BRlCg6DQmXTsztnMQFG5Hoc= -github.com/antchfx/jsonquery v1.3.2/go.mod h1:VsW9O/sNgHoUVvhoMEjR+opjIOjKOViNFTpAlxcI4Ws= +github.com/antchfx/jsonquery v1.3.3 h1:zjZpbnZhYng3uOAbIfdNq81A9mMEeuDJeYIpeKpZ4es= +github.com/antchfx/jsonquery v1.3.3/go.mod h1:1JG4DqRlRCHgVYDPY1ioYFAGSXGfWHzNgrbiGQHsWck= github.com/antchfx/xmlquery v1.3.17 h1:d0qWjPp/D+vtRw7ivCwT5ApH/3CkQU8JOeo3245PpTk= github.com/antchfx/xmlquery v1.3.17/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= github.com/antchfx/xpath v1.1.7/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= diff --git a/plugins/parsers/xpath/parser.go b/plugins/parsers/xpath/parser.go index 25e81d29e..cf5dea616 100644 --- a/plugins/parsers/xpath/parser.go +++ b/plugins/parsers/xpath/parser.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "errors" "fmt" + "reflect" "strconv" "strings" "time" @@ -448,11 +449,21 @@ func (p *Parser) parseQuery(starttime time.Time, doc, selected dataNode, config } } - if config.FieldsHexFilter != nil && config.FieldsHexFilter.Match(name) { - if b, ok := v.([]byte); ok { - v = hex.EncodeToString(b) + // Handle complex types which would be dropped otherwise for + // native type handling + if v != nil { + switch reflect.TypeOf(v).Kind() { + case reflect.Array, reflect.Slice, reflect.Map: + if b, ok := v.([]byte); ok { + if config.FieldsHexFilter != nil && config.FieldsHexFilter.Match(name) { + v = hex.EncodeToString(b) + } + } else { + v = fmt.Sprintf("%v", v) + } } } + fields[name] = v } } else { @@ -500,7 +511,7 @@ func (p *Parser) executeQuery(doc, selected dataNode, query string) (r interface return nn.GetValue(), nil } } - // Fallback to get the string value representation + return iter.Current().Value(), nil } diff --git a/plugins/parsers/xpath/testcases/json_native_nonnested/expected.out b/plugins/parsers/xpath/testcases/json_native_nonnested/expected.out new file mode 100644 index 000000000..e626a6aea --- /dev/null +++ b/plugins/parsers/xpath/testcases/json_native_nonnested/expected.out @@ -0,0 +1 @@ +foo a="a string",b=3.1415,c=true,d="map[d1:1 d2:foo d3:true d4:]",e="[master 42 true]",timestamp=1690193829 1690193829000000000 \ No newline at end of file diff --git a/plugins/parsers/xpath/testcases/json_native_nonnested/telegraf.conf b/plugins/parsers/xpath/testcases/json_native_nonnested/telegraf.conf new file mode 100644 index 000000000..a5df3a269 --- /dev/null +++ b/plugins/parsers/xpath/testcases/json_native_nonnested/telegraf.conf @@ -0,0 +1,11 @@ +[[inputs.file]] + files = ["./testcases/json_string_representation/test.json"] + data_format = "xpath_json" + + xpath_native_types = true + + [[inputs.file.xpath]] + metric_name = "'foo'" + field_selection = "*" + timestamp = "timestamp" + timestamp_format = "unix" diff --git a/plugins/parsers/xpath/testcases/json_native_nonnested/test.json b/plugins/parsers/xpath/testcases/json_native_nonnested/test.json new file mode 100644 index 000000000..976ee9492 --- /dev/null +++ b/plugins/parsers/xpath/testcases/json_native_nonnested/test.json @@ -0,0 +1,13 @@ +{ + "a": "a string", + "b": 3.1415, + "c": true, + "d": { + "d1": 1, + "d2": "foo", + "d3": true, + "d4": null + }, + "e": ["master", 42, true], + "timestamp": 1690193829 + } \ No newline at end of file diff --git a/plugins/parsers/xpath/testcases/json_string_representation/expected.out b/plugins/parsers/xpath/testcases/json_string_representation/expected.out new file mode 100644 index 000000000..3bf13e46d --- /dev/null +++ b/plugins/parsers/xpath/testcases/json_string_representation/expected.out @@ -0,0 +1 @@ +foo a="a string",b="3.1415",c="true",d="{\"d1\":1,\"d2\":\"foo\",\"d3\":true,\"d4\":null}",e="[\"master\",42,true]",timestamp="1690193829" 1690193829000000000 \ No newline at end of file diff --git a/plugins/parsers/xpath/testcases/json_string_representation/telegraf.conf b/plugins/parsers/xpath/testcases/json_string_representation/telegraf.conf new file mode 100644 index 000000000..d7c0b47cf --- /dev/null +++ b/plugins/parsers/xpath/testcases/json_string_representation/telegraf.conf @@ -0,0 +1,9 @@ +[[inputs.file]] + files = ["./testcases/json_string_representation/test.json"] + data_format = "xpath_json" + + [[inputs.file.xpath]] + metric_name = "'foo'" + field_selection = "*" + timestamp = "timestamp" + timestamp_format = "unix" diff --git a/plugins/parsers/xpath/testcases/json_string_representation/test.json b/plugins/parsers/xpath/testcases/json_string_representation/test.json new file mode 100644 index 000000000..976ee9492 --- /dev/null +++ b/plugins/parsers/xpath/testcases/json_string_representation/test.json @@ -0,0 +1,13 @@ +{ + "a": "a string", + "b": 3.1415, + "c": true, + "d": { + "d1": 1, + "d2": "foo", + "d3": true, + "d4": null + }, + "e": ["master", 42, true], + "timestamp": 1690193829 + } \ No newline at end of file diff --git a/plugins/parsers/xpath/testcases/name_expansion/telegraf.conf b/plugins/parsers/xpath/testcases/name_expansion/telegraf.conf index 060720a34..a96d6ae5c 100644 --- a/plugins/parsers/xpath/testcases/name_expansion/telegraf.conf +++ b/plugins/parsers/xpath/testcases/name_expansion/telegraf.conf @@ -6,5 +6,5 @@ [[inputs.file.xpath]] metric_name = "'devices'" metric_selection = "/devices/*" - field_selection = "descendant::*" + field_selection = "descendant::*[not(*)]" field_name_expansion = true