CL-Softwares
/
telegraf
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(parsers.xpath): Add support for returning underlying data-types (#11558)

This commit is contained in:
Sven Rebhan 2022-07-28 14:18:55 +02:00 committed by GitHub
parent 13b0ed0e6e
commit ff17ede62d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 200 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
go.mod
View File

@ -22,7 +22,7 @@ require (
github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9
github.com/antchfx/jsonquery v1.2.0
github.com/antchfx/jsonquery v1.3.0
github.com/antchfx/xmlquery v1.3.9
github.com/antchfx/xpath v1.2.1
github.com/apache/thrift v0.15.0
@ -53,7 +53,7 @@ require (
github.com/djherbis/times v1.5.0
github.com/docker/docker v20.10.17+incompatible
github.com/docker/go-connections v0.4.0
github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60
github.com/doclambda/protobufquery v0.0.0-20220727165953-0da287796ee9
github.com/dynatrace-oss/dynatrace-metric-utils-go v0.5.0
github.com/eclipse/paho.golang v0.10.0
github.com/eclipse/paho.mqtt.golang v1.3.5

8
go.sum
View File

@ -263,8 +263,8 @@ github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RD
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/antchfx/jsonquery v1.2.0 h1:P7hhEM/+YQA5FPbBk+11QbmAtraSIu4d3FXXLtRzmho=
github.com/antchfx/jsonquery v1.2.0/go.mod h1:fZ88NWso7HlXESJ2hrNKnYx+xyT6pmvV1N6KMIg7FHo=
github.com/antchfx/jsonquery v1.3.0 h1:rftVBKEXpj8C9WVu+4mbqL5hd6nLz7/AbIvAQlq3D7o=
github.com/antchfx/jsonquery v1.3.0/go.mod h1:fZ88NWso7HlXESJ2hrNKnYx+xyT6pmvV1N6KMIg7FHo=
github.com/antchfx/xmlquery v1.3.9 h1:Y+zyMdiUZ4fasTQTkDb3DflOXP7+obcYEh80SISBmnQ=
github.com/antchfx/xmlquery v1.3.9/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks=
github.com/antchfx/xpath v1.1.7/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
@ -683,8 +683,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60 h1:27379cxrsKlr7hAnW/xrusefspUPjqHVRW1K/bZgfGw=
github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60/go.mod h1:8Ia4zp86glrUhC29AAdK9hwTYh8RB6v0WRCtpplYqEg=
github.com/doclambda/protobufquery v0.0.0-20220727165953-0da287796ee9 h1:677nbAF3nq56BEZ2R/VMl0wROQqJo4vJ/ZWuzm+vsUU=
github.com/doclambda/protobufquery v0.0.0-20220727165953-0da287796ee9/go.mod h1:8Ia4zp86glrUhC29AAdK9hwTYh8RB6v0WRCtpplYqEg=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dropbox/godropbox v0.0.0-20180512210157-31879d3884b9 h1:NAvZb7gqQfLSNBPzVsvI7eZMosXtg2g2kxXrei90CtU=
github.com/dropbox/godropbox v0.0.0-20180512210157-31879d3884b9/go.mod h1:glr97hP/JuXb+WMYCizc4PIFuzw1lCR97mwbe1VVXhQ=

8
plugins/parsers/xpath/README.md
View File

@ -102,6 +102,10 @@ You should use the following setting
## Useful when not all selected files have the exact same structure.
# xpath_allow_empty_selection = false
## Get native data-types for all data-format that contain type information.
## Currently, protobuf, msgpack and JSON support native data-types
# xpath_native_types = false
## Multiple parsing sections are allowed
[[inputs.file.xpath]]
## Optional: XPath-query to select a subset of nodes from the XML document.
@ -180,6 +184,10 @@ in the metric.
## Useful when not all selected files have the exact same structure.
# xpath_allow_empty_selection = false
## Get native data-types for all data-format that contain type information.
## Currently, protobuf, msgpack and JSON support native data-types
# xpath_native_types = false
## Multiple parsing sections are allowed
[[inputs.file.xpath]]
## Optional: XPath-query to select a subset of nodes from the XML document.

31
plugins/parsers/xpath/parser.go
View File

@ -7,7 +7,9 @@ import (
"strings"
"time"
"github.com/antchfx/jsonquery"
path "github.com/antchfx/xpath"
"github.com/doclambda/protobufquery"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
@ -34,6 +36,7 @@ type Parser struct {
ProtobufImportPaths []string `toml:"xpath_protobuf_import_paths"`
PrintDocument bool `toml:"xpath_print_document"`
AllowEmptySelection bool `toml:"xpath_allow_empty_selection"`
NativeTypes bool `toml:"xpath_native_types"`
Configs []xpath.Config `toml:"xpath"`
DefaultMetricName string `toml:"-"`
DefaultTags map[string]string `toml:"-"`
@ -451,17 +454,29 @@ func (p *Parser) executeQuery(doc, selected dataNode, query string) (r interface
// separately. Those iterators will be returned for queries directly
// referencing a node (value or attribute).
n := expr.Evaluate(p.document.CreateXPathNavigator(root))
if iter, ok := n.(*path.NodeIterator); ok {
// We got an iterator, so take the first match and get the referenced
// property. This will always be a string.
if iter.MoveNext() {
r = iter.Current().Value()
iter, ok := n.(*path.NodeIterator)
if !ok {
return n, nil
}
// We got an iterator, so take the first match and get the referenced
// property. This will always be a string.
if iter.MoveNext() {
current := iter.Current()
// If the dataformat supports native types and if support is
// enabled, we should return the native type of the data
if p.NativeTypes {
switch nn := current.(type) {
case *jsonquery.NodeNavigator:
return nn.GetValue(), nil
case *protobufquery.NodeNavigator:
return nn.GetValue(), nil
}
}
} else {
r = n
// Fallback to get the string value representation
return iter.Current().Value(), nil
}
return r, nil
return nil, nil
}
func splitLastPathElement(query string) []string {

81
plugins/parsers/xpath/parser_test.go
View File

@ -1,6 +1,9 @@
package xpath
import (
"bufio"
"errors"
"fmt"
"math"
"os"
"path/filepath"
@ -9,6 +12,9 @@ import (
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/plugins/inputs/file"
"github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/plugins/parsers/temporary/xpath"
"github.com/influxdata/telegraf/testutil"
@ -1388,6 +1394,81 @@ func TestProtobufImporting(t *testing.T) {
require.NoError(t, parser.Init())
}
func TestMultipleConfigs(t *testing.T) {
// Get all directories in testdata
folders, err := os.ReadDir("testcases")
require.NoError(t, err)
// Make sure the folder contains data
require.NotEmpty(t, folders)
for _, f := range folders {
if !f.IsDir() {
continue
}
t.Run(f.Name(), func(t *testing.T) {
configFilename := filepath.Join("testcases", f.Name(), "telegraf.conf")
expectedFilename := filepath.Join("testcases", f.Name(), "expected.out")
// Process the telegraf config file for the test
buf, err := os.ReadFile(configFilename)
if err != nil && errors.Is(err, os.ErrNotExist) {
return
}
require.NoError(t, err)
inputs.Add("file", func() telegraf.Input {
return &file.File{}
})
cfg := config.NewConfig()
require.NoError(t, cfg.LoadConfigData(buf))
// Gather the metrics from the input file configure
acc := testutil.Accumulator{}
for _, input := range cfg.Inputs {
require.NoError(t, input.Init())
require.NoError(t, input.Gather(&acc))
}
// Process expected metrics and compare with resulting metrics
expected, err := readMetricFile(expectedFilename)
require.NoError(t, err)
actual := acc.GetTelegrafMetrics()
testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime())
})
}
}
func readMetricFile(filename string) ([]telegraf.Metric, error) {
var metrics []telegraf.Metric
f, err := os.Open(filename)
if err != nil {
return metrics, err
}
defer f.Close()
parser := &influx.Parser{}
if err := parser.Init(); err != nil {
return nil, err
}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if line != "" {
m, err := parser.ParseLine(line)
if err != nil {
return nil, fmt.Errorf("unable to parse metric in %q failed: %v", line, err)
}
// The timezone needs to be UTC to match the timestamp test results
m.SetTime(m.Time().UTC())
metrics = append(metrics, m)
}
}
return metrics, nil
}
func loadTestConfiguration(filename string) (*xpath.Config, []string, error) {
buf, err := os.ReadFile(filename)
if err != nil {

1
plugins/parsers/xpath/testcases/native_types_json/expected.out Normal file
View File

@ -0,0 +1 @@
native_types value_a="a string",value_b=3.1415,value_c=42.0,value_d=true

13
plugins/parsers/xpath/testcases/native_types_json/telegraf.conf Normal file
View File

@ -0,0 +1,13 @@
[[inputs.file]]
files = ["./testcases/native_types_json/test.json"]
data_format = "xpath_json"
xpath_native_types = true
[[inputs.file.xpath]]
metric_name = "'native_types'"
[inputs.file.xpath.fields]
value_a = "//a"
value_b = "//b"
value_c = "//c"
value_d = "//d"

6
plugins/parsers/xpath/testcases/native_types_json/test.json Normal file
View File

@ -0,0 +1,6 @@
{
"a": "a string",
"b": 3.1415,
"c": 42,
"d": true
}

1
plugins/parsers/xpath/testcases/native_types_msgpack/expected.out Normal file
View File

@ -0,0 +1 @@
native_types value_a="a string",value_b=3.1415,value_c=42.0,value_d=true

1
plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/expected.out Normal file
View File

@ -0,0 +1 @@
native_types value_a="a string",value_b=3.1415,value_c=true

12
plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/telegraf.conf Normal file
View File

@ -0,0 +1,12 @@
[[inputs.file]]
files = ["./testcases/native_types_json/test.json"]
data_format = "xpath_json"
xpath_native_types = true
[[inputs.file.xpath]]
metric_name = "'native_types'"
[inputs.file.xpath.fields]
value_a = "//a"
value_b = "//b"
value_c = "//c"

5
plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/test.json Normal file
View File

@ -0,0 +1,5 @@
{
"a": "a string",
"b": 3.1415,
"c": true
}

13
plugins/parsers/xpath/testcases/native_types_msgpack/telegraf.conf Normal file
View File

@ -0,0 +1,13 @@
[[inputs.file]]
files = ["./testcases/native_types_msgpack/test.msg"]
data_format = "xpath_msgpack"
xpath_native_types = true
[[inputs.file.xpath]]
metric_name = "'native_types'"
[inputs.file.xpath.fields]
value_a = "//a"
value_b = "//b"
value_c = "//c"
value_d = "//d"

BIN
plugins/parsers/xpath/testcases/native_types_msgpack/test.msg Normal file
View File

Binary file not shown.

1
plugins/parsers/xpath/testcases/native_types_protobuf/expected.out Normal file
View File

@ -0,0 +1 @@
native_types value_a="a string",value_b=3.1415,value_c=42i,value_d=true

10
plugins/parsers/xpath/testcases/native_types_protobuf/message.proto Normal file
View File

@ -0,0 +1,10 @@
syntax = "proto3";
package native_type;
message Message {
string a = 1;
double b = 2;
int32 c = 3;
bool d = 4;
}

17
plugins/parsers/xpath/testcases/native_types_protobuf/telegraf.conf Normal file
View File

@ -0,0 +1,17 @@
[[inputs.file]]
files = ["./testcases/native_types_protobuf/test.dat"]
data_format = "xpath_protobuf"
xpath_native_types = true
xpath_protobuf_file = "message.proto"
xpath_protobuf_type = "native_type.Message"
xpath_protobuf_import_paths = [".", "./testcases/native_types_protobuf"]
[[inputs.file.xpath]]
metric_name = "'native_types'"
[inputs.file.xpath.fields]
value_a = "//a"
value_b = "//b"
value_c = "//c"
value_d = "//d"

2
plugins/parsers/xpath/testcases/native_types_protobuf/test.dat Normal file
View File

@ -0,0 +1,2 @@
a stringoƒÀÊ! @*