Add XML parser using XPath queries (#8931)
This commit is contained in:
reimda
GitHub
parent
17efd172b7
commit
431d06acc0
|
|
@ -1268,7 +1268,14 @@ func (c *Config) buildParser(name string, tbl *ast.Table) (parsers.Parser, error
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parsers.NewParser(config)
|
||||
parser, err := parsers.NewParser(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logger := models.NewLogger("parsers", config.DataFormat, name)
|
||||
models.SetLoggerOnPlugin(parser, logger)
|
||||
|
||||
return parser, nil
|
||||
}
|
||||
|
||||
func (c *Config) getParserConfig(name string, tbl *ast.Table) (*parsers.Config, error) {
|
||||
|
|
@ -1335,6 +1342,28 @@ func (c *Config) getParserConfig(name string, tbl *ast.Table) (*parsers.Config,
|
|||
|
||||
c.getFieldStringSlice(tbl, "form_urlencoded_tag_keys", &pc.FormUrlencodedTagKeys)
|
||||
|
||||
//for XML parser
|
||||
if node, ok := tbl.Fields["xml"]; ok {
|
||||
if subtbls, ok := node.([]*ast.Table); ok {
|
||||
pc.XMLConfig = make([]parsers.XMLConfig, len(subtbls))
|
||||
for i, subtbl := range subtbls {
|
||||
subcfg := pc.XMLConfig[i]
|
||||
c.getFieldString(subtbl, "metric_name", &subcfg.MetricQuery)
|
||||
c.getFieldString(subtbl, "metric_selection", &subcfg.Selection)
|
||||
c.getFieldString(subtbl, "timestamp", &subcfg.Timestamp)
|
||||
c.getFieldString(subtbl, "timestamp_format", &subcfg.TimestampFmt)
|
||||
c.getFieldStringMap(subtbl, "tags", &subcfg.Tags)
|
||||
c.getFieldStringMap(subtbl, "fields", &subcfg.Fields)
|
||||
c.getFieldStringMap(subtbl, "fields_int", &subcfg.FieldsInt)
|
||||
c.getFieldString(subtbl, "field_selection", &subcfg.FieldSelection)
|
||||
c.getFieldBool(subtbl, "field_name_expansion", &subcfg.FieldNameExpand)
|
||||
c.getFieldString(subtbl, "field_name", &subcfg.FieldNameQuery)
|
||||
c.getFieldString(subtbl, "field_value", &subcfg.FieldValueQuery)
|
||||
pc.XMLConfig[i] = subcfg
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pc.MetricName = name
|
||||
|
||||
if c.hasErrs() {
|
||||
|
|
@ -1439,7 +1468,7 @@ func (c *Config) missingTomlField(typ reflect.Type, key string) error {
|
|||
"prefix", "prometheus_export_timestamp", "prometheus_sort_metrics", "prometheus_string_as_label",
|
||||
"separator", "splunkmetric_hec_routing", "splunkmetric_multimetric", "tag_keys",
|
||||
"tagdrop", "tagexclude", "taginclude", "tagpass", "tags", "template", "templates",
|
||||
"wavefront_source_override", "wavefront_use_strict":
|
||||
"wavefront_source_override", "wavefront_use_strict", "xml":
|
||||
|
||||
// ignore fields that are common to all plugins.
|
||||
default:
|
||||
|
|
@ -1545,6 +1574,7 @@ func (c *Config) getFieldStringSlice(tbl *ast.Table, fieldName string, target *[
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) getFieldTagFilter(tbl *ast.Table, fieldName string, target *[]models.TagFilter) {
|
||||
if node, ok := tbl.Fields[fieldName]; ok {
|
||||
if subtbl, ok := node.(*ast.Table); ok {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ Protocol or in JSON format.
|
|||
- [Prometheus](/plugins/parsers/prometheus)
|
||||
- [Value](/plugins/parsers/value), ie: 45 or "booyah"
|
||||
- [Wavefront](/plugins/parsers/wavefront)
|
||||
- [XML](/plugins/parsers/xml)
|
||||
|
||||
Any input plugin containing the `data_format` option can use it to select the
|
||||
desired parser:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ following works:
|
|||
- github.com/aerospike/aerospike-client-go [Apache License 2.0](https://github.com/aerospike/aerospike-client-go/blob/master/LICENSE)
|
||||
- github.com/alecthomas/units [MIT License](https://github.com/alecthomas/units/blob/master/COPYING)
|
||||
- github.com/amir/raidman [The Unlicense](https://github.com/amir/raidman/blob/master/UNLICENSE)
|
||||
- github.com/antchfx/xmlquery [MIT License](https://github.com/antchfx/xmlquery/blob/master/LICENSE)
|
||||
- github.com/antchfx/xpath [MIT License](https://github.com/antchfx/xpath/blob/master/LICENSE)
|
||||
- github.com/apache/thrift [Apache License 2.0](https://github.com/apache/thrift/blob/master/LICENSE)
|
||||
- github.com/aristanetworks/glog [Apache License 2.0](https://github.com/aristanetworks/glog/blob/master/LICENSE)
|
||||
- github.com/aristanetworks/goarista [Apache License 2.0](https://github.com/aristanetworks/goarista/blob/master/COPYING)
|
||||
|
|
|
|||
10
go.mod
10
go.mod
|
|
@ -21,6 +21,8 @@ require (
|
|||
github.com/aerospike/aerospike-client-go v1.27.0
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4
|
||||
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9
|
||||
github.com/antchfx/xmlquery v1.3.3
|
||||
github.com/antchfx/xpath v1.1.11
|
||||
github.com/apache/thrift v0.12.0
|
||||
github.com/aristanetworks/glog v0.0.0-20191112221043-67e8567f59f3 // indirect
|
||||
github.com/aristanetworks/goarista v0.0.0-20190325233358-a123909ec740
|
||||
|
|
@ -118,7 +120,7 @@ require (
|
|||
github.com/safchain/ethtool v0.0.0-20200218184317-f459e2d13664
|
||||
github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect
|
||||
github.com/sensu/sensu-go/api/core/v2 v2.6.0
|
||||
github.com/shirou/gopsutil v2.20.9+incompatible
|
||||
github.com/shirou/gopsutil v3.20.11+incompatible
|
||||
github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 // indirect
|
||||
github.com/signalfx/golib/v3 v3.3.0
|
||||
github.com/sirupsen/logrus v1.6.0
|
||||
|
|
@ -141,11 +143,11 @@ require (
|
|||
go.starlark.net v0.0.0-20200901195727-6e684ef5eeee
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 // indirect
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974
|
||||
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f
|
||||
golang.org/x/text v0.3.3
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68
|
||||
golang.org/x/text v0.3.4
|
||||
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200205215550-e35592f146e4
|
||||
google.golang.org/api v0.20.0
|
||||
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884
|
||||
|
|
|
|||
34
go.sum
34
go.sum
|
|
@ -101,6 +101,12 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1C
|
|||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ=
|
||||
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc=
|
||||
github.com/antchfx/xmlquery v1.3.3 h1:HYmadPG0uz8CySdL68rB4DCLKXz2PurCjS3mnkVF4CQ=
|
||||
github.com/antchfx/xmlquery v1.3.3/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc=
|
||||
github.com/antchfx/xpath v1.1.10 h1:cJ0pOvEdN/WvYXxvRrzQH9x5QWKpzHacYO8qzCcDYAg=
|
||||
github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||
github.com/antchfx/xpath v1.1.11 h1:WOFtK8TVAjLm3lbgqeP0arlHpvCEeTANeWZ/csPpJkQ=
|
||||
github.com/antchfx/xpath v1.1.11/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
|
|
@ -406,10 +412,10 @@ github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 h1:vr3AYkKovP8uR8AvSGGU
|
|||
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ=
|
||||
github.com/jackc/pgx v3.6.0+incompatible h1:bJeo4JdVbDAW8KB2m8XkFeo8CPipREoG37BwEoKGz+Q=
|
||||
github.com/jackc/pgx v3.6.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
|
||||
github.com/james4k/rcon v0.0.0-20120923215419-8fbb8268b60a h1:JxcWget6X/VfBMKxPIc28Jel37LGREut2fpV+ObkwJ0=
|
||||
github.com/james4k/rcon v0.0.0-20120923215419-8fbb8268b60a/go.mod h1:1qNVsDcmNQDsAXYfUuF/Z0rtK5eT8x9D6Pi7S3PjXAg=
|
||||
github.com/jaegertracing/jaeger v1.15.1 h1:7QzNAXq+4ko9GtCjozDNAp2uonoABu+B2Rk94hjQcp4=
|
||||
github.com/jaegertracing/jaeger v1.15.1/go.mod h1:LUWPSnzNPGRubM8pk0inANGitpiMOOxihXx0+53llXI=
|
||||
github.com/james4k/rcon v0.0.0-20120923215419-8fbb8268b60a h1:JxcWget6X/VfBMKxPIc28Jel37LGREut2fpV+ObkwJ0=
|
||||
github.com/james4k/rcon v0.0.0-20120923215419-8fbb8268b60a/go.mod h1:1qNVsDcmNQDsAXYfUuF/Z0rtK5eT8x9D6Pi7S3PjXAg=
|
||||
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
|
||||
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
|
|
@ -617,8 +623,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
|
|||
github.com/sensu/sensu-go/api/core/v2 v2.6.0 h1:hEKPHFZZNDuWTlKr7Kgm2yog65ZdkBUqNesE5qaWEGo=
|
||||
github.com/sensu/sensu-go/api/core/v2 v2.6.0/go.mod h1:97IK4ZQuvVjWvvoLkp+NgrD6ot30WDRz3LEbFUc/N34=
|
||||
github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/gopsutil v2.20.9+incompatible h1:msXs2frUV+O/JLva9EDLpuJ84PrFsdCTCQex8PUdtkQ=
|
||||
github.com/shirou/gopsutil v2.20.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/gopsutil v3.20.11+incompatible h1:LJr4ZQK4mPpIV5gOa4jCOKOGb4ty4DZO54I4FGqIpto=
|
||||
github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
|
||||
github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 h1:Pm6R878vxWWWR+Sa3ppsLce/Zq+JNTs6aVvRu13jv9A=
|
||||
github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
|
|
@ -785,10 +791,21 @@ golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLL
|
|||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
|
||||
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA=
|
||||
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -834,12 +851,18 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
@ -847,6 +870,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
|
|||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
|
||||
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import (
|
|||
"github.com/influxdata/telegraf/plugins/parsers/prometheus"
|
||||
"github.com/influxdata/telegraf/plugins/parsers/value"
|
||||
"github.com/influxdata/telegraf/plugins/parsers/wavefront"
|
||||
"github.com/influxdata/telegraf/plugins/parsers/xml"
|
||||
)
|
||||
|
||||
type ParserFunc func() (Parser, error)
|
||||
|
|
@ -150,6 +151,13 @@ type Config struct {
|
|||
|
||||
// FormData configuration
|
||||
FormUrlencodedTagKeys []string `toml:"form_urlencoded_tag_keys"`
|
||||
|
||||
// XML configuration
|
||||
XMLConfig []XMLConfig `toml:"xml"`
|
||||
}
|
||||
|
||||
type XMLConfig struct {
|
||||
xml.Config
|
||||
}
|
||||
|
||||
// NewParser returns a Parser interface based on the given config.
|
||||
|
|
@ -237,6 +245,8 @@ func NewParser(config *Config) (Parser, error) {
|
|||
)
|
||||
case "prometheus":
|
||||
parser, err = NewPrometheusParser(config.DefaultTags)
|
||||
case "xml":
|
||||
parser, err = NewXMLParser(config.MetricName, config.DefaultTags, config.XMLConfig)
|
||||
default:
|
||||
err = fmt.Errorf("Invalid data format: %s", config.DataFormat)
|
||||
}
|
||||
|
|
@ -350,3 +360,29 @@ func NewPrometheusParser(defaultTags map[string]string) (Parser, error) {
|
|||
DefaultTags: defaultTags,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewXMLParser(metricName string, defaultTags map[string]string, xmlConfigs []XMLConfig) (Parser, error) {
|
||||
// Convert the config formats which is a one-to-one copy
|
||||
configs := make([]xml.Config, len(xmlConfigs))
|
||||
for i, cfg := range xmlConfigs {
|
||||
configs[i].MetricName = metricName
|
||||
configs[i].MetricQuery = cfg.MetricQuery
|
||||
configs[i].Selection = cfg.Selection
|
||||
configs[i].Timestamp = cfg.Timestamp
|
||||
configs[i].TimestampFmt = cfg.TimestampFmt
|
||||
configs[i].Tags = cfg.Tags
|
||||
configs[i].Fields = cfg.Fields
|
||||
configs[i].FieldsInt = cfg.FieldsInt
|
||||
|
||||
configs[i].FieldSelection = cfg.FieldSelection
|
||||
configs[i].FieldNameQuery = cfg.FieldNameQuery
|
||||
configs[i].FieldValueQuery = cfg.FieldValueQuery
|
||||
|
||||
configs[i].FieldNameExpand = cfg.FieldNameExpand
|
||||
}
|
||||
|
||||
return &xml.Parser{
|
||||
Configs: configs,
|
||||
DefaultTags: defaultTags,
|
||||
}, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,345 @@
|
|||
# XML
|
||||
|
||||
The XML data format parser parses a [XML][xml] string into metric fields using [XPath][xpath] expressions. For supported
|
||||
XPath functions check [the underlying XPath library][xpath lib].
|
||||
|
||||
**NOTE:** The type of fields are specified using [XPath functions][xpath lib]. The only exception are *integer* fields
|
||||
that need to be specified in a `fields_int` section.
|
||||
|
||||
### Configuration
|
||||
|
||||
```toml
|
||||
[[inputs.file]]
|
||||
files = ["example.xml"]
|
||||
|
||||
## Data format to consume.
|
||||
## Each data format has its own unique set of configuration options, read
|
||||
## more about them here:
|
||||
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
|
||||
data_format = "xml"
|
||||
|
||||
## Multiple parsing sections are allowed
|
||||
[[inputs.file.xml]]
|
||||
## Optional: XPath-query to select a subset of nodes from the XML document.
|
||||
#metric_selection = "/Bus/child::Sensor"
|
||||
|
||||
## Optional: XPath-query to set the metric (measurement) name.
|
||||
#metric_name = "string('example')"
|
||||
|
||||
## Optional: Query to extract metric timestamp.
|
||||
## If not specified the time of execution is used.
|
||||
#timestamp = "/Gateway/Timestamp"
|
||||
## Optional: Format of the timestamp determined by the query above.
|
||||
## This can be any of "unix", "unix_ms", "unix_us", "unix_ns" or a valid Golang
|
||||
## time format. If not specified, a "unix" timestamp (in seconds) is expected.
|
||||
#timestamp_format = "2006-01-02T15:04:05Z"
|
||||
|
||||
## Tag definitions using the given XPath queries.
|
||||
[inputs.file.xml.tags]
|
||||
name = "substring-after(Sensor/@name, ' ')"
|
||||
device = "string('the ultimate sensor')"
|
||||
|
||||
## Integer field definitions using XPath queries.
|
||||
[inputs.file.xml.fields_int]
|
||||
consumers = "Variable/@consumers"
|
||||
|
||||
## Non-integer field definitions using XPath queries.
|
||||
## The field type is defined using XPath expressions such as number(), boolean() or string(). If no conversion is performed the field will be of type string.
|
||||
[inputs.file.xml.fields]
|
||||
temperature = "number(Variable/@temperature)"
|
||||
power = "number(Variable/@power)"
|
||||
frequency = "number(Variable/@frequency)"
|
||||
ok = "Mode != 'ok'"
|
||||
```
|
||||
|
||||
A configuration can contain muliple *xml* subsections for e.g. the file plugin to process the xml-string multiple times.
|
||||
Consult the [XPath syntax][xpath] and the [underlying library's functions][xpath lib] for details and help regarding XPath queries.
|
||||
|
||||
Alternatively to the configuration above, fields can also be specified in a batch way. So contrary to specify the fields
|
||||
in a section, you can define a `name` and a `value` selector used to determine the name and value of the fields in the
|
||||
metric.
|
||||
```toml
|
||||
[[inputs.file]]
|
||||
files = ["example.xml"]
|
||||
|
||||
## Data format to consume.
|
||||
## Each data format has its own unique set of configuration options, read
|
||||
## more about them here:
|
||||
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
|
||||
data_format = "xml"
|
||||
|
||||
## Multiple parsing sections are allowed
|
||||
[[inputs.file.xml]]
|
||||
## Optional: XPath-query to select a subset of nodes from the XML document.
|
||||
metric_selection = "/Bus/child::Sensor"
|
||||
|
||||
## Optional: XPath-query to set the metric (measurement) name.
|
||||
#metric_name = "string('example')"
|
||||
|
||||
## Optional: Query to extract metric timestamp.
|
||||
## If not specified the time of execution is used.
|
||||
#timestamp = "/Gateway/Timestamp"
|
||||
## Optional: Format of the timestamp determined by the query above.
|
||||
## This can be any of "unix", "unix_ms", "unix_us", "unix_ns" or a valid Golang
|
||||
## time format. If not specified, a "unix" timestamp (in seconds) is expected.
|
||||
#timestamp_format = "2006-01-02T15:04:05Z"
|
||||
|
||||
## Field specifications using a selector.
|
||||
field_selection = "child::*"
|
||||
## Optional: Queries to specify field name and value.
|
||||
## These options are only to be used in combination with 'field_selection'!
|
||||
## By default the node name and node content is used if a field-selection
|
||||
## is specified.
|
||||
#field_name = "name()"
|
||||
#field_value = "."
|
||||
|
||||
## Optional: Expand field names relative to the selected node
|
||||
## This allows to flatten out nodes with non-unique names in the subtree
|
||||
#field_name_expansion = false
|
||||
|
||||
## Tag definitions using the given XPath queries.
|
||||
[inputs.file.xml.tags]
|
||||
name = "substring-after(Sensor/@name, ' ')"
|
||||
device = "string('the ultimate sensor')"
|
||||
|
||||
```
|
||||
*Please note*: The resulting fields are _always_ of type string!
|
||||
|
||||
It is also possible to specify a mixture of the two alternative ways of specifying fields.
|
||||
|
||||
#### metric_selection (optional)
|
||||
|
||||
You can specify a [XPath][xpath] query to select a subset of nodes from the XML document, each used to generate a new
|
||||
metrics with the specified fields, tags etc.
|
||||
|
||||
For relative queries in subsequent queries they are relative to the `metric_selection`. To specify absolute paths, please start the query with a slash (`/`).
|
||||
|
||||
Specifying `metric_selection` is optional. If not specified all relative queries are relative to the root node of the XML document.
|
||||
|
||||
#### metric_name (optional)
|
||||
|
||||
By specifying `metric_name` you can override the metric/measurement name with the result of the given [XPath][xpath] query. If not specified, the default metric name is used.
|
||||
|
||||
#### timestamp, timestamp_format (optional)
|
||||
|
||||
By default the current time will be used for all created metrics. To set the time from values in the XML document you can specify a [XPath][xpath] query in `timestamp` and set the format in `timestamp_format`.
|
||||
|
||||
The `timestamp_format` can be set to `unix`, `unix_ms`, `unix_us`, `unix_ns`, or
|
||||
an accepted [Go "reference time"][time const]. Consult the Go [time][time parse] package for details and additional examples on how to set the time format.
|
||||
If `timestamp_format` is omitted `unix` format is assumed as result of the `timestamp` query.
|
||||
|
||||
#### tags sub-section
|
||||
|
||||
[XPath][xpath] queries in the `tag name = query` format to add tags to the metrics. The specified path can be absolute (starting with `/`) or relative. Relative paths use the currently selected node as reference.
|
||||
|
||||
**NOTE:** Results of tag-queries will always be converted to strings.
|
||||
|
||||
#### fields_int sub-section
|
||||
|
||||
[XPath][xpath] queries in the `field name = query` format to add integer typed fields to the metrics. The specified path can be absolute (starting with `/`) or relative. Relative paths use the currently selected node as reference.
|
||||
|
||||
**NOTE:** Results of field_int-queries will always be converted to **int64**. The conversion will fail in case the query result is not convertible!
|
||||
|
||||
#### fields sub-section
|
||||
|
||||
[XPath][xpath] queries in the `field name = query` format to add non-integer fields to the metrics. The specified path can be absolute (starting with `/`) or relative. Relative paths use the currently selected node as reference.
|
||||
|
||||
The type of the field is specified in the [XPath][xpath] query using the type conversion functions of XPath such as `number()`, `boolean()` or `string()`
|
||||
If no conversion is performed in the query the field will be of type string.
|
||||
|
||||
**NOTE: Path conversion functions will always succeed even if you convert a text to float!**
|
||||
|
||||
|
||||
#### field_selection, field_name, field_value (optional)
|
||||
|
||||
You can specify a [XPath][xpath] query to select a set of nodes forming the fields of the metric. The specified path can be absolute (starting with `/`) or relative to the currently selected node. Each node selected by `field_selection` forms a new field within the metric.
|
||||
|
||||
The *name* and the *value* of each field can be specified using the optional `field_name` and `field_value` queries. The queries are relative to the selected field if not starting with `/`. If not specified the field's *name* defaults to the node name and the field's *value* defaults to the content of the selected field node.
|
||||
**NOTE**: `field_name` and `field_value` queries are only evaluated if a `field_selection` is specified.
|
||||
|
||||
Specifying `field_selection` is optional. This is an alternative way to specify fields especially for documents where the node names are not known a priori or if there is a large number of fields to be specified. These options can also be combined with the field specifications above.
|
||||
|
||||
**NOTE: Path conversion functions will always succeed even if you convert a text to float!**
|
||||
|
||||
#### field_name_expansion (optional)
|
||||
|
||||
When *true*, field names selected with `field_selection` are expanded to a *path* relative to the *selected node*. This
|
||||
is necessary if we e.g. select all leaf nodes as fields and those leaf nodes do not have unique names. That is in case
|
||||
you have duplicate names in the fields you select you should set this to `true`.
|
||||
|
||||
### Examples
|
||||
|
||||
This `example.xml` file is used in the configuration examples below:
|
||||
```xml
|
||||
<?xml version="1.0"?>
|
||||
<Gateway>
|
||||
<Name>Main Gateway</Name>
|
||||
<Timestamp>2020-08-01T15:04:03Z</Timestamp>
|
||||
<Sequence>12</Sequence>
|
||||
<Status>ok</Status>
|
||||
</Gateway>
|
||||
|
||||
<Bus>
|
||||
<Sensor name="Sensor Facility A">
|
||||
<Variable temperature="20.0"/>
|
||||
<Variable power="123.4"/>
|
||||
<Variable frequency="49.78"/>
|
||||
<Variable consumers="3"/>
|
||||
<Mode>busy</Mode>
|
||||
</Sensor>
|
||||
<Sensor name="Sensor Facility B">
|
||||
<Variable temperature="23.1"/>
|
||||
<Variable power="14.3"/>
|
||||
<Variable frequency="49.78"/>
|
||||
<Variable consumers="1"/>
|
||||
<Mode>standby</Mode>
|
||||
</Sensor>
|
||||
<Sensor name="Sensor Facility C">
|
||||
<Variable temperature="19.7"/>
|
||||
<Variable power="0.02"/>
|
||||
<Variable frequency="49.78"/>
|
||||
<Variable consumers="0"/>
|
||||
<Mode>error</Mode>
|
||||
</Sensor>
|
||||
</Bus>
|
||||
```
|
||||
|
||||
#### Basic Parsing
|
||||
|
||||
This example shows the basic usage of the xml parser.
|
||||
|
||||
Config:
|
||||
```toml
|
||||
[[inputs.file]]
|
||||
files = ["example.xml"]
|
||||
data_format = "xml"
|
||||
|
||||
[[inputs.file.xml]]
|
||||
[inputs.file.xml.tags]
|
||||
gateway = "substring-before(/Gateway/Name, ' ')"
|
||||
|
||||
[inputs.file.xml.fields_int]
|
||||
seqnr = "/Gateway/Sequence"
|
||||
|
||||
[inputs.file.xml.fields]
|
||||
ok = "/Gateway/Status = 'ok'"
|
||||
```
|
||||
|
||||
Output:
|
||||
```
|
||||
file,gateway=Main,host=Hugin seqnr=12i,ok=true 1598610830000000000
|
||||
```
|
||||
|
||||
In the *tags* definition the XPath function `substring-before()` is used to only extract the sub-string before the space. To get the integer value of `/Gateway/Sequence` we have to use the *fields_int* section as there is no XPath expression to convert node values to integers (only float).
|
||||
The `ok` field is filled with a boolean by specifying a query comparing the query result of `/Gateway/Status` with the string *ok*. Use the type conversions available in the XPath syntax to specify field types.
|
||||
|
||||
#### Time and metric names
|
||||
|
||||
This is an example for using time and name of the metric from the XML document itself.
|
||||
|
||||
Config:
|
||||
```toml
|
||||
[[inputs.file]]
|
||||
files = ["example.xml"]
|
||||
data_format = "xml"
|
||||
|
||||
[[inputs.file.xml]]
|
||||
metric_name = "name(/Gateway/Status)"
|
||||
|
||||
timestamp = "/Gateway/Timestamp"
|
||||
timestamp_format = "2006-01-02T15:04:05Z"
|
||||
|
||||
[inputs.file.xml.tags]
|
||||
gateway = "substring-before(/Gateway/Name, ' ')"
|
||||
|
||||
[inputs.file.xml.fields]
|
||||
ok = "/Gateway/Status = 'ok'"
|
||||
```
|
||||
|
||||
Output:
|
||||
```
|
||||
Status,gateway=Main,host=Hugin ok=true 1596294243000000000
|
||||
```
|
||||
Additionally to the basic parsing example, the metric name is defined as the name of the `/Gateway/Status` node and the timestamp is derived from the XML document instead of using the execution time.
|
||||
|
||||
#### Multi-node selection
|
||||
|
||||
For XML documents containing metrics for e.g. multiple devices (like `Sensor`s in the *example.xml*), multiple metrics can be generated using node selection. This example shows how to generate a metric for each *Sensor* in the example.
|
||||
|
||||
Config:
|
||||
```toml
|
||||
[[inputs.file]]
|
||||
files = ["example.xml"]
|
||||
data_format = "xml"
|
||||
|
||||
[[inputs.file.xml]]
|
||||
metric_selection = "/Bus/child::Sensor"
|
||||
|
||||
metric_name = "string('sensors')"
|
||||
|
||||
timestamp = "/Gateway/Timestamp"
|
||||
timestamp_format = "2006-01-02T15:04:05Z"
|
||||
|
||||
[inputs.file.xml.tags]
|
||||
name = "substring-after(@name, ' ')"
|
||||
|
||||
[inputs.file.xml.fields_int]
|
||||
consumers = "Variable/@consumers"
|
||||
|
||||
[inputs.file.xml.fields]
|
||||
temperature = "number(Variable/@temperature)"
|
||||
power = "number(Variable/@power)"
|
||||
frequency = "number(Variable/@frequency)"
|
||||
ok = "Mode != 'error'"
|
||||
|
||||
```
|
||||
|
||||
Output:
|
||||
```
|
||||
sensors,host=Hugin,name=Facility\ A consumers=3i,frequency=49.78,ok=true,power=123.4,temperature=20 1596294243000000000
|
||||
sensors,host=Hugin,name=Facility\ B consumers=1i,frequency=49.78,ok=true,power=14.3,temperature=23.1 1596294243000000000
|
||||
sensors,host=Hugin,name=Facility\ C consumers=0i,frequency=49.78,ok=false,power=0.02,temperature=19.7 1596294243000000000
|
||||
```
|
||||
|
||||
Using the `metric_selection` option we select all `Sensor` nodes in the XML document. Please note that all field and tag definitions are relative to these selected nodes. An exception is the timestamp definition which is relative to the root node of the XML document.
|
||||
|
||||
#### Batch field processing with multi-node selection
|
||||
|
||||
For XML documents containing metrics with a large number of fields or where the fields are not known before (e.g. an unknown set of `Variable` nodes in the *example.xml*), field selectors can be used. This example shows how to generate a metric for each *Sensor* in the example with fields derived from the *Variable* nodes.
|
||||
|
||||
Config:
|
||||
```toml
|
||||
[[inputs.file]]
|
||||
files = ["example.xml"]
|
||||
data_format = "xml"
|
||||
|
||||
[[inputs.file.xml]]
|
||||
metric_selection = "/Bus/child::Sensor"
|
||||
metric_name = "string('sensors')"
|
||||
|
||||
timestamp = "/Gateway/Timestamp"
|
||||
timestamp_format = "2006-01-02T15:04:05Z"
|
||||
|
||||
field_selection = "child::Variable"
|
||||
field_name = "name(@*[1])"
|
||||
field_value = "number(@*[1])"
|
||||
|
||||
[inputs.file.xml.tags]
|
||||
name = "substring-after(@name, ' ')"
|
||||
```
|
||||
|
||||
Output:
|
||||
```
|
||||
sensors,host=Hugin,name=Facility\ A consumers=3,frequency=49.78,power=123.4,temperature=20 1596294243000000000
|
||||
sensors,host=Hugin,name=Facility\ B consumers=1,frequency=49.78,power=14.3,temperature=23.1 1596294243000000000
|
||||
sensors,host=Hugin,name=Facility\ C consumers=0,frequency=49.78,power=0.02,temperature=19.7 1596294243000000000
|
||||
```
|
||||
|
||||
Using the `metric_selection` option we select all `Sensor` nodes in the XML document. For each *Sensor* we then use `field_selection` to select all child nodes of the sensor as *field-nodes* Please note that the field selection is relative to the selected nodes.
|
||||
For each selected *field-node* we use `field_name` and `field_value` to determining the field's name and value, respectively. The `field_name` derives the name of the first attribute of the node, while `field_value` derives the value of the first attribute and converts the result to a number.
|
||||
|
||||
[xpath lib]: https://github.com/antchfx/xpath
|
||||
[xml]: https://www.w3.org/XML/
|
||||
[xpath]: https://www.w3.org/TR/xpath/
|
||||
[time const]: https://golang.org/pkg/time/#pkg-constants
|
||||
[time parse]: https://golang.org/pkg/time/#Parse
|
||||
|
|
@ -0,0 +1,422 @@
|
|||
package xml
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/antchfx/xmlquery"
|
||||
"github.com/antchfx/xpath"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/metric"
|
||||
)
|
||||
|
||||
type Parser struct {
|
||||
Configs []Config
|
||||
DefaultTags map[string]string
|
||||
Log telegraf.Logger
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
MetricName string
|
||||
MetricQuery string `toml:"metric_name"`
|
||||
Selection string `toml:"metric_selection"`
|
||||
Timestamp string `toml:"timestamp"`
|
||||
TimestampFmt string `toml:"timestamp_format"`
|
||||
Tags map[string]string `toml:"tags"`
|
||||
Fields map[string]string `toml:"fields"`
|
||||
FieldsInt map[string]string `toml:"fields_int"`
|
||||
|
||||
FieldSelection string `toml:"field_selection"`
|
||||
FieldNameQuery string `toml:"field_name"`
|
||||
FieldValueQuery string `toml:"field_value"`
|
||||
FieldNameExpand bool `toml:"field_name_expansion"`
|
||||
}
|
||||
|
||||
func (p *Parser) Parse(buf []byte) ([]telegraf.Metric, error) {
|
||||
t := time.Now()
|
||||
|
||||
// Parse the XML
|
||||
doc, err := xmlquery.Parse(strings.NewReader(string(buf)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Queries
|
||||
metrics := make([]telegraf.Metric, 0)
|
||||
for _, config := range p.Configs {
|
||||
if len(config.Selection) == 0 {
|
||||
config.Selection = "/"
|
||||
}
|
||||
selectedNodes, err := xmlquery.QueryAll(doc, config.Selection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(selectedNodes) < 1 || selectedNodes[0] == nil {
|
||||
p.debugEmptyQuery("metric selection", doc, config.Selection)
|
||||
return nil, fmt.Errorf("cannot parse with empty selection node")
|
||||
}
|
||||
|
||||
for _, selected := range selectedNodes {
|
||||
m, err := p.parseQuery(t, doc, selected, config)
|
||||
if err != nil {
|
||||
return metrics, err
|
||||
}
|
||||
|
||||
metrics = append(metrics, m)
|
||||
}
|
||||
}
|
||||
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
|
||||
t := time.Now()
|
||||
|
||||
switch len(p.Configs) {
|
||||
case 0:
|
||||
return nil, nil
|
||||
case 1:
|
||||
config := p.Configs[0]
|
||||
|
||||
doc, err := xmlquery.Parse(strings.NewReader(line))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
selected := doc
|
||||
if len(config.Selection) > 0 {
|
||||
selectedNodes, err := xmlquery.QueryAll(doc, config.Selection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(selectedNodes) < 1 || selectedNodes[0] == nil {
|
||||
p.debugEmptyQuery("metric selection", doc, config.Selection)
|
||||
return nil, fmt.Errorf("cannot parse line with empty selection")
|
||||
} else if len(selectedNodes) != 1 {
|
||||
return nil, fmt.Errorf("cannot parse line with multiple selected nodes (%d)", len(selectedNodes))
|
||||
}
|
||||
selected = selectedNodes[0]
|
||||
}
|
||||
|
||||
return p.parseQuery(t, doc, selected, config)
|
||||
}
|
||||
return nil, fmt.Errorf("cannot parse line with multiple (%d) configurations", len(p.Configs))
|
||||
}
|
||||
|
||||
func (p *Parser) SetDefaultTags(tags map[string]string) {
|
||||
p.DefaultTags = tags
|
||||
}
|
||||
|
||||
func (p *Parser) parseQuery(starttime time.Time, doc, selected *xmlquery.Node, config Config) (telegraf.Metric, error) {
|
||||
var timestamp time.Time
|
||||
var metricname string
|
||||
|
||||
// Determine the metric name. If a query was specified, use the result of this query and the default metric name
|
||||
// otherwise.
|
||||
metricname = config.MetricName
|
||||
if len(config.MetricQuery) > 0 {
|
||||
v, err := executeQuery(doc, selected, config.MetricQuery)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query metric name: %v", err)
|
||||
}
|
||||
metricname = v.(string)
|
||||
}
|
||||
|
||||
// By default take the time the parser was invoked and override the value
|
||||
// with the queried timestamp if an expresion was specified.
|
||||
timestamp = starttime
|
||||
if len(config.Timestamp) > 0 {
|
||||
v, err := executeQuery(doc, selected, config.Timestamp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query timestamp: %v", err)
|
||||
}
|
||||
switch v.(type) {
|
||||
case string:
|
||||
// Parse the string with the given format or assume the string to contain
|
||||
// a unix timestamp in seconds if no format is given.
|
||||
if len(config.TimestampFmt) < 1 || strings.HasPrefix(config.TimestampFmt, "unix") {
|
||||
var nanoseconds int64
|
||||
|
||||
t, err := strconv.ParseFloat(v.(string), 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse unix timestamp: %v", err)
|
||||
}
|
||||
|
||||
switch config.TimestampFmt {
|
||||
case "unix_ns":
|
||||
nanoseconds = int64(t)
|
||||
case "unix_us":
|
||||
nanoseconds = int64(t * 1e3)
|
||||
case "unix_ms":
|
||||
nanoseconds = int64(t * 1e6)
|
||||
default:
|
||||
nanoseconds = int64(t * 1e9)
|
||||
}
|
||||
timestamp = time.Unix(0, nanoseconds)
|
||||
} else {
|
||||
timestamp, err = time.Parse(config.TimestampFmt, v.(string))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query timestamp format: %v", err)
|
||||
}
|
||||
}
|
||||
case float64:
|
||||
// Assume the value to contain a timestamp in seconds and fractions thereof.
|
||||
timestamp = time.Unix(0, int64(v.(float64)*1e9))
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown format '%T' for timestamp query '%v'", v, config.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
// Query tags and add default ones
|
||||
tags := make(map[string]string)
|
||||
for name, query := range config.Tags {
|
||||
// Execute the query and cast the returned values into strings
|
||||
v, err := executeQuery(doc, selected, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query tag '%s': %v", name, err)
|
||||
}
|
||||
switch v.(type) {
|
||||
case string:
|
||||
tags[name] = v.(string)
|
||||
case bool:
|
||||
tags[name] = strconv.FormatBool(v.(bool))
|
||||
case float64:
|
||||
tags[name] = strconv.FormatFloat(v.(float64), 'G', -1, 64)
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown format '%T' for tag '%s'", v, name)
|
||||
}
|
||||
}
|
||||
for name, v := range p.DefaultTags {
|
||||
tags[name] = v
|
||||
}
|
||||
|
||||
// Query fields
|
||||
fields := make(map[string]interface{})
|
||||
for name, query := range config.FieldsInt {
|
||||
// Execute the query and cast the returned values into integers
|
||||
v, err := executeQuery(doc, selected, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query field (int) '%s': %v", name, err)
|
||||
}
|
||||
switch v.(type) {
|
||||
case string:
|
||||
fields[name], err = strconv.ParseInt(v.(string), 10, 54)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse field (int) '%s': %v", name, err)
|
||||
}
|
||||
case bool:
|
||||
fields[name] = int64(0)
|
||||
if v.(bool) {
|
||||
fields[name] = int64(1)
|
||||
}
|
||||
case float64:
|
||||
fields[name] = int64(v.(float64))
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown format '%T' for field (int) '%s'", v, name)
|
||||
}
|
||||
}
|
||||
|
||||
for name, query := range config.Fields {
|
||||
// Execute the query and store the result in fields
|
||||
v, err := executeQuery(doc, selected, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query field '%s': %v", name, err)
|
||||
}
|
||||
fields[name] = v
|
||||
}
|
||||
|
||||
// Handle the field batch definitions if any.
|
||||
if len(config.FieldSelection) > 0 {
|
||||
fieldnamequery := "name()"
|
||||
fieldvaluequery := "."
|
||||
if len(config.FieldNameQuery) > 0 {
|
||||
fieldnamequery = config.FieldNameQuery
|
||||
}
|
||||
if len(config.FieldValueQuery) > 0 {
|
||||
fieldvaluequery = config.FieldValueQuery
|
||||
}
|
||||
|
||||
// Query all fields
|
||||
selectedFieldNodes, err := xmlquery.QueryAll(selected, config.FieldSelection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(selectedFieldNodes) > 0 && selectedFieldNodes[0] != nil {
|
||||
for _, selectedfield := range selectedFieldNodes {
|
||||
n, err := executeQuery(doc, selectedfield, fieldnamequery)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query field name with query '%s': %v", fieldnamequery, err)
|
||||
}
|
||||
name, ok := n.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("failed to query field name with query '%s': result is not a string (%v)", fieldnamequery, n)
|
||||
}
|
||||
v, err := executeQuery(doc, selectedfield, fieldvaluequery)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query field value for '%s': %v", name, err)
|
||||
}
|
||||
path := name
|
||||
if config.FieldNameExpand {
|
||||
p := getNodePath(selectedfield, selected, "_")
|
||||
if len(p) > 0 {
|
||||
path = p + "_" + name
|
||||
}
|
||||
}
|
||||
|
||||
// Check if field name already exists and if so, append an index number.
|
||||
if _, ok := fields[path]; ok {
|
||||
for i := 1; ; i++ {
|
||||
p := path + "_" + strconv.Itoa(i)
|
||||
if _, ok := fields[p]; !ok {
|
||||
path = p
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fields[path] = v
|
||||
}
|
||||
} else {
|
||||
p.debugEmptyQuery("field selection", selected, config.FieldSelection)
|
||||
}
|
||||
}
|
||||
|
||||
return metric.New(metricname, tags, fields, timestamp)
|
||||
}
|
||||
|
||||
func getNodePath(node, relativeTo *xmlquery.Node, sep string) string {
|
||||
names := make([]string, 0)
|
||||
|
||||
// Climb up the tree and collect the node names
|
||||
n := node.Parent
|
||||
for n != nil && n != relativeTo {
|
||||
names = append(names, n.Data)
|
||||
n = n.Parent
|
||||
}
|
||||
|
||||
if len(names) < 1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Construct the nodes
|
||||
path := ""
|
||||
for _, name := range names {
|
||||
path = name + sep + path
|
||||
}
|
||||
|
||||
return path[:len(path)-1]
|
||||
}
|
||||
|
||||
func executeQuery(doc, selected *xmlquery.Node, query string) (r interface{}, err error) {
|
||||
// Check if the query is relative or absolute and set the root for the query
|
||||
root := selected
|
||||
if strings.HasPrefix(query, "/") {
|
||||
root = doc
|
||||
}
|
||||
|
||||
// Compile the query
|
||||
expr, err := xpath.Compile(query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile query '%s': %v", query, err)
|
||||
}
|
||||
|
||||
// Evaluate the compiled expression and handle returned node-iterators
|
||||
// separately. Those iterators will be returned for queries directly
|
||||
// referencing a node (value or attribute).
|
||||
n := expr.Evaluate(xmlquery.CreateXPathNavigator(root))
|
||||
if iter, ok := n.(*xpath.NodeIterator); ok {
|
||||
// We got an iterator, so take the first match and get the referenced
|
||||
// property. This will always be a string.
|
||||
if iter.MoveNext() {
|
||||
r = iter.Current().Value()
|
||||
}
|
||||
} else {
|
||||
r = n
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func splitLastPathElement(query string) []string {
|
||||
// This is a rudimentary xpath-parser that splits the path
|
||||
// into the last path element and the remaining path-part.
|
||||
// The last path element is then further splitted into
|
||||
// parts such as attributes or selectors. Each returned
|
||||
// element is a full path!
|
||||
|
||||
// Nothing left
|
||||
if query == "" || query == "/" || query == "//" || query == "." {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
seperatorIdx := strings.LastIndex(query, "/")
|
||||
if seperatorIdx < 0 {
|
||||
query = "./" + query
|
||||
seperatorIdx = 1
|
||||
}
|
||||
|
||||
// For double slash we want to split at the first slash
|
||||
if seperatorIdx > 0 && query[seperatorIdx-1] == byte('/') {
|
||||
seperatorIdx--
|
||||
}
|
||||
|
||||
base := query[:seperatorIdx]
|
||||
if base == "" {
|
||||
base = "/"
|
||||
}
|
||||
|
||||
elements := make([]string, 1)
|
||||
elements[0] = base
|
||||
|
||||
offset := seperatorIdx
|
||||
if i := strings.Index(query[offset:], "::"); i >= 0 {
|
||||
// Check for axis operator
|
||||
offset += i
|
||||
elements = append(elements, query[:offset]+"::*")
|
||||
}
|
||||
|
||||
if i := strings.Index(query[offset:], "["); i >= 0 {
|
||||
// Check for predicates
|
||||
offset += i
|
||||
elements = append(elements, query[:offset])
|
||||
} else if i := strings.Index(query[offset:], "@"); i >= 0 {
|
||||
// Check for attributes
|
||||
offset += i
|
||||
elements = append(elements, query[:offset])
|
||||
}
|
||||
|
||||
return elements
|
||||
}
|
||||
|
||||
func (p *Parser) debugEmptyQuery(operation string, root *xmlquery.Node, initialquery string) {
|
||||
if p.Log == nil {
|
||||
return
|
||||
}
|
||||
|
||||
query := initialquery
|
||||
|
||||
// We already know that the
|
||||
p.Log.Debugf("got 0 nodes for query %q in %s", query, operation)
|
||||
for {
|
||||
parts := splitLastPathElement(query)
|
||||
if len(parts) < 1 {
|
||||
return
|
||||
}
|
||||
for i := len(parts) - 1; i >= 0; i-- {
|
||||
q := parts[i]
|
||||
nodes, err := xmlquery.QueryAll(root, q)
|
||||
if err != nil {
|
||||
p.Log.Debugf("executing query %q in %s failed: %v", q, operation, err)
|
||||
return
|
||||
}
|
||||
p.Log.Debugf("got %d nodes for query %q in %s", len(nodes), q, operation)
|
||||
if len(nodes) > 0 && nodes[0] != nil {
|
||||
return
|
||||
}
|
||||
query = parts[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,31 @@
|
|||
<?xml version="1.0"?>
|
||||
<Gateway>
|
||||
<Name>Main Gateway</Name>
|
||||
<Timestamp>2020-08-01T15:04:03Z</Timestamp>
|
||||
<Sequence>12</Sequence>
|
||||
<Status>ok</Status>
|
||||
</Gateway>
|
||||
|
||||
<Bus>
|
||||
<Sensor name="Sensor Facility A">
|
||||
<Variable temperature="20.0"/>
|
||||
<Variable power="123.4"/>
|
||||
<Variable frequency="49.78"/>
|
||||
<Variable consumers="3"/>
|
||||
<Mode>busy</Mode>
|
||||
</Sensor>
|
||||
<Sensor name="Sensor Facility B">
|
||||
<Variable temperature="23.1"/>
|
||||
<Variable power="14.3"/>
|
||||
<Variable frequency="49.78"/>
|
||||
<Variable consumers="1"/>
|
||||
<Mode>standby</Mode>
|
||||
</Sensor>
|
||||
<Sensor name="Sensor Facility C">
|
||||
<Variable temperature="19.7"/>
|
||||
<Variable power="0.02"/>
|
||||
<Variable frequency="49.78"/>
|
||||
<Variable consumers="0"/>
|
||||
<Mode>error</Mode>
|
||||
</Sensor>
|
||||
</Bus>
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
# Simple example for using the xml-parser.
|
||||
#
|
||||
# File:
|
||||
# testcases/multisensor.xml
|
||||
#
|
||||
# Expected Output:
|
||||
# xml,gateway=Main seqnr=12i,ok=true
|
||||
#
|
||||
|
||||
[tags]
|
||||
gateway = "substring-before(/Gateway/Name, ' ')"
|
||||
|
||||
[fields_int]
|
||||
seqnr = "/Gateway/Sequence"
|
||||
|
||||
[fields]
|
||||
ok = "/Gateway/Status = 'ok'"
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
# Example for explicitly selecting fields from a bunch of selected metrics.
|
||||
#
|
||||
# File:
|
||||
# testcases/multisensor.xml
|
||||
#
|
||||
# Expected Output:
|
||||
# sensors,name=Facility\ A consumers=3i,frequency=49.78,power=123.4,temperature=20,ok=true 1596294243000000000
|
||||
# sensors,name=Facility\ B consumers=1i,frequency=49.78,power=14.3,temperature=23.1,ok=true 1596294243000000000
|
||||
# sensors,name=Facility\ C consumers=0i,frequency=49.78,power=0.02,temperature=19.7,ok=false 1596294243000000000
|
||||
#
|
||||
|
||||
metric_selection = "/Bus/child::Sensor"
|
||||
metric_name = "string('sensors')"
|
||||
|
||||
timestamp = "/Gateway/Timestamp"
|
||||
timestamp_format = "2006-01-02T15:04:05Z"
|
||||
|
||||
[tags]
|
||||
name = "substring-after(@name, ' ')"
|
||||
|
||||
[fields_int]
|
||||
consumers = "Variable/@consumers"
|
||||
|
||||
[fields]
|
||||
temperature = "number(Variable/@temperature)"
|
||||
power = "number(Variable/@power)"
|
||||
frequency = "number(Variable/@frequency)"
|
||||
ok = "Mode != 'error'"
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
# Example for batch selecting fields from a bunch of selected metrics.
|
||||
#
|
||||
# File:
|
||||
# testcases/multisensor.xml
|
||||
#
|
||||
# Expected Output:
|
||||
# sensors,name=Facility\ A consumers=3,frequency=49.78,power=123.4,temperature=20 1596294243000000000
|
||||
# sensors,name=Facility\ B consumers=1,frequency=49.78,power=14.3,temperature=23.1 1596294243000000000
|
||||
# sensors,name=Facility\ C consumers=0,frequency=49.78,power=0.02,temperature=19.7 1596294243000000000
|
||||
#
|
||||
|
||||
metric_selection = "/Bus/child::Sensor"
|
||||
metric_name = "string('sensors')"
|
||||
|
||||
timestamp = "/Gateway/Timestamp"
|
||||
timestamp_format = "2006-01-02T15:04:05Z"
|
||||
|
||||
field_selection = "child::Variable"
|
||||
field_name = "name(@*[1])"
|
||||
field_value = "number(@*[1])"
|
||||
|
||||
[tags]
|
||||
name = "substring-after(@name, ' ')"
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
# Example for parsing openweathermap five-day-forecast data.
|
||||
#
|
||||
# File:
|
||||
# testcases/openweathermap_5d.xml
|
||||
#
|
||||
# Expected Output:
|
||||
# weather,city=London,country=GB clouds=64i,humidity=96i,precipitation=5,temperature=16.89,wind_direction=253.5,wind_speed=4.9 1435654800000000000
|
||||
# weather,city=London,country=GB clouds=44i,humidity=97i,precipitation=99,temperature=17.23,wind_direction=248.001,wind_speed=4.86 1435665600000000000
|
||||
#
|
||||
|
||||
metric_name = "'weather'"
|
||||
metric_selection = "//forecast/*"
|
||||
timestamp = "@from"
|
||||
timestamp_format = "2006-01-02T15:04:05"
|
||||
|
||||
[tags]
|
||||
city = "/weatherdata/location/name"
|
||||
country = "/weatherdata/location/country"
|
||||
|
||||
[fields_int]
|
||||
humidity = "humidity/@value"
|
||||
clouds = "clouds/@all"
|
||||
|
||||
[fields]
|
||||
precipitation = "number(precipitation/@value)"
|
||||
wind_direction = "number(windDirection/@deg)"
|
||||
wind_speed = "number(windSpeed/@mps)"
|
||||
temperature = "number(temperature/@value)"
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Taken from https://openweathermap.org/forecast5#XML -->
|
||||
<weatherdata>
|
||||
<location>
|
||||
<name>London</name>
|
||||
<type/>
|
||||
<country>GB</country>
|
||||
<timezone>3600</timezone>
|
||||
<location altitude="0" latitude="51.5085" longitude="-0.1258" geobase="geonames" geobaseid="2643743"/>
|
||||
</location>
|
||||
<meta>
|
||||
<lastupdate>2015-06-30T00:00:00Z</lastupdate>
|
||||
</meta>
|
||||
<sun rise="2015-06-30T10:08:46" set="2015-07-01T01:06:20"/>
|
||||
<forecast>
|
||||
<time from="2015-06-30T09:00:00" to="2015-06-30T12:00:00">
|
||||
<symbol number="500" name="light rain" var="10n"/>
|
||||
<precipitation value="5" unit="3h" type="rain"/>
|
||||
<windDirection deg="253.5" code="WSW" name="West-southwest"/>
|
||||
<windSpeed mps="4.9" name="Gentle Breeze"/>
|
||||
<temperature unit="celsius" value="16.89" min="16.89" max="17.375"/>
|
||||
<feels_like value="281.37" unit="kelvin"/>
|
||||
<pressure unit="hPa" value="989.51"/>
|
||||
<humidity value="96" unit="%"/>
|
||||
<clouds value="broken clouds" all="64" unit="%"/>
|
||||
</time>
|
||||
<time from="2015-06-30T12:00:00" to="2015-06-30T15:00:00">
|
||||
<symbol number="500" name="light rain" var="10d"/>
|
||||
<precipitation value="99" unit="3h" type="rain"/>
|
||||
<windDirection deg="248.001" code="WSW" name="West-southwest"/>
|
||||
<windSpeed mps="4.86" name="Gentle Breeze"/>
|
||||
<temperature unit="celsius" value="17.23" min="17.23" max="17.614"/>
|
||||
<pressure unit="hPa" value="991.29"/>
|
||||
<humidity value="97" unit="%"/>
|
||||
<clouds value="scattered clouds" all="44" unit="%"/>
|
||||
</time>
|
||||
</forecast>
|
||||
</weatherdata>
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
package testutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
)
|
||||
|
||||
type LineParser interface {
|
||||
ParseLine(line string) (telegraf.Metric, error)
|
||||
}
|
||||
|
||||
//ParseRawLinesFrom returns the raw lines between the given header and a trailing blank line
|
||||
func ParseRawLinesFrom(lines []string, header string) ([]string, error) {
|
||||
if len(lines) < 2 {
|
||||
// We need a line for HEADER and EMPTY TRAILING LINE
|
||||
return nil, fmt.Errorf("expected at least two lines to parse from")
|
||||
}
|
||||
start := -1
|
||||
for i := range lines {
|
||||
if strings.TrimLeft(lines[i], "# ") == header {
|
||||
start = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
if start < 0 {
|
||||
return nil, fmt.Errorf("header %q does not exist", header)
|
||||
}
|
||||
|
||||
output := make([]string, 0)
|
||||
for _, line := range lines[start:] {
|
||||
if !strings.HasPrefix(strings.TrimLeft(line, "\t "), "#") {
|
||||
return nil, fmt.Errorf("section does not end with trailing empty line")
|
||||
}
|
||||
|
||||
// Stop at empty line
|
||||
content := strings.TrimLeft(line, "# \t")
|
||||
if content == "" || content == "'''" {
|
||||
break
|
||||
}
|
||||
|
||||
output = append(output, content)
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
//ParseMetricsFrom parses metrics from the given lines in line-protocol following a header, with a trailing blank line
|
||||
func ParseMetricsFrom(lines []string, header string, parser LineParser) ([]telegraf.Metric, error) {
|
||||
if len(lines) < 2 {
|
||||
// We need a line for HEADER and EMPTY TRAILING LINE
|
||||
return nil, fmt.Errorf("expected at least two lines to parse from")
|
||||
}
|
||||
start := -1
|
||||
for i := range lines {
|
||||
if strings.TrimLeft(lines[i], "# ") == header {
|
||||
start = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
if start < 0 {
|
||||
return nil, fmt.Errorf("header %q does not exist", header)
|
||||
}
|
||||
|
||||
metrics := make([]telegraf.Metric, 0)
|
||||
for _, line := range lines[start:] {
|
||||
if !strings.HasPrefix(strings.TrimLeft(line, "\t "), "#") {
|
||||
return nil, fmt.Errorf("section does not end with trailing empty line")
|
||||
}
|
||||
|
||||
// Stop at empty line
|
||||
content := strings.TrimLeft(line, "# \t")
|
||||
if content == "" || content == "'''" {
|
||||
break
|
||||
}
|
||||
|
||||
m, err := parser.ParseLine(content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse metric in %q failed: %v", content, err)
|
||||
}
|
||||
metrics = append(metrics, m)
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
Loading…
Reference in New Issue