feat: Migrate xpath parser to new style (#11218)

This commit is contained in:
Sven Rebhan 2022-06-08 21:39:02 +02:00 committed by GitHub
parent 0b7c3c4b24
commit 0d96968819
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 261 additions and 181 deletions

View File

@ -1842,9 +1842,7 @@ func (c *Config) missingTomlField(_ reflect.Type, key string) error {
"prefix", "prometheus_export_timestamp", "prometheus_ignore_timestamp", "prometheus_sort_metrics", "prometheus_string_as_label", "prefix", "prometheus_export_timestamp", "prometheus_ignore_timestamp", "prometheus_sort_metrics", "prometheus_string_as_label",
"separator", "splunkmetric_hec_routing", "splunkmetric_multimetric", "tag_keys", "separator", "splunkmetric_hec_routing", "splunkmetric_multimetric", "tag_keys",
"tagdrop", "tagexclude", "taginclude", "tagpass", "tags", "template", "templates", "tagdrop", "tagexclude", "taginclude", "tagpass", "tags", "template", "templates",
"value_field_name", "wavefront_source_override", "wavefront_use_strict", "wavefront_disable_prefix_conversion", "value_field_name", "wavefront_source_override", "wavefront_use_strict", "wavefront_disable_prefix_conversion":
"xml", "xpath", "xpath_json", "xpath_msgpack", "xpath_protobuf", "xpath_print_document",
"xpath_protobuf_file", "xpath_protobuf_type", "xpath_protobuf_import_paths":
// ignore fields that are common to all plugins. // ignore fields that are common to all plugins.
default: default:

View File

@ -406,7 +406,6 @@ func TestConfig_ParserInterfaceNewFormat(t *testing.T) {
} }
override := map[string]struct { override := map[string]struct {
cfg *parsers.Config
param map[string]interface{} param map[string]interface{}
mask []string mask []string
}{ }{
@ -420,11 +419,6 @@ func TestConfig_ParserInterfaceNewFormat(t *testing.T) {
mask: []string{"Now"}, mask: []string{"Now"},
}, },
"xpath_protobuf": { "xpath_protobuf": {
cfg: &parsers.Config{
MetricName: "parser_test_new",
XPathProtobufFile: "testdata/addressbook.proto",
XPathProtobufType: "addressbook.AddressBook",
},
param: map[string]interface{}{ param: map[string]interface{}{
"ProtobufMessageDef": "testdata/addressbook.proto", "ProtobufMessageDef": "testdata/addressbook.proto",
"ProtobufMessageType": "addressbook.AddressBook", "ProtobufMessageType": "addressbook.AddressBook",
@ -435,10 +429,6 @@ func TestConfig_ParserInterfaceNewFormat(t *testing.T) {
expected := make([]telegraf.Parser, 0, len(formats)) expected := make([]telegraf.Parser, 0, len(formats))
for _, format := range formats { for _, format := range formats {
formatCfg := &cfg formatCfg := &cfg
settings, hasOverride := override[format]
if hasOverride && settings.cfg != nil {
formatCfg = settings.cfg
}
formatCfg.DataFormat = format formatCfg.DataFormat = format
logger := models.NewLogger("parsers", format, cfg.MetricName) logger := models.NewLogger("parsers", format, cfg.MetricName)
@ -555,7 +545,6 @@ func TestConfig_ParserInterfaceOldFormat(t *testing.T) {
} }
override := map[string]struct { override := map[string]struct {
cfg *parsers.Config
param map[string]interface{} param map[string]interface{}
mask []string mask []string
}{ }{
@ -569,11 +558,6 @@ func TestConfig_ParserInterfaceOldFormat(t *testing.T) {
mask: []string{"Now"}, mask: []string{"Now"},
}, },
"xpath_protobuf": { "xpath_protobuf": {
cfg: &parsers.Config{
MetricName: "parser_test_new",
XPathProtobufFile: "testdata/addressbook.proto",
XPathProtobufType: "addressbook.AddressBook",
},
param: map[string]interface{}{ param: map[string]interface{}{
"ProtobufMessageDef": "testdata/addressbook.proto", "ProtobufMessageDef": "testdata/addressbook.proto",
"ProtobufMessageType": "addressbook.AddressBook", "ProtobufMessageType": "addressbook.AddressBook",
@ -584,10 +568,6 @@ func TestConfig_ParserInterfaceOldFormat(t *testing.T) {
expected := make([]telegraf.Parser, 0, len(formats)) expected := make([]telegraf.Parser, 0, len(formats))
for _, format := range formats { for _, format := range formats {
formatCfg := &cfg formatCfg := &cfg
settings, hasOverride := override[format]
if hasOverride && settings.cfg != nil {
formatCfg = settings.cfg
}
formatCfg.DataFormat = format formatCfg.DataFormat = format
logger := models.NewLogger("parsers", format, cfg.MetricName) logger := models.NewLogger("parsers", format, cfg.MetricName)

View File

@ -3,4 +3,5 @@ package all
import ( import (
//Blank imports for plugins to register themselves //Blank imports for plugins to register themselves
_ "github.com/influxdata/telegraf/plugins/parsers/csv" _ "github.com/influxdata/telegraf/plugins/parsers/csv"
_ "github.com/influxdata/telegraf/plugins/parsers/xpath"
) )

View File

@ -19,7 +19,6 @@ import (
"github.com/influxdata/telegraf/plugins/parsers/prometheusremotewrite" "github.com/influxdata/telegraf/plugins/parsers/prometheusremotewrite"
"github.com/influxdata/telegraf/plugins/parsers/value" "github.com/influxdata/telegraf/plugins/parsers/value"
"github.com/influxdata/telegraf/plugins/parsers/wavefront" "github.com/influxdata/telegraf/plugins/parsers/wavefront"
"github.com/influxdata/telegraf/plugins/parsers/xpath"
) )
// Creator is the function to create a new parser // Creator is the function to create a new parser
@ -184,12 +183,12 @@ type Config struct {
ValueFieldName string `toml:"value_field_name"` ValueFieldName string `toml:"value_field_name"`
// XPath configuration // XPath configuration
XPathPrintDocument bool `toml:"xpath_print_document"` XPathPrintDocument bool `toml:"xpath_print_document"`
XPathProtobufFile string `toml:"xpath_protobuf_file"` XPathProtobufFile string `toml:"xpath_protobuf_file"`
XPathProtobufType string `toml:"xpath_protobuf_type"` XPathProtobufType string `toml:"xpath_protobuf_type"`
XPathProtobufImportPaths []string `toml:"xpath_protobuf_import_paths"` XPathProtobufImportPaths []string `toml:"xpath_protobuf_import_paths"`
XPathAllowEmptySelection bool `toml:"xpath_allow_empty_selection"` XPathAllowEmptySelection bool `toml:"xpath_allow_empty_selection"`
XPathConfig []XPathConfig XPathConfig []XPathConfig `toml:"xpath"`
// JSONPath configuration // JSONPath configuration
JSONV2Config []JSONV2Config `toml:"json_v2"` JSONV2Config []JSONV2Config `toml:"json_v2"`
@ -201,7 +200,29 @@ type Config struct {
LogFmtTagKeys []string `toml:"logfmt_tag_keys"` LogFmtTagKeys []string `toml:"logfmt_tag_keys"`
} }
type XPathConfig xpath.Config // XPathConfig definition for backward compatibitlity ONLY.
// We need this here to avoid cyclic dependencies. However, we need
// to move this to plugins/parsers/xpath once we deprecate parser
// construction via `NewParser()`.
type XPathConfig struct {
MetricQuery string `toml:"metric_name"`
Selection string `toml:"metric_selection"`
Timestamp string `toml:"timestamp"`
TimestampFmt string `toml:"timestamp_format"`
Tags map[string]string `toml:"tags"`
Fields map[string]string `toml:"fields"`
FieldsInt map[string]string `toml:"fields_int"`
FieldSelection string `toml:"field_selection"`
FieldNameQuery string `toml:"field_name"`
FieldValueQuery string `toml:"field_value"`
FieldNameExpand bool `toml:"field_name_expansion"`
TagSelection string `toml:"tag_selection"`
TagNameQuery string `toml:"tag_name"`
TagValueQuery string `toml:"tag_value"`
TagNameExpand bool `toml:"tag_name_expansion"`
}
type JSONV2Config struct { type JSONV2Config struct {
json_v2.Config json_v2.Config
@ -280,17 +301,6 @@ func NewParser(config *Config) (Parser, error) {
) )
case "prometheusremotewrite": case "prometheusremotewrite":
parser, err = NewPrometheusRemoteWriteParser(config.DefaultTags) parser, err = NewPrometheusRemoteWriteParser(config.DefaultTags)
case "xml", "xpath_json", "xpath_msgpack", "xpath_protobuf":
parser = &xpath.Parser{
Format: config.DataFormat,
ProtobufMessageDef: config.XPathProtobufFile,
ProtobufMessageType: config.XPathProtobufType,
ProtobufImportPaths: config.XPathProtobufImportPaths,
PrintDocument: config.XPathPrintDocument,
DefaultTags: config.DefaultTags,
AllowEmptySelection: config.XPathAllowEmptySelection,
Configs: NewXPathParserConfigs(config.MetricName, config.XPathConfig),
}
case "json_v2": case "json_v2":
parser, err = NewJSONPathParser(config.JSONV2Config) parser, err = NewJSONPathParser(config.JSONV2Config)
default: default:
@ -429,17 +439,6 @@ func NewPrometheusRemoteWriteParser(defaultTags map[string]string) (Parser, erro
}, nil }, nil
} }
func NewXPathParserConfigs(metricName string, cfgs []XPathConfig) []xpath.Config {
// Convert the config formats which is a one-to-one copy
configs := make([]xpath.Config, 0, len(cfgs))
for _, cfg := range cfgs {
config := xpath.Config(cfg)
config.MetricDefaultName = metricName
configs = append(configs, config)
}
return configs
}
func NewJSONPathParser(jsonv2config []JSONV2Config) (Parser, error) { func NewJSONPathParser(jsonv2config []JSONV2Config) (Parser, error) {
configs := make([]json_v2.Config, len(jsonv2config)) configs := make([]json_v2.Config, len(jsonv2config))
for i, cfg := range jsonv2config { for i, cfg := range jsonv2config {

View File

@ -2,8 +2,11 @@ package parsers_test
import ( import (
"reflect" "reflect"
"sync"
"testing" "testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
@ -15,6 +18,8 @@ func TestRegistry_BackwardCompatibility(t *testing.T) {
cfg := &parsers.Config{ cfg := &parsers.Config{
MetricName: "parser_compatibility_test", MetricName: "parser_compatibility_test",
CSVHeaderRowCount: 42, CSVHeaderRowCount: 42,
XPathProtobufFile: "xpath/testcases/protos/addressbook.proto",
XPathProtobufType: "addressbook.AddressBook",
} }
// Some parsers need certain settings to not error. Furthermore, we // Some parsers need certain settings to not error. Furthermore, we
@ -29,6 +34,12 @@ func TestRegistry_BackwardCompatibility(t *testing.T) {
}, },
mask: []string{"TimeFunc"}, mask: []string{"TimeFunc"},
}, },
"xpath_protobuf": {
param: map[string]interface{}{
"ProtobufMessageDef": cfg.XPathProtobufFile,
"ProtobufMessageType": cfg.XPathProtobufType,
},
},
} }
for name, creator := range parsers.Parsers { for name, creator := range parsers.Parsers {
@ -52,19 +63,23 @@ func TestRegistry_BackwardCompatibility(t *testing.T) {
actual, err := parsers.NewParser(cfg) actual, err := parsers.NewParser(cfg)
require.NoError(t, err) require.NoError(t, err)
// Compare with mask // Determine the underlying type of the parser
if settings, found := override[name]; found { stype := reflect.Indirect(reflect.ValueOf(expected)).Interface()
a := reflect.Indirect(reflect.ValueOf(actual)) // Ignore all unexported fields and fields not relevant for functionality
e := reflect.Indirect(reflect.ValueOf(expected)) options := []cmp.Option{
for _, key := range settings.mask { cmpopts.IgnoreUnexported(stype),
af := a.FieldByName(key) cmpopts.IgnoreTypes(sync.Mutex{}),
ef := e.FieldByName(key) cmpopts.IgnoreInterfaces(struct{ telegraf.Logger }{}),
v := reflect.Zero(ef.Type())
af.Set(v)
ef.Set(v)
}
} }
require.EqualValuesf(t, expected, actual, "format %q", name)
// Add overrides and masks to compare options
if settings, found := override[name]; found {
options = append(options, cmpopts.IgnoreFields(stype, settings.mask...))
}
// Do a manual comparision as require.EqualValues will also work on unexported fields
// that cannot be cleared or ignored.
diff := cmp.Diff(expected, actual, options...)
require.Emptyf(t, diff, "Difference for %q", name)
} }
} }

View File

@ -1,6 +1,7 @@
package xpath package xpath
import ( import (
"errors"
"fmt" "fmt"
"strconv" "strconv"
"strings" "strings"
@ -11,6 +12,7 @@ import (
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/metric"
"github.com/influxdata/telegraf/plugins/parsers"
) )
type dataNode interface{} type dataNode interface{}
@ -24,39 +26,25 @@ type dataDocument interface {
} }
type Parser struct { type Parser struct {
Format string Format string `toml:"-"`
ProtobufMessageDef string ProtobufMessageDef string `toml:"xpath_protobuf_file"`
ProtobufMessageType string ProtobufMessageType string `toml:"xpath_protobuf_type"`
ProtobufImportPaths []string ProtobufImportPaths []string `toml:"xpath_protobuf_import_paths"`
PrintDocument bool PrintDocument bool `toml:"xpath_print_document"`
AllowEmptySelection bool AllowEmptySelection bool `toml:"xpath_allow_empty_selection"`
Configs []Config Configs []Config `toml:"xpath"`
DefaultTags map[string]string DefaultMetricName string `toml:"-"`
Log telegraf.Logger DefaultTags map[string]string `toml:"-"`
Log telegraf.Logger `toml:"-"`
document dataDocument document dataDocument
} }
type Config struct { // Config definition
MetricDefaultName string `toml:"-"` // This should be replaced by the actual definition once
MetricQuery string `toml:"metric_name"` // the compatibitlity-code is removed.
Selection string `toml:"metric_selection"` // Please check plugins/parsers/registry.go for now.
Timestamp string `toml:"timestamp"` type Config parsers.XPathConfig
TimestampFmt string `toml:"timestamp_format"`
Tags map[string]string `toml:"tags"`
Fields map[string]string `toml:"fields"`
FieldsInt map[string]string `toml:"fields_int"`
FieldSelection string `toml:"field_selection"`
FieldNameQuery string `toml:"field_name"`
FieldValueQuery string `toml:"field_value"`
FieldNameExpand bool `toml:"field_name_expansion"`
TagSelection string `toml:"tag_selection"`
TagNameQuery string `toml:"tag_name"`
TagValueQuery string `toml:"tag_value"`
TagNameExpand bool `toml:"tag_name_expansion"`
}
func (p *Parser) Init() error { func (p *Parser) Init() error {
switch p.Format { switch p.Format {
@ -81,6 +69,11 @@ func (p *Parser) Init() error {
return fmt.Errorf("unknown data-format %q for xpath parser", p.Format) return fmt.Errorf("unknown data-format %q for xpath parser", p.Format)
} }
// Make sure we do have a metric name
if p.DefaultMetricName == "" {
return errors.New("missing default metric name")
}
return nil return nil
} }
@ -129,7 +122,6 @@ func (p *Parser) Parse(buf []byte) ([]telegraf.Metric, error) {
} }
func (p *Parser) ParseLine(line string) (telegraf.Metric, error) { func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
metrics, err := p.Parse([]byte(line)) metrics, err := p.Parse([]byte(line))
if err != nil { if err != nil {
return nil, err return nil, err
@ -155,7 +147,7 @@ func (p *Parser) parseQuery(starttime time.Time, doc, selected dataNode, config
// Determine the metric name. If a query was specified, use the result of this query and the default metric name // Determine the metric name. If a query was specified, use the result of this query and the default metric name
// otherwise. // otherwise.
metricname = config.MetricDefaultName metricname = p.DefaultMetricName
if len(config.MetricQuery) > 0 { if len(config.MetricQuery) > 0 {
v, err := p.executeQuery(doc, selected, config.MetricQuery) v, err := p.executeQuery(doc, selected, config.MetricQuery)
if err != nil { if err != nil {
@ -512,3 +504,62 @@ func (p *Parser) debugEmptyQuery(operation string, root dataNode, initialquery s
} }
} }
} }
func init() {
// Register all variants
parsers.Add("xml",
func(defaultMetricName string) telegraf.Parser {
return &Parser{
Format: "xml",
DefaultMetricName: defaultMetricName,
}
},
)
parsers.Add("xpath_json",
func(defaultMetricName string) telegraf.Parser {
return &Parser{
Format: "xpath_json",
DefaultMetricName: defaultMetricName,
}
},
)
parsers.Add("xpath_msgpack",
func(defaultMetricName string) telegraf.Parser {
return &Parser{
Format: "xpath_msgpack",
DefaultMetricName: defaultMetricName,
}
},
)
parsers.Add("xpath_protobuf",
func(defaultMetricName string) telegraf.Parser {
return &Parser{
Format: "xpath_protobuf",
DefaultMetricName: defaultMetricName,
}
},
)
}
// InitFromConfig is a compatibitlity function to construct the parser the old way
func (p *Parser) InitFromConfig(config *parsers.Config) error {
p.Format = config.DataFormat
if p.Format == "xpath_protobuf" {
p.ProtobufMessageDef = config.XPathProtobufFile
p.ProtobufMessageType = config.XPathProtobufType
}
p.PrintDocument = config.XPathPrintDocument
p.DefaultMetricName = config.MetricName
p.DefaultTags = config.DefaultTags
// Convert the config formats which is a one-to-one copy
if len(config.XPathConfig) > 0 {
p.Configs = make([]Config, 0, len(config.XPathConfig))
for _, cfg := range config.XPathConfig {
config := Config(cfg)
p.Configs = append(p.Configs, config)
}
}
return p.Init()
}

View File

@ -126,7 +126,12 @@ func TestParseInvalidXML(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: tt.defaultTags, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "xml",
Configs: tt.configs,
DefaultTags: tt.defaultTags,
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
_, err := parser.ParseLine(tt.input) _, err := parser.ParseLine(tt.input)
@ -149,8 +154,7 @@ func TestInvalidTypeQueriesFail(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
FieldsInt: map[string]string{ FieldsInt: map[string]string{
"a": "/Device_1/value_string", "a": "/Device_1/value_string",
}, },
@ -163,7 +167,12 @@ func TestInvalidTypeQueriesFail(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: tt.defaultTags, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "xml",
Configs: tt.configs,
DefaultTags: tt.defaultTags,
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
_, err := parser.ParseLine(tt.input) _, err := parser.ParseLine(tt.input)
@ -186,8 +195,7 @@ func TestInvalidTypeQueries(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Fields: map[string]string{ Fields: map[string]string{
"a": "number(/Device_1/value_string)", "a": "number(/Device_1/value_string)",
}, },
@ -208,8 +216,7 @@ func TestInvalidTypeQueries(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Fields: map[string]string{ Fields: map[string]string{
"a": "boolean(/Device_1/value_string)", "a": "boolean(/Device_1/value_string)",
}, },
@ -229,7 +236,12 @@ func TestInvalidTypeQueries(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: tt.defaultTags, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "test",
Configs: tt.configs,
DefaultTags: tt.defaultTags,
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
actual, err := parser.ParseLine(tt.input) actual, err := parser.ParseLine(tt.input)
@ -253,8 +265,7 @@ func TestParseTimestamps(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
}, },
}, },
defaultTags: map[string]string{}, defaultTags: map[string]string{},
@ -270,9 +281,8 @@ func TestParseTimestamps(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix", TimestampFmt: "unix",
TimestampFmt: "unix",
}, },
}, },
defaultTags: map[string]string{}, defaultTags: map[string]string{},
@ -288,9 +298,8 @@ func TestParseTimestamps(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix_ms",
Timestamp: "/Device_1/Timestamp_unix_ms", TimestampFmt: "unix_ms",
TimestampFmt: "unix_ms",
}, },
}, },
defaultTags: map[string]string{}, defaultTags: map[string]string{},
@ -306,9 +315,8 @@ func TestParseTimestamps(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix_us",
Timestamp: "/Device_1/Timestamp_unix_us", TimestampFmt: "unix_us",
TimestampFmt: "unix_us",
}, },
}, },
defaultTags: map[string]string{}, defaultTags: map[string]string{},
@ -324,9 +332,8 @@ func TestParseTimestamps(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix_ns",
Timestamp: "/Device_1/Timestamp_unix_ns", TimestampFmt: "unix_ns",
TimestampFmt: "unix_ns",
}, },
}, },
defaultTags: map[string]string{}, defaultTags: map[string]string{},
@ -342,9 +349,8 @@ func TestParseTimestamps(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_iso",
Timestamp: "/Device_1/Timestamp_iso", TimestampFmt: "2006-01-02T15:04:05Z",
TimestampFmt: "2006-01-02T15:04:05Z",
}, },
}, },
defaultTags: map[string]string{}, defaultTags: map[string]string{},
@ -359,7 +365,12 @@ func TestParseTimestamps(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: tt.defaultTags, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "test",
Configs: tt.configs,
DefaultTags: tt.defaultTags,
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
actual, err := parser.ParseLine(tt.input) actual, err := parser.ParseLine(tt.input)
@ -383,8 +394,7 @@ func TestParseSingleValues(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Fields: map[string]string{ Fields: map[string]string{
"a": "/Device_1/value_int", "a": "/Device_1/value_int",
"b": "/Device_1/value_float", "b": "/Device_1/value_float",
@ -411,8 +421,7 @@ func TestParseSingleValues(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Fields: map[string]string{ Fields: map[string]string{
"a": "number(Device_1/value_int)", "a": "number(Device_1/value_int)",
"b": "number(/Device_1/value_float)", "b": "number(/Device_1/value_float)",
@ -439,8 +448,7 @@ func TestParseSingleValues(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Fields: map[string]string{ Fields: map[string]string{
"b": "number(/Device_1/value_float)", "b": "number(/Device_1/value_float)",
"c": "boolean(/Device_1/value_bool)", "c": "boolean(/Device_1/value_bool)",
@ -469,8 +477,7 @@ func TestParseSingleValues(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Fields: map[string]string{ Fields: map[string]string{
"x": "substring-before(/Device_1/value_position, ';')", "x": "substring-before(/Device_1/value_position, ';')",
"y": "substring-after(/Device_1/value_position, ';')", "y": "substring-after(/Device_1/value_position, ';')",
@ -493,8 +500,7 @@ func TestParseSingleValues(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Fields: map[string]string{ Fields: map[string]string{
"x": "number(substring-before(/Device_1/value_position, ';'))", "x": "number(substring-before(/Device_1/value_position, ';'))",
"y": "number(substring-after(/Device_1/value_position, ';'))", "y": "number(substring-after(/Device_1/value_position, ';'))",
@ -517,8 +523,7 @@ func TestParseSingleValues(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
FieldsInt: map[string]string{ FieldsInt: map[string]string{
"x": "substring-before(/Device_1/value_position, ';')", "x": "substring-before(/Device_1/value_position, ';')",
"y": "substring-after(/Device_1/value_position, ';')", "y": "substring-after(/Device_1/value_position, ';')",
@ -541,8 +546,7 @@ func TestParseSingleValues(t *testing.T) {
input: singleMetricValuesXML, input: singleMetricValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Tags: map[string]string{ Tags: map[string]string{
"state": "/Device_1/State", "state": "/Device_1/State",
"name": "substring-after(/Device_1/Name, ' ')", "name": "substring-after(/Device_1/Name, ' ')",
@ -564,7 +568,12 @@ func TestParseSingleValues(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: tt.defaultTags, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "test",
Configs: tt.configs,
DefaultTags: tt.defaultTags,
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
actual, err := parser.ParseLine(tt.input) actual, err := parser.ParseLine(tt.input)
@ -588,8 +597,7 @@ func TestParseSingleAttributes(t *testing.T) {
input: singleMetricAttributesXML, input: singleMetricAttributesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix/@value",
Timestamp: "/Device_1/Timestamp_unix/@value",
}, },
}, },
defaultTags: map[string]string{}, defaultTags: map[string]string{},
@ -605,9 +613,8 @@ func TestParseSingleAttributes(t *testing.T) {
input: singleMetricAttributesXML, input: singleMetricAttributesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_iso/@value",
Timestamp: "/Device_1/Timestamp_iso/@value", TimestampFmt: "2006-01-02T15:04:05Z",
TimestampFmt: "2006-01-02T15:04:05Z",
}, },
}, },
defaultTags: map[string]string{}, defaultTags: map[string]string{},
@ -623,8 +630,7 @@ func TestParseSingleAttributes(t *testing.T) {
input: singleMetricAttributesXML, input: singleMetricAttributesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix/@value",
Timestamp: "/Device_1/Timestamp_unix/@value",
Fields: map[string]string{ Fields: map[string]string{
"a": "/Device_1/attr_int/@_", "a": "/Device_1/attr_int/@_",
"b": "/Device_1/attr_float/@_", "b": "/Device_1/attr_float/@_",
@ -651,8 +657,7 @@ func TestParseSingleAttributes(t *testing.T) {
input: singleMetricAttributesXML, input: singleMetricAttributesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix/@value",
Timestamp: "/Device_1/Timestamp_unix/@value",
Fields: map[string]string{ Fields: map[string]string{
"a": "number(/Device_1/attr_int/@_)", "a": "number(/Device_1/attr_int/@_)",
"b": "number(/Device_1/attr_float/@_)", "b": "number(/Device_1/attr_float/@_)",
@ -679,8 +684,7 @@ func TestParseSingleAttributes(t *testing.T) {
input: singleMetricAttributesXML, input: singleMetricAttributesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix/@value",
Timestamp: "/Device_1/Timestamp_unix/@value",
Fields: map[string]string{ Fields: map[string]string{
"b": "number(/Device_1/attr_float/@_)", "b": "number(/Device_1/attr_float/@_)",
"c": "boolean(/Device_1/attr_bool/@_)", "c": "boolean(/Device_1/attr_bool/@_)",
@ -709,8 +713,7 @@ func TestParseSingleAttributes(t *testing.T) {
input: singleMetricAttributesXML, input: singleMetricAttributesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix/@value",
Timestamp: "/Device_1/Timestamp_unix/@value",
Fields: map[string]string{ Fields: map[string]string{
"name": "substring-after(/Device_1/Name/@value, ' ')", "name": "substring-after(/Device_1/Name/@value, ' ')",
}, },
@ -731,8 +734,7 @@ func TestParseSingleAttributes(t *testing.T) {
input: singleMetricAttributesXML, input: singleMetricAttributesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix/@value",
Timestamp: "/Device_1/Timestamp_unix/@value",
Tags: map[string]string{ Tags: map[string]string{
"state": "/Device_1/State/@_", "state": "/Device_1/State/@_",
"name": "substring-after(/Device_1/Name/@value, ' ')", "name": "substring-after(/Device_1/Name/@value, ' ')",
@ -755,8 +757,7 @@ func TestParseSingleAttributes(t *testing.T) {
input: singleMetricAttributesXML, input: singleMetricAttributesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Device_1/Timestamp_unix/@value",
Timestamp: "/Device_1/Timestamp_unix/@value",
Fields: map[string]string{ Fields: map[string]string{
"a": "/Device_1/attr_bool_numeric/@_ = 1", "a": "/Device_1/attr_bool_numeric/@_ = 1",
}, },
@ -776,7 +777,12 @@ func TestParseSingleAttributes(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: tt.defaultTags, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "test",
Configs: tt.configs,
DefaultTags: tt.defaultTags,
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
actual, err := parser.ParseLine(tt.input) actual, err := parser.ParseLine(tt.input)
@ -800,8 +806,7 @@ func TestParseMultiValues(t *testing.T) {
input: singleMetricMultiValuesXML, input: singleMetricMultiValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Timestamp/@value",
Timestamp: "/Timestamp/@value",
Fields: map[string]string{ Fields: map[string]string{
"a": "number(/Device/Value[1])", "a": "number(/Device/Value[1])",
"b": "number(/Device/Value[2])", "b": "number(/Device/Value[2])",
@ -832,8 +837,7 @@ func TestParseMultiValues(t *testing.T) {
input: singleMetricMultiValuesXML, input: singleMetricMultiValuesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Timestamp: "/Timestamp/@value",
Timestamp: "/Timestamp/@value",
FieldsInt: map[string]string{ FieldsInt: map[string]string{
"a": "/Device/Value[1]", "a": "/Device/Value[1]",
"b": "/Device/Value[2]", "b": "/Device/Value[2]",
@ -863,7 +867,12 @@ func TestParseMultiValues(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: tt.defaultTags, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "test",
Configs: tt.configs,
DefaultTags: tt.defaultTags,
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
actual, err := parser.ParseLine(tt.input) actual, err := parser.ParseLine(tt.input)
@ -887,9 +896,8 @@ func TestParseMultiNodes(t *testing.T) {
input: multipleNodesXML, input: multipleNodesXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", Selection: "/Device",
Selection: "/Device", Timestamp: "/Timestamp/@value",
Timestamp: "/Timestamp/@value",
Fields: map[string]string{ Fields: map[string]string{
"value": "number(Value)", "value": "number(Value)",
"active": "Active = 1", "active": "Active = 1",
@ -976,7 +984,12 @@ func TestParseMultiNodes(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: tt.defaultTags, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "test",
Configs: tt.configs,
DefaultTags: tt.defaultTags,
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
actual, err := parser.Parse([]byte(tt.input)) actual, err := parser.Parse([]byte(tt.input))
@ -1000,9 +1013,8 @@ func TestParseMetricQuery(t *testing.T) {
input: metricNameQueryXML, input: metricNameQueryXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", MetricQuery: "name(/Device_1/Metric/@*[1])",
MetricQuery: "name(/Device_1/Metric/@*[1])", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Fields: map[string]string{ Fields: map[string]string{
"value": "/Device_1/Metric/@*[1]", "value": "/Device_1/Metric/@*[1]",
}, },
@ -1023,9 +1035,8 @@ func TestParseMetricQuery(t *testing.T) {
input: metricNameQueryXML, input: metricNameQueryXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", MetricQuery: "'the_metric'",
MetricQuery: "'the_metric'", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Fields: map[string]string{ Fields: map[string]string{
"value": "/Device_1/Metric/@*[1]", "value": "/Device_1/Metric/@*[1]",
}, },
@ -1045,7 +1056,12 @@ func TestParseMetricQuery(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: tt.defaultTags, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "test",
Configs: tt.configs,
DefaultTags: tt.defaultTags,
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
actual, err := parser.ParseLine(tt.input) actual, err := parser.ParseLine(tt.input)
@ -1068,9 +1084,8 @@ func TestParseErrors(t *testing.T) {
input: metricNameQueryXML, input: metricNameQueryXML,
configs: []Config{ configs: []Config{
{ {
MetricDefaultName: "test", MetricQuery: "arbitrary",
MetricQuery: "arbitrary", Timestamp: "/Device_1/Timestamp_unix",
Timestamp: "/Device_1/Timestamp_unix",
Fields: map[string]string{ Fields: map[string]string{
"value": "/Device_1/Metric/@*[1]", "value": "/Device_1/Metric/@*[1]",
}, },
@ -1082,7 +1097,12 @@ func TestParseErrors(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: map[string]string{}, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "test",
Configs: tt.configs,
DefaultTags: map[string]string{},
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
_, err := parser.ParseLine(tt.input) _, err := parser.ParseLine(tt.input)
@ -1150,12 +1170,17 @@ func TestEmptySelection(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, DefaultTags: map[string]string{}, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "test",
Configs: tt.configs,
DefaultTags: map[string]string{},
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
_, err := parser.Parse([]byte(tt.input)) _, err := parser.Parse([]byte(tt.input))
require.Error(t, err) require.Error(t, err)
require.Equal(t, "cannot parse with empty selection node", err.Error()) require.Equal(t, err.Error(), "cannot parse with empty selection node")
}) })
} }
} }
@ -1218,7 +1243,13 @@ func TestEmptySelectionAllowed(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := &Parser{Configs: tt.configs, AllowEmptySelection: true, DefaultTags: map[string]string{}, Log: testutil.Logger{Name: "parsers.xml"}} parser := &Parser{
DefaultMetricName: "xml",
Configs: tt.configs,
AllowEmptySelection: true,
DefaultTags: map[string]string{},
Log: testutil.Logger{Name: "parsers.xml"},
}
require.NoError(t, parser.Init()) require.NoError(t, parser.Init())
_, err := parser.Parse([]byte(tt.input)) _, err := parser.Parse([]byte(tt.input))
@ -1277,7 +1308,6 @@ func TestTestCases(t *testing.T) {
filename := filepath.FromSlash(tt.filename) filename := filepath.FromSlash(tt.filename)
cfg, header, err := loadTestConfiguration(filename) cfg, header, err := loadTestConfiguration(filename)
require.NoError(t, err) require.NoError(t, err)
cfg.MetricDefaultName = "xml"
// Load the xml-content // Load the xml-content
input, err := testutil.ParseRawLinesFrom(header, "File:") input, err := testutil.ParseRawLinesFrom(header, "File:")
@ -1315,7 +1345,12 @@ func TestTestCases(t *testing.T) {
expectedErrors, _ := testutil.ParseRawLinesFrom(header, "Expected Error:") expectedErrors, _ := testutil.ParseRawLinesFrom(header, "Expected Error:")
// Setup the parser and run it. // Setup the parser and run it.
metricName := "xml"
if fileformat != "" {
metricName = fileformat
}
parser := &Parser{ parser := &Parser{
DefaultMetricName: metricName,
Format: fileformat, Format: fileformat,
ProtobufMessageDef: pbmsgdef, ProtobufMessageDef: pbmsgdef,
ProtobufMessageType: pbmsgtype, ProtobufMessageType: pbmsgtype,
@ -1340,6 +1375,7 @@ func TestTestCases(t *testing.T) {
func TestProtobufImporting(t *testing.T) { func TestProtobufImporting(t *testing.T) {
// Setup the parser and run it. // Setup the parser and run it.
parser := &Parser{ parser := &Parser{
DefaultMetricName: "xpath_protobuf",
Format: "xpath_protobuf", Format: "xpath_protobuf",
ProtobufMessageDef: "person.proto", ProtobufMessageDef: "person.proto",
ProtobufMessageType: "importtest.Person", ProtobufMessageType: "importtest.Person",