diff --git a/config/config.go b/config/config.go index 39cb8c4d3..97772a6ca 100644 --- a/config/config.go +++ b/config/config.go @@ -1624,6 +1624,7 @@ func (c *Config) getParserConfig(name string, tbl *ast.Table) (*parsers.Config, if choice.Contains(pc.DataFormat, []string{"xml", "xpath_json", "xpath_msgpack", "xpath_protobuf"}) { c.getFieldString(tbl, "xpath_protobuf_file", &pc.XPathProtobufFile) c.getFieldString(tbl, "xpath_protobuf_type", &pc.XPathProtobufType) + c.getFieldStringSlice(tbl, "xpath_protobuf_import_paths", &pc.XPathProtobufImportPaths) c.getFieldBool(tbl, "xpath_print_document", &pc.XPathPrintDocument) // Determine the actual xpath configuration tables @@ -1852,7 +1853,7 @@ func (c *Config) missingTomlField(_ reflect.Type, key string) error { "tagdrop", "tagexclude", "taginclude", "tagpass", "tags", "template", "templates", "value_field_name", "wavefront_source_override", "wavefront_use_strict", "wavefront_disable_prefix_conversion", "xml", "xpath", "xpath_json", "xpath_msgpack", "xpath_protobuf", "xpath_print_document", - "xpath_protobuf_file", "xpath_protobuf_type": + "xpath_protobuf_file", "xpath_protobuf_type", "xpath_protobuf_import_paths": // ignore fields that are common to all plugins. default: diff --git a/plugins/parsers/registry.go b/plugins/parsers/registry.go index 5514b818c..b56b6d094 100644 --- a/plugins/parsers/registry.go +++ b/plugins/parsers/registry.go @@ -184,10 +184,11 @@ type Config struct { ValueFieldName string `toml:"value_field_name"` // XPath configuration - XPathPrintDocument bool `toml:"xpath_print_document"` - XPathProtobufFile string `toml:"xpath_protobuf_file"` - XPathProtobufType string `toml:"xpath_protobuf_type"` - XPathConfig []XPathConfig + XPathPrintDocument bool `toml:"xpath_print_document"` + XPathProtobufFile string `toml:"xpath_protobuf_file"` + XPathProtobufType string `toml:"xpath_protobuf_type"` + XPathProtobufImportPaths []string `toml:"xpath_protobuf_import_paths"` + XPathConfig []XPathConfig // JSONPath configuration JSONV2Config []JSONV2Config `toml:"json_v2"` @@ -280,6 +281,7 @@ func NewParser(config *Config) (Parser, error) { Format: config.DataFormat, ProtobufMessageDef: config.XPathProtobufFile, ProtobufMessageType: config.XPathProtobufType, + ProtobufImportPaths: config.XPathProtobufImportPaths, PrintDocument: config.XPathPrintDocument, DefaultTags: config.DefaultTags, Configs: NewXPathParserConfigs(config.MetricName, config.XPathConfig), diff --git a/plugins/parsers/xpath/README.md b/plugins/parsers/xpath/README.md index 04b76d90e..1cca277c9 100644 --- a/plugins/parsers/xpath/README.md +++ b/plugins/parsers/xpath/README.md @@ -13,11 +13,51 @@ For supported XPath functions check [the underlying XPath library][xpath lib]. | [Extensible Markup Language (XML)][xml] | `"xml"` | | | [JSON][json] | `"xpath_json"` | | | [MessagePack][msgpack] | `"xpath_msgpack"` | | -| [Protocol buffers][protobuf] | `"xpath_protobuf"` | [see additional parameters](#protocol-buffers-additional-settings)| +| [Protocol-buffers][protobuf] | `"xpath_protobuf"` | [see additional parameters](#protocol-buffers-additional-settings)| -### Protocol buffers additional settings +### Protocol-buffers additional settings -For using the protocol-buffer format you need to specify a protocol buffer definition file (`.proto`) in `xpath_protobuf_file`, Furthermore, you need to specify which message type you want to use via `xpath_protobuf_type`. +For using the protocol-buffer format you need to specify additional (*mandatory*) properties for the parser. Those options are described here. + +#### `xpath_protobuf_file` (mandatory) + +Use this option to specify the name of the protocol-buffer definition file (`.proto`). + +#### `xpath_protobuf_type` (mandatory) + +This option contains the top-level message file to use for deserializing the data to be parsed. Usually, this is constructed from the `package` name in the protocol-buffer definition file and the `message` name as `.`. + +#### `xpath_protobuf_import_paths` (optional) + +In case you import other protocol-buffer definitions within your `.proto` file (i.e. you use the `import` statement) you can use this option to specify paths to search for the imported definition file(s). By default the imports are only searched in `.` which is the current-working-directory, i.e. usually the directory you are in when starting telegraf. + +Imagine you do have multiple protocol-buffer definitions (e.g. `A.proto`, `B.proto` and `C.proto`) in a directory (e.g. `/data/my_proto_files`) where your top-level file (e.g. `A.proto`) imports at least one other definition + +```protobuf +syntax = "proto3"; + +package foo; + +import "B.proto"; + +message Measurement { + ... +} +``` + +You should use the following setting + +```toml +[[inputs.file]] + files = ["example.dat"] + + data_format = "xpath_protobuf" + xpath_protobuf_file = "A.proto" + xpath_protobuf_type = "foo.Measurement" + xpath_protobuf_import_paths = [".", "/data/my_proto_files"] + + ... +``` ## Configuration (explicit) @@ -33,14 +73,16 @@ In this configuration mode, you explicitly specify the field and tags you want t ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md data_format = "xml" - ## PROTOCOL BUFFER definitions - ## Protocol buffer definition file + ## PROTOCOL-BUFFER definitions + ## Protocol-buffer definition file # xpath_protobuf_file = "sparkplug_b.proto" - ## Name of the protocol buffer message type to use in a fully qualified form. - # xpath_protobuf_type = ""org.eclipse.tahu.protobuf.Payload"" + ## Name of the protocol-buffer message type to use in a fully qualified form. + # xpath_protobuf_type = "org.eclipse.tahu.protobuf.Payload" + ## List of paths to use when looking up imported protocol-buffer definition files. + # xpath_protobuf_import_paths = ["."] ## Print the internal XML document when in debug logging mode. - ## This is especially useful when using the parser with non-XML formats like protocol buffers + ## This is especially useful when using the parser with non-XML formats like protocol-buffers ## to get an idea on the expression necessary to derive fields etc. # xpath_print_document = false @@ -97,13 +139,16 @@ metric. ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md data_format = "xml" - ## Name of the protocol buffer type to use. - ## This is only relevant when parsing protocol buffers and must contain the fully qualified - ## name of the type e.g. "org.eclipse.tahu.protobuf.Payload". - # xpath_protobuf_type = "" + ## PROTOCOL-BUFFER definitions + ## Protocol-buffer definition file + # xpath_protobuf_file = "sparkplug_b.proto" + ## Name of the protocol-buffer message type to use in a fully qualified form. + # xpath_protobuf_type = "org.eclipse.tahu.protobuf.Payload" + ## List of paths to use when looking up imported protocol-buffer definition files. + # xpath_protobuf_import_paths = ["."] ## Print the internal XML document when in debug logging mode. - ## This is especially useful when using the parser with non-XML formats like protocol buffers + ## This is especially useful when using the parser with non-XML formats like protocol-buffers ## to get an idea on the expression necessary to derive fields etc. # xpath_print_document = false diff --git a/plugins/parsers/xpath/parser.go b/plugins/parsers/xpath/parser.go index 4f8ac5e3a..53ff4ca7c 100644 --- a/plugins/parsers/xpath/parser.go +++ b/plugins/parsers/xpath/parser.go @@ -27,6 +27,7 @@ type Parser struct { Format string ProtobufMessageDef string ProtobufMessageType string + ProtobufImportPaths []string PrintDocument bool Configs []Config DefaultTags map[string]string @@ -68,6 +69,7 @@ func (p *Parser) Init() error { pbdoc := protobufDocument{ MessageDefinition: p.ProtobufMessageDef, MessageType: p.ProtobufMessageType, + ImportPaths: p.ProtobufImportPaths, Log: p.Log, } if err := pbdoc.Init(); err != nil { diff --git a/plugins/parsers/xpath/parser_test.go b/plugins/parsers/xpath/parser_test.go index 9f8902fff..1e5ffeb8d 100644 --- a/plugins/parsers/xpath/parser_test.go +++ b/plugins/parsers/xpath/parser_test.go @@ -1269,6 +1269,19 @@ func TestTestCases(t *testing.T) { } } +func TestProtobufImporting(t *testing.T) { + // Setup the parser and run it. + parser := &Parser{ + Format: "xpath_protobuf", + ProtobufMessageDef: "person.proto", + ProtobufMessageType: "importtest.Person", + ProtobufImportPaths: []string{"testcases/protos"}, + Configs: []Config{}, + Log: testutil.Logger{Name: "parsers.protobuf"}, + } + require.NoError(t, parser.Init()) +} + func loadTestConfiguration(filename string) (*Config, []string, error) { buf, err := os.ReadFile(filename) if err != nil { diff --git a/plugins/parsers/xpath/protocolbuffer_document.go b/plugins/parsers/xpath/protocolbuffer_document.go index 55e96e463..6d6d4554b 100644 --- a/plugins/parsers/xpath/protocolbuffer_document.go +++ b/plugins/parsers/xpath/protocolbuffer_document.go @@ -1,7 +1,6 @@ package xpath import ( - "errors" "fmt" "sort" "strings" @@ -11,9 +10,9 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoreflect" - "google.golang.org/protobuf/reflect/protoregistry" "google.golang.org/protobuf/types/dynamicpb" + "github.com/jhump/protoreflect/desc" "github.com/jhump/protoreflect/desc/protoparse" path "github.com/antchfx/xpath" @@ -23,6 +22,7 @@ import ( type protobufDocument struct { MessageDefinition string MessageType string + ImportPaths []string Log telegraf.Logger msg *dynamicpb.Message } @@ -37,7 +37,10 @@ func (d *protobufDocument) Init() error { } // Load the file descriptors from the given protocol-buffer definition - parser := protoparse.Parser{} + parser := protoparse.Parser{ + ImportPaths: d.ImportPaths, + InferImportPaths: true, + } fds, err := parser.ParseFiles(d.MessageDefinition) if err != nil { return fmt.Errorf("parsing protocol-buffer definition in %q failed: %v", d.MessageDefinition, err) @@ -47,35 +50,19 @@ func (d *protobufDocument) Init() error { } // Register all definitions in the file in the global registry - for _, fd := range fds { - if fd == nil { - continue - } - fileDescProto := fd.AsFileDescriptorProto() - fileDesc, err := protodesc.NewFile(fileDescProto, nil) - if err != nil { - return fmt.Errorf("creating file descriptor from proto failed: %v", err) - } - if _, err := protoregistry.GlobalFiles.FindFileByPath(fileDesc.Path()); !errors.Is(err, protoregistry.NotFound) { - if err != nil { - return fmt.Errorf("searching for file %q in registry failed: %v", fileDesc.Path(), err) - } - d.Log.Warnf("Protocol buffer with path %q already registered. Skipping...", fileDesc.Path()) - continue - } - if err := protoregistry.GlobalFiles.RegisterFile(fileDesc); err != nil { - return fmt.Errorf("registering file descriptor %q failed: %v", fileDesc.Package(), err) - } + registry, err := protodesc.NewFiles(desc.ToFileDescriptorSet(fds...)) + if err != nil { + return fmt.Errorf("constructing registry failed: %v", err) } // Lookup given type in the loaded file descriptors msgFullName := protoreflect.FullName(d.MessageType) - desc, err := protoregistry.GlobalFiles.FindDescriptorByName(msgFullName) + descriptor, err := registry.FindDescriptorByName(msgFullName) if err != nil { d.Log.Infof("Could not find %q... Known messages:", msgFullName) var known []string - protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool { + registry.RangeFiles(func(fd protoreflect.FileDescriptor) bool { name := strings.TrimSpace(string(fd.FullName())) if name != "" { known = append(known, name) @@ -90,9 +77,9 @@ func (d *protobufDocument) Init() error { } // Get a prototypical message for later use - msgDesc, ok := desc.(protoreflect.MessageDescriptor) + msgDesc, ok := descriptor.(protoreflect.MessageDescriptor) if !ok { - return fmt.Errorf("%q is not a message descriptor (%T)", msgFullName, desc) + return fmt.Errorf("%q is not a message descriptor (%T)", msgFullName, descriptor) } d.msg = dynamicpb.NewMessage(msgDesc) diff --git a/plugins/parsers/xpath/testcases/addressbook.conf b/plugins/parsers/xpath/testcases/addressbook.conf index eeca8921d..f6fcf657f 100644 --- a/plugins/parsers/xpath/testcases/addressbook.conf +++ b/plugins/parsers/xpath/testcases/addressbook.conf @@ -4,7 +4,7 @@ # testcases/addressbook.dat xpath_protobuf # # Protobuf: -# testcases/addressbook.proto addressbook.AddressBook +# testcases/protos/addressbook.proto addressbook.AddressBook # # Expected Output: # addresses,id=101,name=John\ Doe age=42i,email="john@example.com" 1621430181000000000 diff --git a/plugins/parsers/xpath/testcases/addressbook.proto b/plugins/parsers/xpath/testcases/protos/addressbook.proto similarity index 100% rename from plugins/parsers/xpath/testcases/addressbook.proto rename to plugins/parsers/xpath/testcases/protos/addressbook.proto diff --git a/plugins/parsers/xpath/testcases/protos/person.proto b/plugins/parsers/xpath/testcases/protos/person.proto new file mode 100644 index 000000000..561f6f271 --- /dev/null +++ b/plugins/parsers/xpath/testcases/protos/person.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package importtest; + +import "phonenumber.proto"; + +message Person { + optional string name = 1; + optional int32 id = 2; + optional string email = 3; + + repeated PhoneNumber phones = 4; +} diff --git a/plugins/parsers/xpath/testcases/protos/phonenumber.proto b/plugins/parsers/xpath/testcases/protos/phonenumber.proto new file mode 100644 index 000000000..8590b3ef7 --- /dev/null +++ b/plugins/parsers/xpath/testcases/protos/phonenumber.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package importtest; + +enum PhoneType { +MOBILE = 0; +HOME = 1; +WORK = 2; +} + +message PhoneNumber { + optional string number = 1; + optional PhoneType type = 2; +} +