fix: fix handling of imports in protocol-buffer definitions (#10798)

config/config.go

@@ -1624,6 +1624,7 @@ func (c *Config) getParserConfig(name string, tbl *ast.Table) (*parsers.Config,
 	if choice.Contains(pc.DataFormat, []string{"xml", "xpath_json", "xpath_msgpack", "xpath_protobuf"}) {
 		c.getFieldString(tbl, "xpath_protobuf_file", &pc.XPathProtobufFile)
 		c.getFieldString(tbl, "xpath_protobuf_type", &pc.XPathProtobufType)
+		c.getFieldStringSlice(tbl, "xpath_protobuf_import_paths", &pc.XPathProtobufImportPaths)
 		c.getFieldBool(tbl, "xpath_print_document", &pc.XPathPrintDocument)
 
 		// Determine the actual xpath configuration tables
@@ -1852,7 +1853,7 @@ func (c *Config) missingTomlField(_ reflect.Type, key string) error {
 		"tagdrop", "tagexclude", "taginclude", "tagpass", "tags", "template", "templates",
 		"value_field_name", "wavefront_source_override", "wavefront_use_strict", "wavefront_disable_prefix_conversion",
 		"xml", "xpath", "xpath_json", "xpath_msgpack", "xpath_protobuf", "xpath_print_document",
-		"xpath_protobuf_file", "xpath_protobuf_type":
+		"xpath_protobuf_file", "xpath_protobuf_type", "xpath_protobuf_import_paths":
 
 		// ignore fields that are common to all plugins.
 	default:

plugins/parsers/registry.go

@@ -184,10 +184,11 @@ type Config struct {
 	ValueFieldName string `toml:"value_field_name"`
 
 	// XPath configuration
-	XPathPrintDocument bool   `toml:"xpath_print_document"`
-	XPathProtobufFile  string `toml:"xpath_protobuf_file"`
-	XPathProtobufType  string `toml:"xpath_protobuf_type"`
-	XPathConfig        []XPathConfig
+	XPathPrintDocument       bool     `toml:"xpath_print_document"`
+	XPathProtobufFile        string   `toml:"xpath_protobuf_file"`
+	XPathProtobufType        string   `toml:"xpath_protobuf_type"`
+	XPathProtobufImportPaths []string `toml:"xpath_protobuf_import_paths"`
+	XPathConfig              []XPathConfig
 
 	// JSONPath configuration
 	JSONV2Config []JSONV2Config `toml:"json_v2"`
@@ -280,6 +281,7 @@ func NewParser(config *Config) (Parser, error) {
 			Format:              config.DataFormat,
 			ProtobufMessageDef:  config.XPathProtobufFile,
 			ProtobufMessageType: config.XPathProtobufType,
+			ProtobufImportPaths: config.XPathProtobufImportPaths,
 			PrintDocument:       config.XPathPrintDocument,
 			DefaultTags:         config.DefaultTags,
 			Configs:             NewXPathParserConfigs(config.MetricName, config.XPathConfig),

plugins/parsers/xpath/README.md

@@ -13,11 +13,51 @@ For supported XPath functions check [the underlying XPath library][xpath lib].
 | [Extensible Markup Language (XML)][xml] | `"xml"`               |         |
 | [JSON][json]                            | `"xpath_json"`        |         |
 | [MessagePack][msgpack]                  | `"xpath_msgpack"`     |         |
-| [Protocol buffers][protobuf]            | `"xpath_protobuf"`    | [see additional parameters](#protocol-buffers-additional-settings)|
+| [Protocol-buffers][protobuf]            | `"xpath_protobuf"`    | [see additional parameters](#protocol-buffers-additional-settings)|
 
-### Protocol buffers additional settings
+### Protocol-buffers additional settings
 
-For using the protocol-buffer format you need to specify a protocol buffer definition file (`.proto`) in `xpath_protobuf_file`, Furthermore, you need to specify which message type you want to use via `xpath_protobuf_type`.
+For using the protocol-buffer format you need to specify additional (*mandatory*) properties for the parser. Those options are described here.
+
+#### `xpath_protobuf_file` (mandatory)
+
+Use this option to specify the name of the protocol-buffer definition file (`.proto`).
+
+#### `xpath_protobuf_type` (mandatory)
+
+This option contains the top-level message file to use for deserializing the data to be parsed. Usually, this is constructed from the `package` name in the protocol-buffer definition file and the `message` name as `<package name>.<message name>`.
+
+#### `xpath_protobuf_import_paths` (optional)
+
+In case you import other protocol-buffer definitions within your `.proto` file (i.e. you use the `import` statement) you can use this option to specify paths to search for the imported definition file(s). By default the imports are only searched in `.` which is the current-working-directory, i.e. usually the directory you are in when starting telegraf.
+
+Imagine you do have multiple protocol-buffer definitions (e.g. `A.proto`, `B.proto` and `C.proto`) in a directory (e.g. `/data/my_proto_files`) where your top-level file (e.g. `A.proto`) imports at least one other definition
+
+```protobuf
+syntax = "proto3";
+
+package foo;
+
+import "B.proto";
+
+message Measurement {
+    ...
+}
+```
+
+You should use the following setting
+
+```toml
+[[inputs.file]]
+  files = ["example.dat"]
+
+  data_format = "xpath_protobuf"
+  xpath_protobuf_file = "A.proto"
+  xpath_protobuf_type = "foo.Measurement"
+  xpath_protobuf_import_paths = [".", "/data/my_proto_files"]
+
+  ...
+```
 
 ## Configuration (explicit)
 
@@ -33,14 +73,16 @@ In this configuration mode, you explicitly specify the field and tags you want t
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
   data_format = "xml"
 
-  ## PROTOCOL BUFFER definitions
-  ## Protocol buffer definition file
+  ## PROTOCOL-BUFFER definitions
+  ## Protocol-buffer definition file
   # xpath_protobuf_file = "sparkplug_b.proto"
-  ## Name of the protocol buffer message type to use in a fully qualified form.
-  # xpath_protobuf_type = ""org.eclipse.tahu.protobuf.Payload""
+  ## Name of the protocol-buffer message type to use in a fully qualified form.
+  # xpath_protobuf_type = "org.eclipse.tahu.protobuf.Payload"
+  ## List of paths to use when looking up imported protocol-buffer definition files.
+  # xpath_protobuf_import_paths = ["."]
 
   ## Print the internal XML document when in debug logging mode.
-  ## This is especially useful when using the parser with non-XML formats like protocol buffers
+  ## This is especially useful when using the parser with non-XML formats like protocol-buffers
   ## to get an idea on the expression necessary to derive fields etc.
   # xpath_print_document = false
 
@@ -97,13 +139,16 @@ metric.
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
   data_format = "xml"
 
-  ## Name of the protocol buffer type to use.
-  ## This is only relevant when parsing protocol buffers and must contain the fully qualified
-  ## name of the type e.g. "org.eclipse.tahu.protobuf.Payload".
-  # xpath_protobuf_type = ""
+  ## PROTOCOL-BUFFER definitions
+  ## Protocol-buffer definition file
+  # xpath_protobuf_file = "sparkplug_b.proto"
+  ## Name of the protocol-buffer message type to use in a fully qualified form.
+  # xpath_protobuf_type = "org.eclipse.tahu.protobuf.Payload"
+  ## List of paths to use when looking up imported protocol-buffer definition files.
+  # xpath_protobuf_import_paths = ["."]
 
   ## Print the internal XML document when in debug logging mode.
-  ## This is especially useful when using the parser with non-XML formats like protocol buffers
+  ## This is especially useful when using the parser with non-XML formats like protocol-buffers
   ## to get an idea on the expression necessary to derive fields etc.
   # xpath_print_document = false
 

plugins/parsers/xpath/parser.go

@@ -27,6 +27,7 @@ type Parser struct {
 	Format              string
 	ProtobufMessageDef  string
 	ProtobufMessageType string
+	ProtobufImportPaths []string
 	PrintDocument       bool
 	Configs             []Config
 	DefaultTags         map[string]string
@@ -68,6 +69,7 @@ func (p *Parser) Init() error {
 		pbdoc := protobufDocument{
 			MessageDefinition: p.ProtobufMessageDef,
 			MessageType:       p.ProtobufMessageType,
+			ImportPaths:       p.ProtobufImportPaths,
 			Log:               p.Log,
 		}
 		if err := pbdoc.Init(); err != nil {

plugins/parsers/xpath/parser_test.go

@@ -1269,6 +1269,19 @@ func TestTestCases(t *testing.T) {
 	}
 }
 
+func TestProtobufImporting(t *testing.T) {
+	// Setup the parser and run it.
+	parser := &Parser{
+		Format:              "xpath_protobuf",
+		ProtobufMessageDef:  "person.proto",
+		ProtobufMessageType: "importtest.Person",
+		ProtobufImportPaths: []string{"testcases/protos"},
+		Configs:             []Config{},
+		Log:                 testutil.Logger{Name: "parsers.protobuf"},
+	}
+	require.NoError(t, parser.Init())
+}
+
 func loadTestConfiguration(filename string) (*Config, []string, error) {
 	buf, err := os.ReadFile(filename)
 	if err != nil {

plugins/parsers/xpath/protocolbuffer_document.go

@@ -1,7 +1,6 @@
 package xpath
 
 import (
-	"errors"
 	"fmt"
 	"sort"
 	"strings"
@@ -11,9 +10,9 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/reflect/protodesc"
 	"google.golang.org/protobuf/reflect/protoreflect"
-	"google.golang.org/protobuf/reflect/protoregistry"
 	"google.golang.org/protobuf/types/dynamicpb"
 
+	"github.com/jhump/protoreflect/desc"
 	"github.com/jhump/protoreflect/desc/protoparse"
 
 	path "github.com/antchfx/xpath"
@@ -23,6 +22,7 @@ import (
 type protobufDocument struct {
 	MessageDefinition string
 	MessageType       string
+	ImportPaths       []string
 	Log               telegraf.Logger
 	msg               *dynamicpb.Message
 }
@@ -37,7 +37,10 @@ func (d *protobufDocument) Init() error {
 	}
 
 	// Load the file descriptors from the given protocol-buffer definition
-	parser := protoparse.Parser{}
+	parser := protoparse.Parser{
+		ImportPaths:      d.ImportPaths,
+		InferImportPaths: true,
+	}
 	fds, err := parser.ParseFiles(d.MessageDefinition)
 	if err != nil {
 		return fmt.Errorf("parsing protocol-buffer definition in %q failed: %v", d.MessageDefinition, err)
@@ -47,35 +50,19 @@ func (d *protobufDocument) Init() error {
 	}
 
 	// Register all definitions in the file in the global registry
-	for _, fd := range fds {
-		if fd == nil {
-			continue
-		}
-		fileDescProto := fd.AsFileDescriptorProto()
-		fileDesc, err := protodesc.NewFile(fileDescProto, nil)
-		if err != nil {
-			return fmt.Errorf("creating file descriptor from proto failed: %v", err)
-		}
-		if _, err := protoregistry.GlobalFiles.FindFileByPath(fileDesc.Path()); !errors.Is(err, protoregistry.NotFound) {
-			if err != nil {
-				return fmt.Errorf("searching for file %q in registry failed: %v", fileDesc.Path(), err)
-			}
-			d.Log.Warnf("Protocol buffer with path %q already registered. Skipping...", fileDesc.Path())
-			continue
-		}
-		if err := protoregistry.GlobalFiles.RegisterFile(fileDesc); err != nil {
-			return fmt.Errorf("registering file descriptor %q failed: %v", fileDesc.Package(), err)
-		}
+	registry, err := protodesc.NewFiles(desc.ToFileDescriptorSet(fds...))
+	if err != nil {
+		return fmt.Errorf("constructing registry failed: %v", err)
 	}
 
 	// Lookup given type in the loaded file descriptors
 	msgFullName := protoreflect.FullName(d.MessageType)
-	desc, err := protoregistry.GlobalFiles.FindDescriptorByName(msgFullName)
+	descriptor, err := registry.FindDescriptorByName(msgFullName)
 	if err != nil {
 		d.Log.Infof("Could not find %q... Known messages:", msgFullName)
 
 		var known []string
-		protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool {
+		registry.RangeFiles(func(fd protoreflect.FileDescriptor) bool {
 			name := strings.TrimSpace(string(fd.FullName()))
 			if name != "" {
 				known = append(known, name)
@@ -90,9 +77,9 @@ func (d *protobufDocument) Init() error {
 	}
 
 	// Get a prototypical message for later use
-	msgDesc, ok := desc.(protoreflect.MessageDescriptor)
+	msgDesc, ok := descriptor.(protoreflect.MessageDescriptor)
 	if !ok {
-		return fmt.Errorf("%q is not a message descriptor (%T)", msgFullName, desc)
+		return fmt.Errorf("%q is not a message descriptor (%T)", msgFullName, descriptor)
 	}
 
 	d.msg = dynamicpb.NewMessage(msgDesc)

plugins/parsers/xpath/testcases/addressbook.conf

@@ -4,7 +4,7 @@
 #   testcases/addressbook.dat xpath_protobuf
 #
 # Protobuf:
-#   testcases/addressbook.proto addressbook.AddressBook
+#   testcases/protos/addressbook.proto addressbook.AddressBook
 #
 # Expected Output:
 # addresses,id=101,name=John\ Doe age=42i,email="john@example.com" 1621430181000000000

plugins/parsers/xpath/testcases/protos/person.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+package importtest;
+
+import "phonenumber.proto";
+
+message Person {
+  optional string name = 1;
+  optional int32 id = 2;
+  optional string email = 3;
+
+  repeated PhoneNumber phones = 4;
+}

plugins/parsers/xpath/testcases/protos/phonenumber.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+package importtest;
+
+enum PhoneType {
+MOBILE = 0;
+HOME = 1;
+WORK = 2;
+}
+
+message PhoneNumber {
+    optional string number = 1;
+    optional PhoneType type = 2;
+}
+