From e3114357530ddd2c4b24c1cc0c463d0c1f58011f Mon Sep 17 00:00:00 2001 From: Sven Rebhan <36194019+srebhan@users.noreply.github.com> Date: Mon, 21 Nov 2022 20:56:02 +0100 Subject: [PATCH] feat(parsers.xpath): Allow to specify byte-array fields to encode in HEX (#11999) --- .../testcases/powerdns/dnsmessage.proto | 184 ++++++++++++++++++ .../testcases/powerdns/expected.out | 1 + .../testcases/powerdns/powerdns_message.bin | Bin 0 -> 103 bytes .../testcases/powerdns/sequence.json | 5 + .../testcases/powerdns/telegraf.conf | 17 ++ plugins/parsers/temporary/xpath/types.go | 5 + plugins/parsers/xpath/README.md | 6 + plugins/parsers/xpath/parser.go | 13 ++ .../protobuf_powerdns_hex/expected.out | 1 + .../powerdns_message.bin | Bin 0 -> 103 bytes .../powerdns_message.proto | 184 ++++++++++++++++++ .../protobuf_powerdns_hex/telegraf.conf | 14 ++ 12 files changed, 430 insertions(+) create mode 100644 plugins/inputs/socket_listener/testcases/powerdns/dnsmessage.proto create mode 100644 plugins/inputs/socket_listener/testcases/powerdns/expected.out create mode 100644 plugins/inputs/socket_listener/testcases/powerdns/powerdns_message.bin create mode 100644 plugins/inputs/socket_listener/testcases/powerdns/sequence.json create mode 100644 plugins/inputs/socket_listener/testcases/powerdns/telegraf.conf create mode 100644 plugins/parsers/xpath/testcases/protobuf_powerdns_hex/expected.out create mode 100644 plugins/parsers/xpath/testcases/protobuf_powerdns_hex/powerdns_message.bin create mode 100644 plugins/parsers/xpath/testcases/protobuf_powerdns_hex/powerdns_message.proto create mode 100644 plugins/parsers/xpath/testcases/protobuf_powerdns_hex/telegraf.conf diff --git a/plugins/inputs/socket_listener/testcases/powerdns/dnsmessage.proto b/plugins/inputs/socket_listener/testcases/powerdns/dnsmessage.proto new file mode 100644 index 000000000..99355dfa3 --- /dev/null +++ b/plugins/inputs/socket_listener/testcases/powerdns/dnsmessage.proto @@ -0,0 +1,184 @@ +/* + * This file describes the message format used by the protobuf logging feature in PowerDNS and dnsdist. + * + * MIT License + * + * Copyright (c) 2016-now PowerDNS.COM B.V. and its contributors. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +syntax = "proto2"; + +message PBDNSMessage { + enum Type { + DNSQueryType = 1; // Query received by the service + DNSResponseType = 2; // Response returned by the service + DNSOutgoingQueryType = 3; // Query sent out by the service to a remote server + DNSIncomingResponseType = 4; // Response returned by the remote server + } + enum SocketFamily { + INET = 1; // IPv4 (RFC 791) + INET6 = 2; // IPv6 (RFC 2460) + } + enum SocketProtocol { + UDP = 1; // User Datagram Protocol (RFC 768) + TCP = 2; // Transmission Control Protocol (RFC 793) + DOT = 3; // DNS over TLS (RFC 7858) + DOH = 4; // DNS over HTTPS (RFC 8484) + DNSCryptUDP = 5; // DNSCrypt over UDP (https://dnscrypt.info/protocol) + DNSCryptTCP = 6; // DNSCrypt over TCP (https://dnscrypt.info/protocol) + } + enum PolicyType { + UNKNOWN = 1; // No RPZ policy applied, or unknown type + QNAME = 2; // Policy matched on the QName + CLIENTIP = 3; // Policy matched on the client IP + RESPONSEIP = 4; // Policy matched on one of the IPs contained in the answer + NSDNAME = 5; // Policy matched on the name of one nameserver involved + NSIP = 6; // Policy matched on the IP of one nameserver involved + } + enum PolicyKind { + NoAction = 1; // No action taken + Drop = 2; // https://tools.ietf.org/html/draft-vixie-dns-rpz-04 3.4 + NXDOMAIN = 3; // https://tools.ietf.org/html/draft-vixie-dns-rpz-04 3.1 + NODATA = 4; // https://tools.ietf.org/html/draft-vixie-dns-rpz-04 3.2 + Truncate= 5; // https://tools.ietf.org/html/draft-vixie-dns-rpz-04 3.5 + Custom = 6; // https://tools.ietf.org/html/draft-vixie-dns-rpz-04 3.6 + } + enum VState { + Indeterminate = 1; + Insecure = 2; + Secure = 3; + BogusNoValidDNSKEY = 4; + BogusInvalidDenial = 5; + BogusUnableToGetDSs = 6; + BogusUnableToGetDNSKEYs = 7; + BogusSelfSignedDS = 8; + BogusNoRRSIG = 9; + BogusNoValidRRSIG = 10; + BogusMissingNegativeIndication = 11; + BogusSignatureNotYetValid = 12; + BogusSignatureExpired = 13; + BogusUnsupportedDNSKEYAlgo = 14; + BogusUnsupportedDSDigestType = 15; + BogusNoZoneKeyBitSet = 16; + BogusRevokedDNSKEY = 17; + BogusInvalidDNSKEYProtocol = 18; + } + required Type type = 1; // Type of event + optional bytes messageId = 2; // UUID, shared by the query and the response + optional bytes serverIdentity = 3; // ID of the server emitting the protobuf message + optional SocketFamily socketFamily = 4; + optional SocketProtocol socketProtocol = 5; + optional bytes from = 6; // DNS requestor (client) as 4 (IPv4) or 16 (IPv6) raw bytes in network byte order + optional bytes to = 7; // DNS responder (server) as 4 (IPv4) or 16 (IPv6) raw bytes in network byte order + optional uint64 inBytes = 8; // Size of the query or response on the wire + optional uint32 timeSec = 9; // Time of message reception (seconds since epoch) + optional uint32 timeUsec = 10; // Time of message reception (additional micro-seconds) + optional uint32 id = 11; // ID of the query/response as found in the DNS header + + message DNSQuestion { + optional string qName = 1; // Fully qualified DNS name (with trailing dot) + optional uint32 qType = 2; // https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4 + optional uint32 qClass = 3; // Typically 1 (IN), see https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2 + } + optional DNSQuestion question = 12; // DNS query received from client + + message DNSResponse { + // See exportTypes in https://docs.powerdns.com/recursor/lua-config/protobuf.html#protobufServer + // for the list of supported resource record types. + message DNSRR { + optional string name = 1; // Fully qualified DNS name (with trailing dot) + optional uint32 type = 2; // https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4 + optional uint32 class = 3; // Typically 1 (IN), see https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2 + optional uint32 ttl = 4; // TTL in seconds + optional bytes rdata = 5; // raw address bytes in network byte order for A & AAAA; text representation for others, with fully qualified (trailing dot) domain names + optional bool udr = 6; // True if this is the first time this RR has been seen for this question + } + optional uint32 rcode = 1; // DNS Response code, or 65536 for a network error including a timeout + repeated DNSRR rrs = 2; // DNS resource records in response + optional string appliedPolicy = 3; // Filtering policy (RPZ or Lua) applied + repeated string tags = 4; // Additional tags applied + optional uint32 queryTimeSec = 5; // Time of the corresponding query reception (seconds since epoch) + optional uint32 queryTimeUsec = 6; // Time of the corresponding query reception (additional micro-seconds) + optional PolicyType appliedPolicyType = 7; // Type of the filtering policy (RPZ or Lua) applied + optional string appliedPolicyTrigger = 8; // The RPZ trigger + optional string appliedPolicyHit = 9; // The value (qname or IP) that caused the hit + optional PolicyKind appliedPolicyKind = 10; // The Kind (RPZ action) applied by the hit + optional VState validationState = 11; // The DNSSEC Validation State + } + + optional DNSResponse response = 13; + optional bytes originalRequestorSubnet = 14; // EDNS Client Subnet value (4 or 16 raw bytes in network byte order) + optional string requestorId = 15; // Username of the requestor + optional bytes initialRequestId = 16; // UUID of the incoming query that initiated this outgoing query or incoming response + optional bytes deviceId = 17; // Device ID of the requestor (could be mac address IP address or e.g. IMEI, format implementation dependent) + optional bool newlyObservedDomain = 18; // True if the domain has not been seen before + optional string deviceName = 19; // Device name of the requestor + optional uint32 fromPort = 20; // Source port of the DNS query (client) + optional uint32 toPort = 21; // Destination port of the DNS query (server) + + message MetaValue { + repeated string stringVal = 1; + repeated int64 intVal = 2; + } + + message Meta { + required string key = 1; // MUST be unique, so if you have multiple values they must be aggregated into on Meta + required MetaValue value = 2; + } + repeated Meta meta = 22; // Arbitrary meta-data - to be used in future rather than adding new fields all the time + + // The well known EventTrace event numbers + enum EventType { + // Range 0..99: Generic events + CustomEvent = 0; // A custom event + ReqRecv = 1; // A request was received + PCacheCheck = 2; // A packet cache check was initiated or completed; value: bool cacheHit + AnswerSent = 3; // An answer was sent to the client + + // Range 100: Recursor events + SyncRes = 100; // Recursor Syncres main function has started or completed; value: int rcode + LuaGetTag = 101; // Events below mark start or end of Lua hook calls; value: return value of hook + LuaGetTagFFI = 102; + LuaIPFilter = 103; + LuaPreRPZ = 104; + LuaPreResolve = 105; + LuaPreOutQuery = 106; + LuaPostResolve = 107; + LuaNoData = 108; + LuaNXDomain = 109; + LuaPostResolveFFI = 110; + } + + message Event { + required int64 ts = 1; // Timestamp in ns relative to time of creation of event trace data structure + required EventType event = 2; // Type of event + required bool start = 3; // true for "start" events, false for "completed" events + optional bool boolVal = 4; // Below are optional values associated with events + optional int64 intVal = 5; + optional string stringVal = 6; + optional bytes bytesVal = 7; + optional string custom = 8; // The name of the event for custom events + } + repeated Event trace = 23; +} + +message PBDNSMessageList { + repeated PBDNSMessage msg = 1; +} diff --git a/plugins/inputs/socket_listener/testcases/powerdns/expected.out b/plugins/inputs/socket_listener/testcases/powerdns/expected.out new file mode 100644 index 000000000..b9506d034 --- /dev/null +++ b/plugins/inputs/socket_listener/testcases/powerdns/expected.out @@ -0,0 +1 @@ +powerdns from="7f000001",fromPort=45729u,to="7f000001",toPort=53u,inBytes=48u,serverIdentity="xxxxxxxxxxxxxxxxxxx.com",messageId="943f90bea57a4eecbc5b0bea820a8aae",qName="ilse.nl.",qClass=1u,qType=1u,id=64100u,timeSec=1665050957u,timeUsec=500976u diff --git a/plugins/inputs/socket_listener/testcases/powerdns/powerdns_message.bin b/plugins/inputs/socket_listener/testcases/powerdns/powerdns_message.bin new file mode 100644 index 0000000000000000000000000000000000000000..1f5733842a2408b9d44e92e4eee962a9699e9344 GIT binary patch literal 103 zcmZQbXJBBo0+S8~9%nE8n#mUM;iO!| olP}Ckd|VuvImM}Zc{zFlj1r6s7#BWeTES>4C0>CE^pf*)0rJly{{R30 literal 0 HcmV?d00001 diff --git a/plugins/inputs/socket_listener/testcases/powerdns/sequence.json b/plugins/inputs/socket_listener/testcases/powerdns/sequence.json new file mode 100644 index 000000000..07770ce6c --- /dev/null +++ b/plugins/inputs/socket_listener/testcases/powerdns/sequence.json @@ -0,0 +1,5 @@ +[ + { + "file": "powerdns_message.bin" + } +] \ No newline at end of file diff --git a/plugins/inputs/socket_listener/testcases/powerdns/telegraf.conf b/plugins/inputs/socket_listener/testcases/powerdns/telegraf.conf new file mode 100644 index 000000000..7033235e1 --- /dev/null +++ b/plugins/inputs/socket_listener/testcases/powerdns/telegraf.conf @@ -0,0 +1,17 @@ +# Test with broken line protocol lines +[[inputs.socket_listener]] + service_address = "tcp://127.0.0.1:0" + splitting_strategy = "variable length" + splitting_length_field = {offset=0, bytes=2, endianness="be", header_length=2} + + data_format = "xpath_protobuf" + + xpath_native_types = true + xpath_protobuf_file = "dnsmessage.proto" + xpath_protobuf_type = "PBDNSMessage" + xpath_protobuf_import_paths = [".", "./testcases/powerdns"] + + [[inputs.socket_listener.xpath]] + metric_name = "'powerdns'" + fields_bytes_as_hex = ["from", "to", "messageId"] + field_selection = "descendant::*" diff --git a/plugins/parsers/temporary/xpath/types.go b/plugins/parsers/temporary/xpath/types.go index 16f77e64a..10f426a13 100644 --- a/plugins/parsers/temporary/xpath/types.go +++ b/plugins/parsers/temporary/xpath/types.go @@ -1,5 +1,7 @@ package xpath +import "github.com/influxdata/telegraf/filter" + // Config definition for backward compatibility ONLY. // We need this here to avoid cyclic dependencies. However, we need // to move this to plugins/parsers/xpath once we deprecate parser @@ -12,6 +14,7 @@ type Config struct { Tags map[string]string `toml:"tags"` Fields map[string]string `toml:"fields"` FieldsInt map[string]string `toml:"fields_int"` + FieldsHex []string `toml:"fields_bytes_as_hex"` FieldSelection string `toml:"field_selection"` FieldNameQuery string `toml:"field_name"` @@ -22,4 +25,6 @@ type Config struct { TagNameQuery string `toml:"tag_name"` TagValueQuery string `toml:"tag_value"` TagNameExpand bool `toml:"tag_name_expansion"` + + FieldsHexFilter filter.Filter } diff --git a/plugins/parsers/xpath/README.md b/plugins/parsers/xpath/README.md index b01ab8ec8..6768091d6 100644 --- a/plugins/parsers/xpath/README.md +++ b/plugins/parsers/xpath/README.md @@ -141,6 +141,12 @@ This is a list of known headers and the corresponding values for ## time format. If not specified, a "unix" timestamp (in seconds) is expected. # timestamp_format = "2006-01-02T15:04:05Z" + ## Optional: List of fields to convert to hex-strings if they are + ## containing byte-arrays. This might be the case for e.g. protocol-buffer + ## messages encoding data as byte-arrays. Wildcard patterns are allowed. + ## By default, all byte-array-fields are converted to string. + # fields_bytes_as_hex = [] + ## Tag definitions using the given XPath queries. [inputs.file.xpath.tags] name = "substring-after(Sensor/@name, ' ')" diff --git a/plugins/parsers/xpath/parser.go b/plugins/parsers/xpath/parser.go index db217d0f9..ae88db61a 100644 --- a/plugins/parsers/xpath/parser.go +++ b/plugins/parsers/xpath/parser.go @@ -1,6 +1,7 @@ package xpath import ( + "encoding/hex" "errors" "fmt" "strconv" @@ -12,6 +13,7 @@ import ( "github.com/doclambda/protobufquery" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/filter" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/models" @@ -129,6 +131,12 @@ func (p *Parser) Init() error { if config.TimestampFmt == "" { config.TimestampFmt = "unix" } + f, err := filter.Compile(config.FieldsHex) + if err != nil { + return fmt.Errorf("creating hex-fields filter failed: %w", err) + } + config.FieldsHexFilter = f + p.Configs[i] = config } @@ -406,6 +414,11 @@ func (p *Parser) parseQuery(starttime time.Time, doc, selected dataNode, config } } + if config.FieldsHexFilter != nil && config.FieldsHexFilter.Match(name) { + if b, ok := v.([]byte); ok { + v = hex.EncodeToString(b) + } + } fields[name] = v } } else { diff --git a/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/expected.out b/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/expected.out new file mode 100644 index 000000000..b9506d034 --- /dev/null +++ b/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/expected.out @@ -0,0 +1 @@ +powerdns from="7f000001",fromPort=45729u,to="7f000001",toPort=53u,inBytes=48u,serverIdentity="xxxxxxxxxxxxxxxxxxx.com",messageId="943f90bea57a4eecbc5b0bea820a8aae",qName="ilse.nl.",qClass=1u,qType=1u,id=64100u,timeSec=1665050957u,timeUsec=500976u diff --git a/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/powerdns_message.bin b/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/powerdns_message.bin new file mode 100644 index 0000000000000000000000000000000000000000..1f5733842a2408b9d44e92e4eee962a9699e9344 GIT binary patch literal 103 zcmZQbXJBBo0+S8~9%nE8n#mUM;iO!| olP}Ckd|VuvImM}Zc{zFlj1r6s7#BWeTES>4C0>CE^pf*)0rJly{{R30 literal 0 HcmV?d00001 diff --git a/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/powerdns_message.proto b/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/powerdns_message.proto new file mode 100644 index 000000000..99355dfa3 --- /dev/null +++ b/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/powerdns_message.proto @@ -0,0 +1,184 @@ +/* + * This file describes the message format used by the protobuf logging feature in PowerDNS and dnsdist. + * + * MIT License + * + * Copyright (c) 2016-now PowerDNS.COM B.V. and its contributors. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +syntax = "proto2"; + +message PBDNSMessage { + enum Type { + DNSQueryType = 1; // Query received by the service + DNSResponseType = 2; // Response returned by the service + DNSOutgoingQueryType = 3; // Query sent out by the service to a remote server + DNSIncomingResponseType = 4; // Response returned by the remote server + } + enum SocketFamily { + INET = 1; // IPv4 (RFC 791) + INET6 = 2; // IPv6 (RFC 2460) + } + enum SocketProtocol { + UDP = 1; // User Datagram Protocol (RFC 768) + TCP = 2; // Transmission Control Protocol (RFC 793) + DOT = 3; // DNS over TLS (RFC 7858) + DOH = 4; // DNS over HTTPS (RFC 8484) + DNSCryptUDP = 5; // DNSCrypt over UDP (https://dnscrypt.info/protocol) + DNSCryptTCP = 6; // DNSCrypt over TCP (https://dnscrypt.info/protocol) + } + enum PolicyType { + UNKNOWN = 1; // No RPZ policy applied, or unknown type + QNAME = 2; // Policy matched on the QName + CLIENTIP = 3; // Policy matched on the client IP + RESPONSEIP = 4; // Policy matched on one of the IPs contained in the answer + NSDNAME = 5; // Policy matched on the name of one nameserver involved + NSIP = 6; // Policy matched on the IP of one nameserver involved + } + enum PolicyKind { + NoAction = 1; // No action taken + Drop = 2; // https://tools.ietf.org/html/draft-vixie-dns-rpz-04 3.4 + NXDOMAIN = 3; // https://tools.ietf.org/html/draft-vixie-dns-rpz-04 3.1 + NODATA = 4; // https://tools.ietf.org/html/draft-vixie-dns-rpz-04 3.2 + Truncate= 5; // https://tools.ietf.org/html/draft-vixie-dns-rpz-04 3.5 + Custom = 6; // https://tools.ietf.org/html/draft-vixie-dns-rpz-04 3.6 + } + enum VState { + Indeterminate = 1; + Insecure = 2; + Secure = 3; + BogusNoValidDNSKEY = 4; + BogusInvalidDenial = 5; + BogusUnableToGetDSs = 6; + BogusUnableToGetDNSKEYs = 7; + BogusSelfSignedDS = 8; + BogusNoRRSIG = 9; + BogusNoValidRRSIG = 10; + BogusMissingNegativeIndication = 11; + BogusSignatureNotYetValid = 12; + BogusSignatureExpired = 13; + BogusUnsupportedDNSKEYAlgo = 14; + BogusUnsupportedDSDigestType = 15; + BogusNoZoneKeyBitSet = 16; + BogusRevokedDNSKEY = 17; + BogusInvalidDNSKEYProtocol = 18; + } + required Type type = 1; // Type of event + optional bytes messageId = 2; // UUID, shared by the query and the response + optional bytes serverIdentity = 3; // ID of the server emitting the protobuf message + optional SocketFamily socketFamily = 4; + optional SocketProtocol socketProtocol = 5; + optional bytes from = 6; // DNS requestor (client) as 4 (IPv4) or 16 (IPv6) raw bytes in network byte order + optional bytes to = 7; // DNS responder (server) as 4 (IPv4) or 16 (IPv6) raw bytes in network byte order + optional uint64 inBytes = 8; // Size of the query or response on the wire + optional uint32 timeSec = 9; // Time of message reception (seconds since epoch) + optional uint32 timeUsec = 10; // Time of message reception (additional micro-seconds) + optional uint32 id = 11; // ID of the query/response as found in the DNS header + + message DNSQuestion { + optional string qName = 1; // Fully qualified DNS name (with trailing dot) + optional uint32 qType = 2; // https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4 + optional uint32 qClass = 3; // Typically 1 (IN), see https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2 + } + optional DNSQuestion question = 12; // DNS query received from client + + message DNSResponse { + // See exportTypes in https://docs.powerdns.com/recursor/lua-config/protobuf.html#protobufServer + // for the list of supported resource record types. + message DNSRR { + optional string name = 1; // Fully qualified DNS name (with trailing dot) + optional uint32 type = 2; // https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4 + optional uint32 class = 3; // Typically 1 (IN), see https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2 + optional uint32 ttl = 4; // TTL in seconds + optional bytes rdata = 5; // raw address bytes in network byte order for A & AAAA; text representation for others, with fully qualified (trailing dot) domain names + optional bool udr = 6; // True if this is the first time this RR has been seen for this question + } + optional uint32 rcode = 1; // DNS Response code, or 65536 for a network error including a timeout + repeated DNSRR rrs = 2; // DNS resource records in response + optional string appliedPolicy = 3; // Filtering policy (RPZ or Lua) applied + repeated string tags = 4; // Additional tags applied + optional uint32 queryTimeSec = 5; // Time of the corresponding query reception (seconds since epoch) + optional uint32 queryTimeUsec = 6; // Time of the corresponding query reception (additional micro-seconds) + optional PolicyType appliedPolicyType = 7; // Type of the filtering policy (RPZ or Lua) applied + optional string appliedPolicyTrigger = 8; // The RPZ trigger + optional string appliedPolicyHit = 9; // The value (qname or IP) that caused the hit + optional PolicyKind appliedPolicyKind = 10; // The Kind (RPZ action) applied by the hit + optional VState validationState = 11; // The DNSSEC Validation State + } + + optional DNSResponse response = 13; + optional bytes originalRequestorSubnet = 14; // EDNS Client Subnet value (4 or 16 raw bytes in network byte order) + optional string requestorId = 15; // Username of the requestor + optional bytes initialRequestId = 16; // UUID of the incoming query that initiated this outgoing query or incoming response + optional bytes deviceId = 17; // Device ID of the requestor (could be mac address IP address or e.g. IMEI, format implementation dependent) + optional bool newlyObservedDomain = 18; // True if the domain has not been seen before + optional string deviceName = 19; // Device name of the requestor + optional uint32 fromPort = 20; // Source port of the DNS query (client) + optional uint32 toPort = 21; // Destination port of the DNS query (server) + + message MetaValue { + repeated string stringVal = 1; + repeated int64 intVal = 2; + } + + message Meta { + required string key = 1; // MUST be unique, so if you have multiple values they must be aggregated into on Meta + required MetaValue value = 2; + } + repeated Meta meta = 22; // Arbitrary meta-data - to be used in future rather than adding new fields all the time + + // The well known EventTrace event numbers + enum EventType { + // Range 0..99: Generic events + CustomEvent = 0; // A custom event + ReqRecv = 1; // A request was received + PCacheCheck = 2; // A packet cache check was initiated or completed; value: bool cacheHit + AnswerSent = 3; // An answer was sent to the client + + // Range 100: Recursor events + SyncRes = 100; // Recursor Syncres main function has started or completed; value: int rcode + LuaGetTag = 101; // Events below mark start or end of Lua hook calls; value: return value of hook + LuaGetTagFFI = 102; + LuaIPFilter = 103; + LuaPreRPZ = 104; + LuaPreResolve = 105; + LuaPreOutQuery = 106; + LuaPostResolve = 107; + LuaNoData = 108; + LuaNXDomain = 109; + LuaPostResolveFFI = 110; + } + + message Event { + required int64 ts = 1; // Timestamp in ns relative to time of creation of event trace data structure + required EventType event = 2; // Type of event + required bool start = 3; // true for "start" events, false for "completed" events + optional bool boolVal = 4; // Below are optional values associated with events + optional int64 intVal = 5; + optional string stringVal = 6; + optional bytes bytesVal = 7; + optional string custom = 8; // The name of the event for custom events + } + repeated Event trace = 23; +} + +message PBDNSMessageList { + repeated PBDNSMessage msg = 1; +} diff --git a/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/telegraf.conf b/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/telegraf.conf new file mode 100644 index 000000000..0fc1c6420 --- /dev/null +++ b/plugins/parsers/xpath/testcases/protobuf_powerdns_hex/telegraf.conf @@ -0,0 +1,14 @@ +[[inputs.file]] + files = ["./testcases/protobuf_powerdns_hex/powerdns_message.bin"] + data_format = "xpath_protobuf" + + xpath_native_types = true + xpath_protobuf_file = "powerdns_message.proto" + xpath_protobuf_type = "PBDNSMessage" + xpath_protobuf_import_paths = [".", "./testcases/protobuf_powerdns_hex"] + xpath_protobuf_skip_bytes = 2 + + [[inputs.file.xpath]] + metric_name = "'powerdns'" + fields_bytes_as_hex = ["from", "to", "messageId"] + field_selection = "descendant::*"