fix(inputs.gnmi): Add option to guess path tag from subscription (#14951)

This commit is contained in:
Sven Rebhan 2024-03-11 13:16:13 +01:00 committed by GitHub
parent 821865165a
commit eb5407a210
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 624 additions and 25 deletions

View File

@ -0,0 +1,5 @@
//go:build !custom || (migrations && (inputs || inputs.gnmi))
package all
import _ "github.com/influxdata/telegraf/migrations/inputs_gnmi" // register migration

View File

@ -0,0 +1,47 @@
package inputs_gnmi
import (
"github.com/influxdata/toml"
"github.com/influxdata/toml/ast"
"github.com/influxdata/telegraf/migrations"
)
// Migration function
func migrate(tbl *ast.Table) ([]byte, string, error) {
// Decode the old data structure
var plugin map[string]interface{}
if err := toml.UnmarshalTable(tbl, &plugin); err != nil {
return nil, "", err
}
// Check for deprecated option(s) and migrate them
var applied bool
if raw, found := plugin["guess_path_tag"]; found {
applied = true
if v, ok := raw.(bool); ok && v {
plugin["path_guessing_strategy"] = "common path"
}
// Remove the ignored setting
delete(plugin, "guess_path_tag")
}
// No options migrated so we can exit early
if !applied {
return nil, "", migrations.ErrNotApplicable
}
// Create the corresponding plugin configurations
cfg := migrations.CreateTOMLStruct("inputs", "gnmi")
cfg.Add("inputs", "gnmi", plugin)
output, err := toml.Marshal(cfg)
return output, "", err
}
// Register the migration function for the plugin type
func init() {
migrations.AddPluginOptionMigration("inputs.gnmi", migrate)
}

View File

@ -0,0 +1,73 @@
package inputs_gnmi_test
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf/config"
_ "github.com/influxdata/telegraf/migrations/inputs_gnmi" // register migration
"github.com/influxdata/telegraf/plugins/inputs/gnmi"
)
func TestNoMigration(t *testing.T) {
plugin := &gnmi.GNMI{}
defaultCfg := []byte(plugin.SampleConfig())
// Migrate and check that nothing changed
output, n, err := config.ApplyMigrations(defaultCfg)
require.NoError(t, err)
require.NotEmpty(t, output)
require.Zero(t, n)
require.Equal(t, string(defaultCfg), string(output))
}
func TestCases(t *testing.T) {
// Get all directories in testdata
folders, err := os.ReadDir("testcases")
require.NoError(t, err)
for _, f := range folders {
// Only handle folders
if !f.IsDir() {
continue
}
t.Run(f.Name(), func(t *testing.T) {
testcasePath := filepath.Join("testcases", f.Name())
inputFile := filepath.Join(testcasePath, "telegraf.conf")
expectedFile := filepath.Join(testcasePath, "expected.conf")
// Read the expected output
expected := config.NewConfig()
require.NoError(t, expected.LoadConfig(expectedFile))
require.NotEmpty(t, expected.Inputs)
// Read the input data
input, remote, err := config.LoadConfigFile(inputFile)
require.NoError(t, err)
require.False(t, remote)
require.NotEmpty(t, input)
// Migrate
output, n, err := config.ApplyMigrations(input)
require.NoError(t, err)
require.NotEmpty(t, output)
require.GreaterOrEqual(t, n, uint64(1))
actual := config.NewConfig()
require.NoError(t, actual.LoadConfigData(output))
// Test the output
require.Len(t, actual.Inputs, len(expected.Inputs))
actualIDs := make([]string, 0, len(expected.Inputs))
expectedIDs := make([]string, 0, len(expected.Inputs))
for i := range actual.Inputs {
actualIDs = append(actualIDs, actual.Inputs[i].ID())
expectedIDs = append(expectedIDs, expected.Inputs[i].ID())
}
require.ElementsMatch(t, expectedIDs, actualIDs, string(output))
})
}
}

View File

@ -0,0 +1,11 @@
[[inputs.gnmi]]
addresses = ["10.49.234.114:57777"]
password = "cisco"
path_guessing_strategy = "common path"
username = "cisco"
[[inputs.gnmi.subscription]]
name = "ifcounters"
origin = "openconfig-interfaces"
path = "/interfaces/interface/state/counters"
subscription_mode = "sample"
sample_interval = "10s"

View File

@ -0,0 +1,109 @@
# gNMI telemetry input plugin
[[inputs.gnmi]]
## Address and port of the gNMI GRPC server
addresses = ["10.49.234.114:57777"]
## define credentials
username = "cisco"
password = "cisco"
## gNMI encoding requested (one of: "proto", "json", "json_ietf", "bytes")
# encoding = "proto"
## redial in case of failures after
# redial = "10s"
## gRPC Maximum Message Size
# max_msg_size = "4MB"
## Enable to get the canonical path as field-name
# canonical_field_names = false
## Remove leading slashes and dots in field-name
# trim_field_names = false
## Guess the path-tag if an update does not contain a prefix-path
## If enabled, the common-path of all elements in the update is used.
guess_path_tag = true
## enable client-side TLS and define CA to authenticate the device
# enable_tls = false
# tls_ca = "/etc/telegraf/ca.pem"
## Minimal TLS version to accept by the client
# tls_min_version = "TLS12"
## Use TLS but skip chain & host verification
# insecure_skip_verify = true
## define client-side TLS certificate & key to authenticate to the device
# tls_cert = "/etc/telegraf/cert.pem"
# tls_key = "/etc/telegraf/key.pem"
## gNMI subscription prefix (optional, can usually be left empty)
## See: https://github.com/openconfig/reference/blob/master/rpc/gnmi/gnmi-specification.md#222-paths
# origin = ""
# prefix = ""
# target = ""
## Vendor specific options
## This defines what vendor specific options to load.
## * Juniper Header Extension (juniper_header): some sensors are directly managed by
## Linecard, which adds the Juniper GNMI Header Extension. Enabling this
## allows the decoding of the Extension header if present. Currently this knob
## adds component, component_id & sub_component_id as additional tags
# vendor_specific = []
## Define additional aliases to map encoding paths to measurement names
# [inputs.gnmi.aliases]
# ifcounters = "openconfig:/interfaces/interface/state/counters"
[[inputs.gnmi.subscription]]
## Name of the measurement that will be emitted
name = "ifcounters"
## Origin and path of the subscription
## See: https://github.com/openconfig/reference/blob/master/rpc/gnmi/gnmi-specification.md#222-paths
##
## origin usually refers to a (YANG) data model implemented by the device
## and path to a specific substructure inside it that should be subscribed
## to (similar to an XPath). YANG models can be found e.g. here:
## https://github.com/YangModels/yang/tree/master/vendor/cisco/xr
origin = "openconfig-interfaces"
path = "/interfaces/interface/state/counters"
## Subscription mode ("target_defined", "sample", "on_change") and interval
subscription_mode = "sample"
sample_interval = "10s"
## Suppress redundant transmissions when measured values are unchanged
# suppress_redundant = false
## If suppression is enabled, send updates at least every X seconds anyway
# heartbeat_interval = "60s"
## Tag subscriptions are applied as tags to other subscriptions.
# [[inputs.gnmi.tag_subscription]]
# ## When applying this value as a tag to other metrics, use this tag name
# name = "descr"
#
# ## All other subscription fields are as normal
# origin = "openconfig-interfaces"
# path = "/interfaces/interface/state"
# subscription_mode = "on_change"
#
# ## Match strategy to use for the tag.
# ## Tags are only applied for metrics of the same address. The following
# ## settings are valid:
# ## unconditional -- always match
# ## name -- match by the "name" key
# ## This resembles the previsou 'tag-only' behavior.
# ## elements -- match by the keys in the path filtered by the path
# ## parts specified `elements` below
# ## By default, 'elements' is used if the 'elements' option is provided,
# ## otherwise match by 'name'.
# # match = ""
#
# ## For the 'elements' match strategy, at least one path-element name must
# ## be supplied containing at least one key to match on. Multiple path
# ## elements can be specified in any order. All given keys must be equal
# ## for a match.
# # elements = ["description", "interface"]

View File

@ -0,0 +1,10 @@
[[inputs.gnmi]]
addresses = ["10.49.234.114:57777"]
password = "cisco"
username = "cisco"
[[inputs.gnmi.subscription]]
name = "ifcounters"
origin = "openconfig-interfaces"
path = "/interfaces/interface/state/counters"
subscription_mode = "sample"
sample_interval = "10s"

View File

@ -0,0 +1,109 @@
# gNMI telemetry input plugin
[[inputs.gnmi]]
## Address and port of the gNMI GRPC server
addresses = ["10.49.234.114:57777"]
## define credentials
username = "cisco"
password = "cisco"
## gNMI encoding requested (one of: "proto", "json", "json_ietf", "bytes")
# encoding = "proto"
## redial in case of failures after
# redial = "10s"
## gRPC Maximum Message Size
# max_msg_size = "4MB"
## Enable to get the canonical path as field-name
# canonical_field_names = false
## Remove leading slashes and dots in field-name
# trim_field_names = false
## Guess the path-tag if an update does not contain a prefix-path
## If enabled, the common-path of all elements in the update is used.
guess_path_tag = false
## enable client-side TLS and define CA to authenticate the device
# enable_tls = false
# tls_ca = "/etc/telegraf/ca.pem"
## Minimal TLS version to accept by the client
# tls_min_version = "TLS12"
## Use TLS but skip chain & host verification
# insecure_skip_verify = true
## define client-side TLS certificate & key to authenticate to the device
# tls_cert = "/etc/telegraf/cert.pem"
# tls_key = "/etc/telegraf/key.pem"
## gNMI subscription prefix (optional, can usually be left empty)
## See: https://github.com/openconfig/reference/blob/master/rpc/gnmi/gnmi-specification.md#222-paths
# origin = ""
# prefix = ""
# target = ""
## Vendor specific options
## This defines what vendor specific options to load.
## * Juniper Header Extension (juniper_header): some sensors are directly managed by
## Linecard, which adds the Juniper GNMI Header Extension. Enabling this
## allows the decoding of the Extension header if present. Currently this knob
## adds component, component_id & sub_component_id as additional tags
# vendor_specific = []
## Define additional aliases to map encoding paths to measurement names
# [inputs.gnmi.aliases]
# ifcounters = "openconfig:/interfaces/interface/state/counters"
[[inputs.gnmi.subscription]]
## Name of the measurement that will be emitted
name = "ifcounters"
## Origin and path of the subscription
## See: https://github.com/openconfig/reference/blob/master/rpc/gnmi/gnmi-specification.md#222-paths
##
## origin usually refers to a (YANG) data model implemented by the device
## and path to a specific substructure inside it that should be subscribed
## to (similar to an XPath). YANG models can be found e.g. here:
## https://github.com/YangModels/yang/tree/master/vendor/cisco/xr
origin = "openconfig-interfaces"
path = "/interfaces/interface/state/counters"
## Subscription mode ("target_defined", "sample", "on_change") and interval
subscription_mode = "sample"
sample_interval = "10s"
## Suppress redundant transmissions when measured values are unchanged
# suppress_redundant = false
## If suppression is enabled, send updates at least every X seconds anyway
# heartbeat_interval = "60s"
## Tag subscriptions are applied as tags to other subscriptions.
# [[inputs.gnmi.tag_subscription]]
# ## When applying this value as a tag to other metrics, use this tag name
# name = "descr"
#
# ## All other subscription fields are as normal
# origin = "openconfig-interfaces"
# path = "/interfaces/interface/state"
# subscription_mode = "on_change"
#
# ## Match strategy to use for the tag.
# ## Tags are only applied for metrics of the same address. The following
# ## settings are valid:
# ## unconditional -- always match
# ## name -- match by the "name" key
# ## This resembles the previsou 'tag-only' behavior.
# ## elements -- match by the keys in the path filtered by the path
# ## parts specified `elements` below
# ## By default, 'elements' is used if the 'elements' option is provided,
# ## otherwise match by 'name'.
# # match = ""
#
# ## For the 'elements' match strategy, at least one path-element name must
# ## be supplied containing at least one key to match on. Multiple path
# ## elements can be specified in any order. All given keys must be equal
# ## for a match.
# # elements = ["description", "interface"]

View File

@ -62,8 +62,11 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
# trim_field_names = false # trim_field_names = false
## Guess the path-tag if an update does not contain a prefix-path ## Guess the path-tag if an update does not contain a prefix-path
## If enabled, the common-path of all elements in the update is used. ## Supported values are
# guess_path_tag = false ## none -- do not add a 'path' tag
## common path -- use the common path elements of all fields in an update
## subscription -- use the subscription path
# path_guessing_strategy = "none"
## enable client-side TLS and define CA to authenticate the device ## enable client-side TLS and define CA to authenticate the device
# enable_tls = false # enable_tls = false
@ -163,12 +166,14 @@ ifcounters,path=openconfig-interfaces:/interfaces/interface/state/counters,host=
## Troubleshooting ## Troubleshooting
### Empty metric-name warning
Some devices (e.g. Juniper) report spurious data with response paths not Some devices (e.g. Juniper) report spurious data with response paths not
corresponding to any subscription. In those cases, Telegraf will not be able corresponding to any subscription. In those cases, Telegraf will not be able
to determine the metric name for the response and you get an to determine the metric name for the response and you get an
*empty metric-name warning* *empty metric-name warning*
For examplem if you subscribe to `/junos/system/linecard/cpu/memory` but the For example if you subscribe to `/junos/system/linecard/cpu/memory` but the
corresponding response arrives with path corresponding response arrives with path
`/components/component/properties/property/...` To avoid those issues, you can `/components/component/properties/property/...` To avoid those issues, you can
manually map the response to a metric name using the `aliases` option like manually map the response to a metric name using the `aliases` option like
@ -190,3 +195,14 @@ manually map the response to a metric name using the `aliases` option like
If this does *not* solve the issue, please follow the warning instructions and If this does *not* solve the issue, please follow the warning instructions and
open an issue with the response, your configuration and the metric you expect. open an issue with the response, your configuration and the metric you expect.
### Missing `path` tag
Some devices (e.g. Arista) omit the prefix and specify the path in the update
if there is only one value reported. This leads to a missing `path` tag for
the resulting metrics. In those cases you should set `path_guessing_strategy`
to `subscription` to use the subscription path as `path` tag.
Other devices might omit the prefix in updates altogether. Here setting
`path_guessing_strategy` to `common path` can help to infer the `path` tag by
using the part of the path that is common to all values in the update.

View File

@ -54,7 +54,8 @@ type GNMI struct {
Trace bool `toml:"dump_responses"` Trace bool `toml:"dump_responses"`
CanonicalFieldNames bool `toml:"canonical_field_names"` CanonicalFieldNames bool `toml:"canonical_field_names"`
TrimFieldNames bool `toml:"trim_field_names"` TrimFieldNames bool `toml:"trim_field_names"`
GuessPathTag bool `toml:"guess_path_tag"` GuessPathTag bool `toml:"guess_path_tag" deprecated:"1.30.0;use 'path_guessing_strategy' instead"`
GuessPathStrategy string `toml:"path_guessing_strategy"`
EnableTLS bool `toml:"enable_tls" deprecated:"1.27.0;use 'tls_enable' instead"` EnableTLS bool `toml:"enable_tls" deprecated:"1.27.0;use 'tls_enable' instead"`
Log telegraf.Logger `toml:"-"` Log telegraf.Logger `toml:"-"`
internaltls.ClientConfig internaltls.ClientConfig
@ -67,29 +68,23 @@ type GNMI struct {
// Subscription for a gNMI client // Subscription for a gNMI client
type Subscription struct { type Subscription struct {
Name string Name string `toml:"name"`
Origin string Origin string `toml:"origin"`
Path string Path string `toml:"path"`
fullPath *gnmiLib.Path
// Subscription mode and interval
SubscriptionMode string `toml:"subscription_mode"` SubscriptionMode string `toml:"subscription_mode"`
SampleInterval config.Duration `toml:"sample_interval"` SampleInterval config.Duration `toml:"sample_interval"`
// Duplicate suppression
SuppressRedundant bool `toml:"suppress_redundant"` SuppressRedundant bool `toml:"suppress_redundant"`
HeartbeatInterval config.Duration `toml:"heartbeat_interval"` HeartbeatInterval config.Duration `toml:"heartbeat_interval"`
// Mark this subscription as a tag-only lookup source, not emitting any metric
TagOnly bool `toml:"tag_only" deprecated:"1.25.0;2.0.0;please use 'tag_subscription's instead"` TagOnly bool `toml:"tag_only" deprecated:"1.25.0;2.0.0;please use 'tag_subscription's instead"`
fullPath *gnmiLib.Path
} }
// Tag Subscription for a gNMI client // Tag Subscription for a gNMI client
type TagSubscription struct { type TagSubscription struct {
Subscription Subscription
Match string `toml:"match"` Match string `toml:"match"`
Elements []string Elements []string `toml:"elements"`
} }
func (*GNMI) SampleConfig() string { func (*GNMI) SampleConfig() string {
@ -107,6 +102,21 @@ func (c *GNMI) Init() error {
return fmt.Errorf("unsupported vendor_specific option: %w", err) return fmt.Errorf("unsupported vendor_specific option: %w", err)
} }
// Check path guessing and handle deprecated option
if c.GuessPathTag {
if c.GuessPathStrategy == "" {
c.GuessPathStrategy = "common path"
}
if c.GuessPathStrategy != "common path" {
return errors.New("conflicting settings between 'guess_path_tag' and 'path_guessing_strategy'")
}
}
switch c.GuessPathStrategy {
case "", "none", "common path", "subscription":
default:
return fmt.Errorf("invalid 'path_guessing_strategy' %q", c.GuessPathStrategy)
}
// Use the new TLS option for enabling // Use the new TLS option for enabling
// Honor deprecated option // Honor deprecated option
enable := (c.ClientConfig.Enable != nil && *c.ClientConfig.Enable) || c.EnableTLS enable := (c.ClientConfig.Enable != nil && *c.ClientConfig.Enable) || c.EnableTLS
@ -221,7 +231,7 @@ func (c *GNMI) Start(acc telegraf.Accumulator) error {
trace: c.Trace, trace: c.Trace,
canonicalFieldNames: c.CanonicalFieldNames, canonicalFieldNames: c.CanonicalFieldNames,
trimSlash: c.TrimFieldNames, trimSlash: c.TrimFieldNames,
guessPathTag: c.GuessPathTag, guessPathStrategy: c.GuessPathStrategy,
log: c.Log, log: c.Log,
} }
for ctx.Err() == nil { for ctx.Err() == nil {

View File

@ -40,7 +40,7 @@ type handler struct {
trace bool trace bool
canonicalFieldNames bool canonicalFieldNames bool
trimSlash bool trimSlash bool
guessPathTag bool guessPathStrategy string
log telegraf.Logger log telegraf.Logger
} }
@ -198,7 +198,7 @@ func (h *handler) handleSubscribeResponseUpdate(acc telegraf.Accumulator, respon
// Some devices do not provide a prefix, so do some guesswork based // Some devices do not provide a prefix, so do some guesswork based
// on the paths of the fields // on the paths of the fields
if headerTags["path"] == "" && h.guessPathTag { if headerTags["path"] == "" && h.guessPathStrategy == "common path" {
if prefixPath := guessPrefixFromUpdate(valueFields); prefixPath != "" { if prefixPath := guessPrefixFromUpdate(valueFields); prefixPath != "" {
headerTags["path"] = prefixPath headerTags["path"] = prefixPath
} }
@ -232,6 +232,10 @@ func (h *handler) handleSubscribeResponseUpdate(acc telegraf.Accumulator, respon
} }
aliasInfo := newInfoFromString(aliasPath) aliasInfo := newInfoFromString(aliasPath)
if tags["path"] == "" && h.guessPathStrategy == "subscription" {
tags["path"] = aliasInfo.String()
}
// Group metrics // Group metrics
var key string var key string
if h.canonicalFieldNames { if h.canonicalFieldNames {

View File

@ -23,8 +23,11 @@
# trim_field_names = false # trim_field_names = false
## Guess the path-tag if an update does not contain a prefix-path ## Guess the path-tag if an update does not contain a prefix-path
## If enabled, the common-path of all elements in the update is used. ## Supported values are
# guess_path_tag = false ## none -- do not add a 'path' tag
## common path -- use the common path elements of all fields in an update
## subscription -- use the subscription path
# path_guessing_strategy = "none"
## enable client-side TLS and define CA to authenticate the device ## enable client-side TLS and define CA to authenticate the device
# enable_tls = false # enable_tls = false

View File

@ -3,7 +3,8 @@
name_override = "gnmi" name_override = "gnmi"
redial = "10s" redial = "10s"
encoding = "json_ietf" encoding = "json_ietf"
guess_path_tag = true path_guessing_strategy = "common path"
[[inputs.gnmi.subscription]] [[inputs.gnmi.subscription]]
name = "ifdesc" name = "ifdesc"
origin = "openconfig-interfaces" origin = "openconfig-interfaces"

View File

@ -0,0 +1,4 @@
gnmi_sys_memory,path=/system/memory/state,source=127.0.0.1 reserved=6359478272u,used=3479629824u 1709737743568119333
gnmi_sys_memory,path=/system/memory/state,source=127.0.0.1 used=3479527424u 1709737753565697718
gnmi_sys_cpu,index=ALL,path=/system/cpus/cpu/state,source=127.0.0.1 hardware_interrupt/min_time=1709805333568034887u 1709805333566280930
gnmi_sys_cpu,index=ALL,path=/system/cpus/cpu/state,source=127.0.0.1 hardware_interrupt/min_time=1709805343567684412u,idle/avg=89u,idle/instant=90u 1709805343565718902

View File

@ -0,0 +1,182 @@
[
{
"update": {
"timestamp": "1709737743568119333",
"prefix": {
"elem": [
{
"name": "system"
},
{
"name": "memory"
},
{
"name": "state"
}
]
},
"update": [
{
"path": {
"elem": [
{
"name": "reserved"
}
]
},
"val": {
"uintVal": "6359478272"
}
},
{
"path": {
"elem": [
{
"name": "used"
}
]
},
"val": {
"uintVal": "3479629824"
}
}
]
}
},
{
"update": {
"timestamp": "1709737753565697718",
"update": [
{
"path": {
"elem": [
{
"name": "system"
},
{
"name": "memory"
},
{
"name": "state"
},
{
"name": "used"
}
]
},
"val": {
"uintVal": "3479527424"
}
}
]
}
},
{
"update": {
"timestamp": "1709805333566280930",
"update": [
{
"path": {
"elem": [
{
"name": "system"
},
{
"name": "cpus"
},
{
"name": "cpu",
"key": {
"index": "ALL"
}
},
{
"name": "state"
},
{
"name": "hardware-interrupt"
},
{
"name": "min-time"
}
]
},
"val": {
"uintVal": "1709805333568034887"
}
}
]
}
},
{
"update": {
"timestamp": "1709805343565718902",
"prefix": {
"elem": [
{
"name": "system"
},
{
"name": "cpus"
},
{
"name": "cpu",
"key": {
"index": "ALL"
}
},
{
"name": "state"
}
]
},
"update": [
{
"path": {
"elem": [
{
"name": "hardware-interrupt"
},
{
"name": "min-time"
}
]
},
"val": {
"uintVal": "1709805343567684412"
}
},
{
"path": {
"elem": [
{
"name": "idle"
},
{
"name": "avg"
}
]
},
"val": {
"uintVal": "89"
}
},
{
"path": {
"elem": [
{
"name": "idle"
},
{
"name": "instant"
}
]
},
"val": {
"uintVal": "90"
}
}
]
}
}
]

View File

@ -0,0 +1,15 @@
[[inputs.gnmi]]
addresses = ["dummy"]
path_guessing_strategy = "subscription"
[[inputs.gnmi.subscription]]
name = "gnmi_sys_cpu"
path = "/system/cpus/cpu/state"
subscription_mode = "sample"
sample_interval = "10s"
[[inputs.gnmi.subscription]]
name = "gnmi_sys_memory"
path = "/system/memory/state"
subscription_mode = "sample"
sample_interval = "10s"