CL-Softwares
/
telegraf
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(inputs.gnmi): Add option to guess path tag from subscription (#14951)

This commit is contained in:
Sven Rebhan 2024-03-11 13:16:13 +01:00 committed by GitHub
parent 821865165a
commit eb5407a210
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 624 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
migrations/all/inputs_gnmi.go Normal file
View File

@ -0,0 +1,5 @@
//go:build !custom || (migrations && (inputs || inputs.gnmi))
package all
import _ "github.com/influxdata/telegraf/migrations/inputs_gnmi" // register migration

47
migrations/inputs_gnmi/migration.go Normal file
View File

@ -0,0 +1,47 @@
package inputs_gnmi
import (
"github.com/influxdata/toml"
"github.com/influxdata/toml/ast"
"github.com/influxdata/telegraf/migrations"
)
// Migration function
func migrate(tbl *ast.Table) ([]byte, string, error) {
// Decode the old data structure
var plugin map[string]interface{}
if err := toml.UnmarshalTable(tbl, &plugin); err != nil {
return nil, "", err
}
// Check for deprecated option(s) and migrate them
var applied bool
if raw, found := plugin["guess_path_tag"]; found {
applied = true
if v, ok := raw.(bool); ok && v {
plugin["path_guessing_strategy"] = "common path"
}
// Remove the ignored setting
delete(plugin, "guess_path_tag")
}
// No options migrated so we can exit early
if !applied {
return nil, "", migrations.ErrNotApplicable
}
// Create the corresponding plugin configurations
cfg := migrations.CreateTOMLStruct("inputs", "gnmi")
cfg.Add("inputs", "gnmi", plugin)
output, err := toml.Marshal(cfg)
return output, "", err
}
// Register the migration function for the plugin type
func init() {
migrations.AddPluginOptionMigration("inputs.gnmi", migrate)
}

73
migrations/inputs_gnmi/migration_test.go Normal file
View File

@ -0,0 +1,73 @@
package inputs_gnmi_test
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf/config"
_ "github.com/influxdata/telegraf/migrations/inputs_gnmi" // register migration
"github.com/influxdata/telegraf/plugins/inputs/gnmi"
)
func TestNoMigration(t *testing.T) {
plugin := &gnmi.GNMI{}
defaultCfg := []byte(plugin.SampleConfig())
// Migrate and check that nothing changed
output, n, err := config.ApplyMigrations(defaultCfg)
require.NoError(t, err)
require.NotEmpty(t, output)
require.Zero(t, n)
require.Equal(t, string(defaultCfg), string(output))
}
func TestCases(t *testing.T) {
// Get all directories in testdata
folders, err := os.ReadDir("testcases")
require.NoError(t, err)
for _, f := range folders {
// Only handle folders
if !f.IsDir() {
continue
}
t.Run(f.Name(), func(t *testing.T) {
testcasePath := filepath.Join("testcases", f.Name())
inputFile := filepath.Join(testcasePath, "telegraf.conf")
expectedFile := filepath.Join(testcasePath, "expected.conf")
// Read the expected output
expected := config.NewConfig()
require.NoError(t, expected.LoadConfig(expectedFile))
require.NotEmpty(t, expected.Inputs)
// Read the input data
input, remote, err := config.LoadConfigFile(inputFile)
require.NoError(t, err)
require.False(t, remote)
require.NotEmpty(t, input)
// Migrate
output, n, err := config.ApplyMigrations(input)
require.NoError(t, err)
require.NotEmpty(t, output)
require.GreaterOrEqual(t, n, uint64(1))
actual := config.NewConfig()
require.NoError(t, actual.LoadConfigData(output))
// Test the output
require.Len(t, actual.Inputs, len(expected.Inputs))
actualIDs := make([]string, 0, len(expected.Inputs))
expectedIDs := make([]string, 0, len(expected.Inputs))
for i := range actual.Inputs {
actualIDs = append(actualIDs, actual.Inputs[i].ID())
expectedIDs = append(expectedIDs, expected.Inputs[i].ID())
}
require.ElementsMatch(t, expectedIDs, actualIDs, string(output))
})
}
}

11
migrations/inputs_gnmi/testcases/deprecated_guess_path/expected.conf Normal file
View File

@ -0,0 +1,11 @@
[[inputs.gnmi]]
addresses = ["10.49.234.114:57777"]
password = "cisco"
path_guessing_strategy = "common path"
username = "cisco"
[[inputs.gnmi.subscription]]
name = "ifcounters"
origin = "openconfig-interfaces"
path = "/interfaces/interface/state/counters"
subscription_mode = "sample"
sample_interval = "10s"

109
migrations/inputs_gnmi/testcases/deprecated_guess_path/telegraf.conf Normal file
View File

@ -0,0 +1,109 @@
# gNMI telemetry input plugin
[[inputs.gnmi]]
## Address and port of the gNMI GRPC server
addresses = ["10.49.234.114:57777"]
## define credentials
username = "cisco"
password = "cisco"
## gNMI encoding requested (one of: "proto", "json", "json_ietf", "bytes")
# encoding = "proto"
## redial in case of failures after
# redial = "10s"
## gRPC Maximum Message Size
# max_msg_size = "4MB"
## Enable to get the canonical path as field-name
# canonical_field_names = false
## Remove leading slashes and dots in field-name
# trim_field_names = false
## Guess the path-tag if an update does not contain a prefix-path
## If enabled, the common-path of all elements in the update is used.
guess_path_tag = true
## enable client-side TLS and define CA to authenticate the device
# enable_tls = false
# tls_ca = "/etc/telegraf/ca.pem"
## Minimal TLS version to accept by the client
# tls_min_version = "TLS12"
## Use TLS but skip chain & host verification
# insecure_skip_verify = true
## define client-side TLS certificate & key to authenticate to the device
# tls_cert = "/etc/telegraf/cert.pem"
# tls_key = "/etc/telegraf/key.pem"
## gNMI subscription prefix (optional, can usually be left empty)
## See: https://github.com/openconfig/reference/blob/master/rpc/gnmi/gnmi-specification.md#222-paths
# origin = ""
# prefix = ""
# target = ""
## Vendor specific options
## This defines what vendor specific options to load.
## * Juniper Header Extension (juniper_header): some sensors are directly managed by
## Linecard, which adds the Juniper GNMI Header Extension. Enabling this
## allows the decoding of the Extension header if present. Currently this knob
## adds component, component_id & sub_component_id as additional tags
# vendor_specific = []
## Define additional aliases to map encoding paths to measurement names
# [inputs.gnmi.aliases]
# ifcounters = "openconfig:/interfaces/interface/state/counters"
[[inputs.gnmi.subscription]]
## Name of the measurement that will be emitted
name = "ifcounters"
## Origin and path of the subscription
## See: https://github.com/openconfig/reference/blob/master/rpc/gnmi/gnmi-specification.md#222-paths
##
## origin usually refers to a (YANG) data model implemented by the device
## and path to a specific substructure inside it that should be subscribed
## to (similar to an XPath). YANG models can be found e.g. here:
## https://github.com/YangModels/yang/tree/master/vendor/cisco/xr
origin = "openconfig-interfaces"
path = "/interfaces/interface/state/counters"
## Subscription mode ("target_defined", "sample", "on_change") and interval
subscription_mode = "sample"
sample_interval = "10s"
## Suppress redundant transmissions when measured values are unchanged
# suppress_redundant = false
## If suppression is enabled, send updates at least every X seconds anyway
# heartbeat_interval = "60s"
## Tag subscriptions are applied as tags to other subscriptions.
# [[inputs.gnmi.tag_subscription]]
# ## When applying this value as a tag to other metrics, use this tag name
# name = "descr"
#
# ## All other subscription fields are as normal
# origin = "openconfig-interfaces"
# path = "/interfaces/interface/state"
# subscription_mode = "on_change"
#
# ## Match strategy to use for the tag.
# ## Tags are only applied for metrics of the same address. The following
# ## settings are valid:
# ## unconditional -- always match
# ## name -- match by the "name" key
# ## This resembles the previsou 'tag-only' behavior.
# ## elements -- match by the keys in the path filtered by the path
# ## parts specified `elements` below
# ## By default, 'elements' is used if the 'elements' option is provided,
# ## otherwise match by 'name'.
# # match = ""
#
# ## For the 'elements' match strategy, at least one path-element name must
# ## be supplied containing at least one key to match on. Multiple path
# ## elements can be specified in any order. All given keys must be equal
# ## for a match.
# # elements = ["description", "interface"]

10
migrations/inputs_gnmi/testcases/deprecated_guess_path_false/expected.conf Normal file
View File

@ -0,0 +1,10 @@
[[inputs.gnmi]]
addresses = ["10.49.234.114:57777"]
password = "cisco"
username = "cisco"
[[inputs.gnmi.subscription]]
name = "ifcounters"
origin = "openconfig-interfaces"
path = "/interfaces/interface/state/counters"
subscription_mode = "sample"
sample_interval = "10s"

109
migrations/inputs_gnmi/testcases/deprecated_guess_path_false/telegraf.conf Normal file
View File

@ -0,0 +1,109 @@
# gNMI telemetry input plugin
[[inputs.gnmi]]
## Address and port of the gNMI GRPC server
addresses = ["10.49.234.114:57777"]
## define credentials
username = "cisco"
password = "cisco"
## gNMI encoding requested (one of: "proto", "json", "json_ietf", "bytes")
# encoding = "proto"
## redial in case of failures after
# redial = "10s"
## gRPC Maximum Message Size
# max_msg_size = "4MB"
## Enable to get the canonical path as field-name
# canonical_field_names = false
## Remove leading slashes and dots in field-name
# trim_field_names = false
## Guess the path-tag if an update does not contain a prefix-path
## If enabled, the common-path of all elements in the update is used.
guess_path_tag = false
## enable client-side TLS and define CA to authenticate the device
# enable_tls = false
# tls_ca = "/etc/telegraf/ca.pem"
## Minimal TLS version to accept by the client
# tls_min_version = "TLS12"
## Use TLS but skip chain & host verification
# insecure_skip_verify = true
## define client-side TLS certificate & key to authenticate to the device
# tls_cert = "/etc/telegraf/cert.pem"
# tls_key = "/etc/telegraf/key.pem"
## gNMI subscription prefix (optional, can usually be left empty)
## See: https://github.com/openconfig/reference/blob/master/rpc/gnmi/gnmi-specification.md#222-paths
# origin = ""
# prefix = ""
# target = ""
## Vendor specific options
## This defines what vendor specific options to load.
## * Juniper Header Extension (juniper_header): some sensors are directly managed by
## Linecard, which adds the Juniper GNMI Header Extension. Enabling this
## allows the decoding of the Extension header if present. Currently this knob
## adds component, component_id & sub_component_id as additional tags
# vendor_specific = []
## Define additional aliases to map encoding paths to measurement names
# [inputs.gnmi.aliases]
# ifcounters = "openconfig:/interfaces/interface/state/counters"
[[inputs.gnmi.subscription]]
## Name of the measurement that will be emitted
name = "ifcounters"
## Origin and path of the subscription
## See: https://github.com/openconfig/reference/blob/master/rpc/gnmi/gnmi-specification.md#222-paths
##
## origin usually refers to a (YANG) data model implemented by the device
## and path to a specific substructure inside it that should be subscribed
## to (similar to an XPath). YANG models can be found e.g. here:
## https://github.com/YangModels/yang/tree/master/vendor/cisco/xr
origin = "openconfig-interfaces"
path = "/interfaces/interface/state/counters"
## Subscription mode ("target_defined", "sample", "on_change") and interval
subscription_mode = "sample"
sample_interval = "10s"
## Suppress redundant transmissions when measured values are unchanged
# suppress_redundant = false
## If suppression is enabled, send updates at least every X seconds anyway
# heartbeat_interval = "60s"
## Tag subscriptions are applied as tags to other subscriptions.
# [[inputs.gnmi.tag_subscription]]
# ## When applying this value as a tag to other metrics, use this tag name
# name = "descr"
#
# ## All other subscription fields are as normal
# origin = "openconfig-interfaces"
# path = "/interfaces/interface/state"
# subscription_mode = "on_change"
#
# ## Match strategy to use for the tag.
# ## Tags are only applied for metrics of the same address. The following
# ## settings are valid:
# ## unconditional -- always match
# ## name -- match by the "name" key
# ## This resembles the previsou 'tag-only' behavior.
# ## elements -- match by the keys in the path filtered by the path
# ## parts specified `elements` below
# ## By default, 'elements' is used if the 'elements' option is provided,
# ## otherwise match by 'name'.
# # match = ""
#
# ## For the 'elements' match strategy, at least one path-element name must
# ## be supplied containing at least one key to match on. Multiple path
# ## elements can be specified in any order. All given keys must be equal
# ## for a match.
# # elements = ["description", "interface"]

22
plugins/inputs/gnmi/README.md
View File

@ -62,8 +62,11 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
# trim_field_names = false
## Guess the path-tag if an update does not contain a prefix-path
## If enabled, the common-path of all elements in the update is used.
# guess_path_tag = false
## Supported values are
## none -- do not add a 'path' tag
## common path -- use the common path elements of all fields in an update
## subscription -- use the subscription path
# path_guessing_strategy = "none"
## enable client-side TLS and define CA to authenticate the device
# enable_tls = false
@ -163,12 +166,14 @@ ifcounters,path=openconfig-interfaces:/interfaces/interface/state/counters,host=
## Troubleshooting
### Empty metric-name warning
Some devices (e.g. Juniper) report spurious data with response paths not
corresponding to any subscription. In those cases, Telegraf will not be able
to determine the metric name for the response and you get an
*empty metric-name warning*
For examplem if you subscribe to `/junos/system/linecard/cpu/memory` but the
For example if you subscribe to `/junos/system/linecard/cpu/memory` but the
corresponding response arrives with path
`/components/component/properties/property/...` To avoid those issues, you can
manually map the response to a metric name using the `aliases` option like
@ -190,3 +195,14 @@ manually map the response to a metric name using the `aliases` option like
If this does *not* solve the issue, please follow the warning instructions and
open an issue with the response, your configuration and the metric you expect.
### Missing `path` tag
Some devices (e.g. Arista) omit the prefix and specify the path in the update
if there is only one value reported. This leads to a missing `path` tag for
the resulting metrics. In those cases you should set `path_guessing_strategy`
to `subscription` to use the subscription path as `path` tag.
Other devices might omit the prefix in updates altogether. Here setting
`path_guessing_strategy` to `common path` can help to infer the `path` tag by
using the part of the path that is common to all values in the update.

44
plugins/inputs/gnmi/gnmi.go
View File

@ -54,7 +54,8 @@ type GNMI struct {
Trace bool `toml:"dump_responses"`
CanonicalFieldNames bool `toml:"canonical_field_names"`
TrimFieldNames bool `toml:"trim_field_names"`
GuessPathTag bool `toml:"guess_path_tag"`
GuessPathTag bool `toml:"guess_path_tag" deprecated:"1.30.0;use 'path_guessing_strategy' instead"`
GuessPathStrategy string `toml:"path_guessing_strategy"`
EnableTLS bool `toml:"enable_tls" deprecated:"1.27.0;use 'tls_enable' instead"`
Log telegraf.Logger `toml:"-"`
internaltls.ClientConfig
@ -67,29 +68,23 @@ type GNMI struct {
// Subscription for a gNMI client
type Subscription struct {
Name string
Origin string
Path string
fullPath *gnmiLib.Path
// Subscription mode and interval
SubscriptionMode string `toml:"subscription_mode"`
SampleInterval config.Duration `toml:"sample_interval"`
// Duplicate suppression
Name string `toml:"name"`
Origin string `toml:"origin"`
Path string `toml:"path"`
SubscriptionMode string `toml:"subscription_mode"`
SampleInterval config.Duration `toml:"sample_interval"`
SuppressRedundant bool `toml:"suppress_redundant"`
HeartbeatInterval config.Duration `toml:"heartbeat_interval"`
TagOnly bool `toml:"tag_only" deprecated:"1.25.0;2.0.0;please use 'tag_subscription's instead"`
// Mark this subscription as a tag-only lookup source, not emitting any metric
TagOnly bool `toml:"tag_only" deprecated:"1.25.0;2.0.0;please use 'tag_subscription's instead"`
fullPath *gnmiLib.Path
}
// Tag Subscription for a gNMI client
type TagSubscription struct {
Subscription
Match string `toml:"match"`
Elements []string
Match string `toml:"match"`
Elements []string `toml:"elements"`
}
func (*GNMI) SampleConfig() string {
@ -107,6 +102,21 @@ func (c *GNMI) Init() error {
return fmt.Errorf("unsupported vendor_specific option: %w", err)
}
// Check path guessing and handle deprecated option
if c.GuessPathTag {
if c.GuessPathStrategy == "" {
c.GuessPathStrategy = "common path"
}
if c.GuessPathStrategy != "common path" {
return errors.New("conflicting settings between 'guess_path_tag' and 'path_guessing_strategy'")
}
}
switch c.GuessPathStrategy {
case "", "none", "common path", "subscription":
default:
return fmt.Errorf("invalid 'path_guessing_strategy' %q", c.GuessPathStrategy)
}
// Use the new TLS option for enabling
// Honor deprecated option
enable := (c.ClientConfig.Enable != nil && *c.ClientConfig.Enable) || c.EnableTLS
@ -221,7 +231,7 @@ func (c *GNMI) Start(acc telegraf.Accumulator) error {
trace: c.Trace,
canonicalFieldNames: c.CanonicalFieldNames,
trimSlash: c.TrimFieldNames,
guessPathTag: c.GuessPathTag,
guessPathStrategy: c.GuessPathStrategy,
log: c.Log,
}
for ctx.Err() == nil {

8
plugins/inputs/gnmi/handler.go
View File

@ -40,7 +40,7 @@ type handler struct {
trace bool
canonicalFieldNames bool
trimSlash bool
guessPathTag bool
guessPathStrategy string
log telegraf.Logger
}
@ -198,7 +198,7 @@ func (h *handler) handleSubscribeResponseUpdate(acc telegraf.Accumulator, respon
// Some devices do not provide a prefix, so do some guesswork based
// on the paths of the fields
if headerTags["path"] == "" && h.guessPathTag {
if headerTags["path"] == "" && h.guessPathStrategy == "common path" {
if prefixPath := guessPrefixFromUpdate(valueFields); prefixPath != "" {
headerTags["path"] = prefixPath
}
@ -232,6 +232,10 @@ func (h *handler) handleSubscribeResponseUpdate(acc telegraf.Accumulator, respon
}
aliasInfo := newInfoFromString(aliasPath)
if tags["path"] == "" && h.guessPathStrategy == "subscription" {
tags["path"] = aliasInfo.String()
}
// Group metrics
var key string
if h.canonicalFieldNames {

7
plugins/inputs/gnmi/sample.conf
View File

@ -23,8 +23,11 @@
# trim_field_names = false
## Guess the path-tag if an update does not contain a prefix-path
## If enabled, the common-path of all elements in the update is used.
# guess_path_tag = false
## Supported values are
## none -- do not add a 'path' tag
## common path -- use the common path elements of all fields in an update
## subscription -- use the subscription path
# path_guessing_strategy = "none"
## enable client-side TLS and define CA to authenticate the device
# enable_tls = false

3
plugins/inputs/gnmi/testcases/issue_14044/telegraf.conf
View File

@ -3,7 +3,8 @@
name_override = "gnmi"
redial = "10s"
encoding = "json_ietf"
guess_path_tag = true
path_guessing_strategy = "common path"
[[inputs.gnmi.subscription]]
name = "ifdesc"
origin = "openconfig-interfaces"

4
plugins/inputs/gnmi/testcases/issue_14946/expected.out Normal file
View File

@ -0,0 +1,4 @@
gnmi_sys_memory,path=/system/memory/state,source=127.0.0.1 reserved=6359478272u,used=3479629824u 1709737743568119333
gnmi_sys_memory,path=/system/memory/state,source=127.0.0.1 used=3479527424u 1709737753565697718
gnmi_sys_cpu,index=ALL,path=/system/cpus/cpu/state,source=127.0.0.1 hardware_interrupt/min_time=1709805333568034887u 1709805333566280930
gnmi_sys_cpu,index=ALL,path=/system/cpus/cpu/state,source=127.0.0.1 hardware_interrupt/min_time=1709805343567684412u,idle/avg=89u,idle/instant=90u 1709805343565718902

182
plugins/inputs/gnmi/testcases/issue_14946/responses.json Normal file
View File

@ -0,0 +1,182 @@
[
{
"update": {
"timestamp": "1709737743568119333",
"prefix": {
"elem": [
{
"name": "system"
},
{
"name": "memory"
},
{
"name": "state"
}
]
},
"update": [
{
"path": {
"elem": [
{
"name": "reserved"
}
]
},
"val": {
"uintVal": "6359478272"
}
},
{
"path": {
"elem": [
{
"name": "used"
}
]
},
"val": {
"uintVal": "3479629824"
}
}
]
}
},
{
"update": {
"timestamp": "1709737753565697718",
"update": [
{
"path": {
"elem": [
{
"name": "system"
},
{
"name": "memory"
},
{
"name": "state"
},
{
"name": "used"
}
]
},
"val": {
"uintVal": "3479527424"
}
}
]
}
},
{
"update": {
"timestamp": "1709805333566280930",
"update": [
{
"path": {
"elem": [
{
"name": "system"
},
{
"name": "cpus"
},
{
"name": "cpu",
"key": {
"index": "ALL"
}
},
{
"name": "state"
},
{
"name": "hardware-interrupt"
},
{
"name": "min-time"
}
]
},
"val": {
"uintVal": "1709805333568034887"
}
}
]
}
},
{
"update": {
"timestamp": "1709805343565718902",
"prefix": {
"elem": [
{
"name": "system"
},
{
"name": "cpus"
},
{
"name": "cpu",
"key": {
"index": "ALL"
}
},
{
"name": "state"
}
]
},
"update": [
{
"path": {
"elem": [
{
"name": "hardware-interrupt"
},
{
"name": "min-time"
}
]
},
"val": {
"uintVal": "1709805343567684412"
}
},
{
"path": {
"elem": [
{
"name": "idle"
},
{
"name": "avg"
}
]
},
"val": {
"uintVal": "89"
}
},
{
"path": {
"elem": [
{
"name": "idle"
},
{
"name": "instant"
}
]
},
"val": {
"uintVal": "90"
}
}
]
}
}
]

15
plugins/inputs/gnmi/testcases/issue_14946/telegraf.conf Normal file
View File

@ -0,0 +1,15 @@
[[inputs.gnmi]]
addresses = ["dummy"]
path_guessing_strategy = "subscription"
[[inputs.gnmi.subscription]]
name = "gnmi_sys_cpu"
path = "/system/cpus/cpu/state"
subscription_mode = "sample"
sample_interval = "10s"
[[inputs.gnmi.subscription]]
name = "gnmi_sys_memory"
path = "/system/memory/state"
subscription_mode = "sample"
sample_interval = "10s"