From 0a491a7bf3ada78d69a01ec6c48209f64cad637c Mon Sep 17 00:00:00 2001 From: Sven Rebhan <36194019+srebhan@users.noreply.github.com> Date: Wed, 31 May 2023 19:06:50 +0200 Subject: [PATCH] feat(agent): Add option to avoid filtering of explicit plugin tags (#13364) --- config/config.go | 11 +++++- docs/CONFIGURATION.md | 9 ++++- models/running_input.go | 27 ++++++++++--- models/running_input_test.go | 75 ++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 10 deletions(-) diff --git a/config/config.go b/config/config.go index 5ef81be33..86a74ab95 100644 --- a/config/config.go +++ b/config/config.go @@ -252,6 +252,10 @@ type AgentConfig struct { // stateful plugins on termination of Telegraf. If the file exists on start, // the state in the file will be restored for the plugins. Statefile string `toml:"statefile"` + + // Flag to always keep tags explicitly defined in the plugin itself and + // ensure those tags always pass filtering. + AlwaysIncludeLocalTags bool `toml:"always_include_local_tags"` } // InputNames returns a list of strings of the configured inputs. @@ -1428,7 +1432,10 @@ func (c *Config) buildFilter(tbl *ast.Table) (models.Filter, error) { // builds the filter and returns a // models.InputConfig to be inserted into models.RunningInput func (c *Config) buildInput(name string, tbl *ast.Table) (*models.InputConfig, error) { - cp := &models.InputConfig{Name: name} + cp := &models.InputConfig{ + Name: name, + AlwaysIncludeLocalTags: c.Agent.AlwaysIncludeLocalTags, + } c.getFieldDuration(tbl, "interval", &cp.Interval) c.getFieldDuration(tbl, "precision", &cp.Precision) c.getFieldDuration(tbl, "collection_jitter", &cp.CollectionJitter) @@ -1523,7 +1530,7 @@ func (c *Config) buildOutput(name string, tbl *ast.Table) (*models.OutputConfig, func (c *Config) missingTomlField(_ reflect.Type, key string) error { switch key { // General options to ignore - case "alias", + case "alias", "always_include_local_tags", "collection_jitter", "collection_offset", "data_format", "delay", "drop", "drop_original", "fielddrop", "fieldpass", "flush_interval", "flush_jitter", diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 66458d308..d02165925 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -321,6 +321,11 @@ The agent table configures Telegraf and the defaults used across all plugins. stateful plugins on termination of Telegraf. If the file exists on start, the state in the file will be restored for the plugins. +- **always_include_local_tags**: + Ensure tags explicitly defined in a plugin will *always* pass tag-filtering + via `taginclude` or `tagexclude`. This removes the need to specify local tags + twice. + ## Plugins Telegraf plugins are divided into 4 types: [inputs][], [outputs][], @@ -402,7 +407,7 @@ Use the name_override parameter to emit measurements with the name `foobar`: Emit measurements with two additional tags: `tag1=foo` and `tag2=bar` > **NOTE**: With TOML, order matters. Parameters belong to the last defined -> table header, place `[inputs.cpu.tags]` table at the _end_ of the plugin +> table header, place `[inputs.cpu.tags]` table at the *end* of the plugin > definition. ```toml @@ -638,7 +643,7 @@ for time-based filtering. An introduction to the CEL language can be found are provided in the [language definition][CEL lang] as well as in the [extension documentation][CEL ext]. -> NOTE: As CEL is an _interpreted_ languguage, this type of filtering is much +> NOTE: As CEL is an *interpreted* languguage, this type of filtering is much > slower compared to `namepass`/`namedrop` and friends. So consider to use the > more restricted filter options where possible in case of high-throughput > scenarios. diff --git a/models/running_input.go b/models/running_input.go index 3d6e72f60..34fda9a9a 100644 --- a/models/running_input.go +++ b/models/running_input.go @@ -64,11 +64,12 @@ type InputConfig struct { CollectionOffset time.Duration Precision time.Duration - NameOverride string - MeasurementPrefix string - MeasurementSuffix string - Tags map[string]string - Filter Filter + NameOverride string + MeasurementPrefix string + MeasurementSuffix string + Tags map[string]string + Filter Filter + AlwaysIncludeLocalTags bool } func (r *RunningInput) metricFiltered(metric telegraf.Metric) { @@ -105,12 +106,17 @@ func (r *RunningInput) MakeMetric(metric telegraf.Metric) telegraf.Metric { return nil } + tags := r.Config.Tags + if r.Config.AlwaysIncludeLocalTags { + tags = nil + } + m := makemetric( metric, r.Config.NameOverride, r.Config.MeasurementPrefix, r.Config.MeasurementSuffix, - r.Config.Tags, + tags, r.defaultTags) r.Config.Filter.Modify(metric) @@ -119,6 +125,15 @@ func (r *RunningInput) MakeMetric(metric telegraf.Metric) telegraf.Metric { return nil } + if r.Config.AlwaysIncludeLocalTags { + // Apply plugin tags after filtering + for k, v := range r.Config.Tags { + if _, ok := metric.GetTag(k); !ok { + metric.AddTag(k, v) + } + } + } + r.MetricsGathered.Incr(1) GlobalMetricsGathered.Incr(1) return m diff --git a/models/running_input_test.go b/models/running_input_test.go index 8f9390f53..56d961e17 100644 --- a/models/running_input_test.go +++ b/models/running_input_test.go @@ -273,6 +273,81 @@ func TestMetricErrorCounters(t *testing.T) { require.GreaterOrEqual(t, int64(1), GlobalGatherErrors.Get()) } +func TestMakeMetricWithAlwaysKeepingPluginTagsDisabled(t *testing.T) { + now := time.Now() + ri := NewRunningInput(&testInput{}, &InputConfig{ + Name: "TestRunningInput", + Tags: map[string]string{ + "foo": "bar", + }, + Filter: Filter{ + TagInclude: []string{"b"}, + }, + AlwaysIncludeLocalTags: false, + }) + require.NoError(t, ri.Config.Filter.Compile()) + + m := testutil.MustMetric("RITest", + map[string]string{ + "b": "test", + }, + map[string]interface{}{ + "value": int64(101), + }, + now, + telegraf.Untyped) + m = ri.MakeMetric(m) + + expected := metric.New("RITest", + map[string]string{ + "b": "test", + }, + map[string]interface{}{ + "value": 101, + }, + now, + ) + require.Equal(t, expected, m) +} + +func TestMakeMetricWithAlwaysKeepingPluginTagsEnabled(t *testing.T) { + now := time.Now() + ri := NewRunningInput(&testInput{}, &InputConfig{ + Name: "TestRunningInput", + Tags: map[string]string{ + "foo": "bar", + }, + Filter: Filter{ + TagInclude: []string{"b"}, + }, + AlwaysIncludeLocalTags: true, + }) + require.NoError(t, ri.Config.Filter.Compile()) + + m := testutil.MustMetric("RITest", + map[string]string{ + "b": "test", + }, + map[string]interface{}{ + "value": int64(101), + }, + now, + telegraf.Untyped) + m = ri.MakeMetric(m) + + expected := metric.New("RITest", + map[string]string{ + "b": "test", + "foo": "bar", + }, + map[string]interface{}{ + "value": 101, + }, + now, + ) + require.Equal(t, expected, m) +} + type testInput struct{} func (t *testInput) Description() string { return "" }