feat: add ANSI color filter for tail input plugin (#10880)

docs/LICENSE_OF_DEPENDENCIES.md

@@ -202,6 +202,7 @@ following works:
 - github.com/opencontainers/go-digest [Apache License 2.0](https://github.com/opencontainers/go-digest/blob/master/LICENSE)
 - github.com/opencontainers/image-spec [Apache License 2.0](https://github.com/opencontainers/image-spec/blob/master/LICENSE)
 - github.com/opentracing/opentracing-go [Apache License 2.0](https://github.com/opentracing/opentracing-go/blob/master/LICENSE)
+- github.com/pborman/ansi [BSD 3-Clause "New" or "Revised" License](https://github.com/pborman/ansi/blob/master/LICENSE)
 - github.com/philhofer/fwd [MIT License](https://github.com/philhofer/fwd/blob/master/LICENSE.md)
 - github.com/pierrec/lz4 [BSD 3-Clause "New" or "Revised" License](https://github.com/pierrec/lz4/blob/master/LICENSE)
 - github.com/pion/dtls [MIT License](https://github.com/pion/dtls/blob/master/LICENSE)

go.mod

@@ -108,6 +108,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5
 	github.com/openzipkin/zipkin-go v0.2.5
+	github.com/pborman/ansi v1.0.0
 	github.com/pion/dtls/v2 v2.0.13
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.1

go.sum

@@ -1826,6 +1826,8 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
 github.com/pavius/impi v0.0.3/go.mod h1:x/hU0bfdWIhuOT1SKwiJg++yvkk6EuOtJk8WtDZqgr8=
+github.com/pborman/ansi v1.0.0 h1:OqjHMhvlSuCCV5JT07yqPuJPQzQl+WXsiZ14gZsqOrQ=
+github.com/pborman/ansi v1.0.0/go.mod h1:SgWzwMAx1X/Ez7i90VqF8LRiQtx52pWDiQP+x3iGnzw=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=

plugins/inputs/tail/README.md

@@ -67,6 +67,10 @@ The plugin expects messages in one of the
   ## Set the tag that will contain the path of the tailed file. If you don't want this tag, set it to an empty string.
   # path_tag = "path"
 
+  ## Filters to apply to files before generating metrics
+  ## "ansi_color" removes ANSI colors
+  # filters = []
+
   ## multiline parser/codec
   ## https://www.elastic.co/guide/en/logstash/2.4/plugins-filters-multiline.html
   #[inputs.tail.multiline]

plugins/inputs/tail/tail.go

@@ -20,6 +20,7 @@ import (
 	"github.com/influxdata/telegraf/plugins/inputs"
 	"github.com/influxdata/telegraf/plugins/parsers"
 	"github.com/influxdata/telegraf/plugins/parsers/csv"
+	"github.com/pborman/ansi"
 )
 
 const (
@@ -43,6 +44,9 @@ type Tail struct {
 	CharacterEncoding   string   `toml:"character_encoding"`
 	PathTag             string   `toml:"path_tag"`
 
+	Filters      []string `toml:"filters"`
+	filterColors bool
+
 	Log        telegraf.Logger `toml:"-"`
 	tailers    map[string]*tail.Tail
 	offsets    map[string]int64
@@ -121,6 +125,10 @@ const sampleConfig = `
   ## Set the tag that will contain the path of the tailed file. If you don't want this tag, set it to an empty string.
   # path_tag = "path"
 
+  ## Filters to apply to files before generating metrics
+  ## "ansi_color" removes ANSI colors
+  # filters = []
+  
   ## multiline parser/codec
   ## https://www.elastic.co/guide/en/logstash/2.4/plugins-filters-multiline.html
   #[inputs.tail.multiline]
@@ -157,6 +165,12 @@ func (t *Tail) Init() error {
 	}
 	t.sem = make(semaphore, t.MaxUndeliveredLines)
 
+	for _, filter := range t.Filters {
+		if filter == "ansi_color" {
+			t.filterColors = true
+		}
+	}
+
 	var err error
 	t.decoder, err = encoding.NewDecoder(t.CharacterEncoding)
 	return err
@@ -368,6 +382,14 @@ func (t *Tail) receiver(parser parsers.Parser, tailer *tail.Tail) {
 			continue
 		}
 
+		if t.filterColors {
+			out, err := ansi.Strip([]byte(text))
+			if err != nil {
+				t.Log.Errorf("Cannot strip ansi colors from %s: %s", text, err)
+			}
+			text = string(out)
+		}
+
 		metrics, err := parseLine(parser, text)
 		if err != nil {
 			t.Log.Errorf("Malformed log line in %q: [%q]: %s",

plugins/inputs/tail/tail_test.go

@@ -83,6 +83,40 @@ func TestTailBadLine(t *testing.T) {
 	require.Contains(t, buf.String(), "Malformed log line")
 }
 
+func TestColoredLine(t *testing.T) {
+	tmpfile, err := os.CreateTemp("", "")
+	require.NoError(t, err)
+	defer os.Remove(tmpfile.Name())
+	_, err = tmpfile.WriteString("cpu usage_idle=\033[4A\033[4A100\ncpu2 usage_idle=200\n")
+	require.NoError(t, err)
+	require.NoError(t, tmpfile.Close())
+
+	tt := NewTestTail()
+	tt.Log = testutil.Logger{}
+	tt.FromBeginning = true
+	tt.Filters = []string{"ansi_color"}
+	tt.Files = []string{tmpfile.Name()}
+	tt.SetParserFunc(parsers.NewInfluxParser)
+
+	err = tt.Init()
+	require.NoError(t, err)
+
+	acc := testutil.Accumulator{}
+	require.NoError(t, tt.Start(&acc))
+	defer tt.Stop()
+	require.NoError(t, acc.GatherError(tt.Gather))
+
+	acc.Wait(2)
+	acc.AssertContainsFields(t, "cpu",
+		map[string]interface{}{
+			"usage_idle": float64(100),
+		})
+	acc.AssertContainsFields(t, "cpu2",
+		map[string]interface{}{
+			"usage_idle": float64(200),
+		})
+}
+
 func TestTailDosLineEndings(t *testing.T) {
 	tmpfile, err := os.CreateTemp("", "")
 	require.NoError(t, err)