diff --git a/plugins/inputs/file/file_test.go b/plugins/inputs/file/file_test.go index 27c2add99..8eb0c7cb0 100644 --- a/plugins/inputs/file/file_test.go +++ b/plugins/inputs/file/file_test.go @@ -15,8 +15,8 @@ import ( "github.com/stretchr/testify/require" "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/plugins/parsers" "github.com/influxdata/telegraf/plugins/parsers/csv" + "github.com/influxdata/telegraf/plugins/parsers/grok" "github.com/influxdata/telegraf/plugins/parsers/json" "github.com/influxdata/telegraf/testutil" ) @@ -90,12 +90,15 @@ func TestGrokParser(t *testing.T) { err := r.Init() require.NoError(t, err) - parserConfig := parsers.Config{ - DataFormat: "grok", - GrokPatterns: []string{"%{COMMON_LOG_FORMAT}"}, - } + r.SetParserFunc(func() (telegraf.Parser, error) { + parser := &grok.Parser{ + Patterns: []string{"%{COMMON_LOG_FORMAT}"}, + Log: testutil.Logger{}, + } + err := parser.Init() - r.SetParserFunc(func() (telegraf.Parser, error) { return parsers.NewParser(&parserConfig) }) + return parser, err + }) err = r.Gather(&acc) require.NoError(t, err) diff --git a/plugins/inputs/logparser/logparser.go b/plugins/inputs/logparser/logparser.go index 081f16ab5..d297393a9 100644 --- a/plugins/inputs/logparser/logparser.go +++ b/plugins/inputs/logparser/logparser.go @@ -17,6 +17,7 @@ import ( "github.com/influxdata/telegraf/models" "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/parsers" + "github.com/influxdata/telegraf/plugins/parsers/grok" ) // DO NOT REMOVE THE NEXT TWO LINES! This is required to embed the sampleConfig data. @@ -117,22 +118,20 @@ func (l *LogParserPlugin) Start(acc telegraf.Accumulator) error { } // Looks for fields which implement LogParser interface - config := &parsers.Config{ - MetricName: mName, - GrokPatterns: l.GrokConfig.Patterns, - GrokNamedPatterns: l.GrokConfig.NamedPatterns, - GrokCustomPatterns: l.GrokConfig.CustomPatterns, - GrokCustomPatternFiles: l.GrokConfig.CustomPatternFiles, - GrokTimezone: l.GrokConfig.Timezone, - GrokUniqueTimestamp: l.GrokConfig.UniqueTimestamp, - DataFormat: "grok", + parser := grok.Parser{ + Measurement: mName, + Patterns: l.GrokConfig.Patterns, + NamedPatterns: l.GrokConfig.NamedPatterns, + CustomPatterns: l.GrokConfig.CustomPatterns, + CustomPatternFiles: l.GrokConfig.CustomPatternFiles, + Timezone: l.GrokConfig.Timezone, + UniqueTimestamp: l.GrokConfig.UniqueTimestamp, } - - var err error - l.GrokParser, err = parsers.NewParser(config) + err := parser.Init() if err != nil { return err } + l.GrokParser = &parser models.SetLoggerOnPlugin(l.GrokParser, l.Log) l.wg.Add(1) diff --git a/plugins/inputs/logparser/logparser_test.go b/plugins/inputs/logparser/logparser_test.go index f9c094191..58d74cb09 100644 --- a/plugins/inputs/logparser/logparser_test.go +++ b/plugins/inputs/logparser/logparser_test.go @@ -24,7 +24,7 @@ func TestStartNoParsers(t *testing.T) { } acc := testutil.Accumulator{} - require.Error(t, logparser.Start(&acc)) + require.NoError(t, logparser.Start(&acc)) } func TestGrokParseLogFilesNonExistPattern(t *testing.T) { diff --git a/plugins/inputs/tail/tail_test.go b/plugins/inputs/tail/tail_test.go index 7494de6f2..44f11fab6 100644 --- a/plugins/inputs/tail/tail_test.go +++ b/plugins/inputs/tail/tail_test.go @@ -15,6 +15,7 @@ import ( "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/plugins/parsers" "github.com/influxdata/telegraf/plugins/parsers/csv" + "github.com/influxdata/telegraf/plugins/parsers/grok" "github.com/influxdata/telegraf/plugins/parsers/influx" "github.com/influxdata/telegraf/plugins/parsers/json" "github.com/influxdata/telegraf/testutil" @@ -306,13 +307,13 @@ func TestGrokParseLogFilesWithMultilineTailerCloseFlushesMultilineBuffer(t *test } func createGrokParser() (parsers.Parser, error) { - grokConfig := &parsers.Config{ - MetricName: "tail_grok", - GrokPatterns: []string{"%{TEST_LOG_MULTILINE}"}, - GrokCustomPatternFiles: []string{filepath.Join(testdataDir, "test-patterns")}, - DataFormat: "grok", + parser := &grok.Parser{ + Measurement: "tail_grok", + Patterns: []string{"%{TEST_LOG_MULTILINE}"}, + CustomPatternFiles: []string{filepath.Join(testdataDir, "test-patterns")}, + Log: testutil.Logger{}, } - parser, err := parsers.NewParser(grokConfig) + err := parser.Init() return parser, err } diff --git a/plugins/parsers/all/all.go b/plugins/parsers/all/all.go index f9eec51ea..441d53866 100644 --- a/plugins/parsers/all/all.go +++ b/plugins/parsers/all/all.go @@ -7,6 +7,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/parsers/dropwizard" _ "github.com/influxdata/telegraf/plugins/parsers/form_urlencoded" _ "github.com/influxdata/telegraf/plugins/parsers/graphite" + _ "github.com/influxdata/telegraf/plugins/parsers/grok" _ "github.com/influxdata/telegraf/plugins/parsers/json" _ "github.com/influxdata/telegraf/plugins/parsers/json_v2" _ "github.com/influxdata/telegraf/plugins/parsers/logfmt" diff --git a/plugins/parsers/grok/parser.go b/plugins/parsers/grok/parser.go index 026785f03..3f829e322 100644 --- a/plugins/parsers/grok/parser.go +++ b/plugins/parsers/grok/parser.go @@ -14,6 +14,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/metric" + "github.com/influxdata/telegraf/plugins/parsers" ) var timeLayouts = map[string]string{ @@ -66,17 +67,17 @@ var ( // Parser is the primary struct to handle and grok-patterns defined in the config toml type Parser struct { - Patterns []string + Patterns []string `toml:"grok_patterns"` // namedPatterns is a list of internally-assigned names to the patterns // specified by the user in Patterns. // They will look like: // GROK_INTERNAL_PATTERN_0, GROK_INTERNAL_PATTERN_1, etc. - NamedPatterns []string - CustomPatterns string - CustomPatternFiles []string - Measurement string - DefaultTags map[string]string - Log telegraf.Logger `toml:"-"` + NamedPatterns []string `toml:"grok_named_patterns"` + CustomPatterns string `toml:"grok_custom_patterns"` + CustomPatternFiles []string `toml:"grok_custom_pattern_files"` + Measurement string `toml:"-"` + DefaultTags map[string]string `toml:"-"` + Log telegraf.Logger `toml:"-"` // Timezone is an optional component to help render log dates to // your chosen zone. @@ -85,11 +86,11 @@ type Parser struct { // 1. Local -- interpret based on machine localtime // 2. "America/Chicago" -- Unix TZ values like those found in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones // 3. UTC -- or blank/unspecified, will return timestamp in UTC - Timezone string + Timezone string `toml:"grok_timezone"` loc *time.Location // UniqueTimestamp when set to "disable", timestamp will not incremented if there is a duplicate. - UniqueTimestamp string + UniqueTimestamp string `toml:"grok_unique_timestamp"` // typeMap is a map of patterns -> capture name -> modifier, // ie, { @@ -558,3 +559,43 @@ func (t *tsModder) tsMod(ts time.Time) time.Time { } return ts.Add(t.incr*t.incrn + t.rollover) } + +// InitFromConfig is a compatibility function to construct the parser the old way +func (p *Parser) InitFromConfig(config *parsers.Config) error { + p.Measurement = config.MetricName + p.DefaultTags = config.DefaultTags + p.CustomPatterns = config.GrokCustomPatterns + p.CustomPatternFiles = config.GrokCustomPatternFiles + p.NamedPatterns = config.GrokNamedPatterns + p.Patterns = config.GrokPatterns + p.Timezone = config.GrokTimezone + p.UniqueTimestamp = config.GrokUniqueTimestamp + + return p.Init() +} + +func (p *Parser) Init() error { + if len(p.Patterns) == 0 { + p.Patterns = []string{"%{COMBINED_LOG_FORMAT}"} + } + + if p.UniqueTimestamp == "" { + p.UniqueTimestamp = "auto" + } + + if p.Timezone == "" { + p.Timezone = "Canada/Eastern" + } + + return p.Compile() +} + +func init() { + parsers.Add("grok", + func(defaultMetricName string) telegraf.Parser { + return &Parser{ + Measurement: defaultMetricName, + } + }, + ) +} diff --git a/plugins/parsers/registry.go b/plugins/parsers/registry.go index 368b34ee0..eef89f049 100644 --- a/plugins/parsers/registry.go +++ b/plugins/parsers/registry.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/plugins/parsers/grok" "github.com/influxdata/telegraf/plugins/parsers/influx" "github.com/influxdata/telegraf/plugins/parsers/influx/influx_upstream" "github.com/influxdata/telegraf/plugins/parsers/nagios" @@ -206,15 +205,6 @@ func NewParser(config *Config) (Parser, error) { } case "nagios": parser, err = NewNagiosParser() - case "grok": - parser, err = newGrokParser( - config.MetricName, - config.GrokPatterns, - config.GrokNamedPatterns, - config.GrokCustomPatterns, - config.GrokCustomPatternFiles, - config.GrokTimezone, - config.GrokUniqueTimestamp) case "prometheus": parser, err = NewPrometheusParser( config.DefaultTags, @@ -240,24 +230,6 @@ func NewParser(config *Config) (Parser, error) { return parser, err } -func newGrokParser(metricName string, - patterns []string, nPatterns []string, - cPatterns string, cPatternFiles []string, - tZone string, uniqueTimestamp string) (Parser, error) { - parser := grok.Parser{ - Measurement: metricName, - Patterns: patterns, - NamedPatterns: nPatterns, - CustomPatterns: cPatterns, - CustomPatternFiles: cPatternFiles, - Timezone: tZone, - UniqueTimestamp: uniqueTimestamp, - } - - err := parser.Compile() - return &parser, err -} - func NewNagiosParser() (Parser, error) { return &nagios.NagiosParser{}, nil }