diff --git a/plugins/inputs/tail/README.md b/plugins/inputs/tail/README.md index 4163bc2bf..4bd2866de 100644 --- a/plugins/inputs/tail/README.md +++ b/plugins/inputs/tail/README.md @@ -104,6 +104,11 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ## When handling quotes, escaped quotes (e.g. \") are handled correctly. #quotation = "ignore" + ## The preserve_newline option can be true or false (defaults to false). + ## If true, the newline character is preserved for multiline elements, + ## this is useful to preserve message-structure e.g. for logging outputs. + #preserver_newline = false + #After the specified timeout, this plugin sends the multiline event even if no new pattern is found to start a new event. The default is 5s. #timeout = 5s ``` diff --git a/plugins/inputs/tail/multiline.go b/plugins/inputs/tail/multiline.go index 2b03aa11a..a774d4871 100644 --- a/plugins/inputs/tail/multiline.go +++ b/plugins/inputs/tail/multiline.go @@ -23,11 +23,12 @@ type Multiline struct { } type MultilineConfig struct { - Pattern string `toml:"pattern"` - MatchWhichLine MultilineMatchWhichLine `toml:"match_which_line"` - InvertMatch bool `toml:"invert_match"` - Quotation string `toml:"quotation"` - Timeout *config.Duration `toml:"timeout"` + Pattern string `toml:"pattern"` + MatchWhichLine MultilineMatchWhichLine `toml:"match_which_line"` + InvertMatch bool `toml:"invert_match"` + PreserveNewline bool `toml:"preserve_newline"` + Quotation string `toml:"quotation"` + Timeout *config.Duration `toml:"timeout"` } const ( @@ -80,12 +81,11 @@ func (m *Multiline) IsEnabled() bool { } func (m *Multiline) ProcessLine(text string, buffer *bytes.Buffer) string { - if m.matchQuotation(text) { - // Ignore the returned error as we cannot do anything about it anyway - _, _ = buffer.WriteString(text + "\n") - return "" - } - if m.matchString(text) { + if m.matchQuotation(text) || m.matchString(text) { + // Restore the newline removed by tail's scanner + if buffer.Len() > 0 && m.config.PreserveNewline { + _, _ = buffer.WriteString("\n") + } // Ignore the returned error as we cannot do anything about it anyway _, _ = buffer.WriteString(text) return "" @@ -101,6 +101,9 @@ func (m *Multiline) ProcessLine(text string, buffer *bytes.Buffer) string { } else { // Next if buffer.Len() > 0 { + if m.config.PreserveNewline { + _, _ = buffer.WriteString("\n") + } if _, err := buffer.WriteString(text); err != nil { return "" } diff --git a/plugins/inputs/tail/multiline_test.go b/plugins/inputs/tail/multiline_test.go index 936b5df47..5992481c5 100644 --- a/plugins/inputs/tail/multiline_test.go +++ b/plugins/inputs/tail/multiline_test.go @@ -239,7 +239,7 @@ func TestMultilineWhat(t *testing.T) { require.Equal(t, MultilineMatchWhichLine(-1), w7) } -func TestMultiLineQuoted(t *testing.T) { +func TestMultilineQuoted(t *testing.T) { tests := []struct { name string quotation string @@ -281,8 +281,9 @@ func TestMultiLineQuoted(t *testing.T) { } c := &MultilineConfig{ - MatchWhichLine: Next, - Quotation: tt.quotation, + MatchWhichLine: Next, + Quotation: tt.quotation, + PreserveNewline: true, } m, err := c.NewMultiline() require.NoError(t, err) @@ -303,13 +304,16 @@ func TestMultiLineQuoted(t *testing.T) { } result = append(result, text) } + if text := m.Flush(&buffer); text != "" { + result = append(result, text) + } require.EqualValues(t, expected, result) }) } } -func TestMultiLineQuotedError(t *testing.T) { +func TestMultilineQuotedError(t *testing.T) { tests := []struct { name string filename string @@ -332,15 +336,16 @@ func TestMultiLineQuotedError(t *testing.T) { filename: "multiline_quoted_missing_close.csv", quotation: "single-quotes", quote: `'`, - expected: nil, + expected: []string{"1660819827411,2,'some text all quoted,B\n1660819827410,1,some text without quotes,A"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &MultilineConfig{ - MatchWhichLine: Next, - Quotation: tt.quotation, + MatchWhichLine: Next, + Quotation: tt.quotation, + PreserveNewline: true, } m, err := c.NewMultiline() require.NoError(t, err) @@ -361,6 +366,84 @@ func TestMultiLineQuotedError(t *testing.T) { } result = append(result, text) } + if text := m.Flush(&buffer); text != "" { + result = append(result, text) + } + + require.EqualValues(t, tt.expected, result) + }) + } +} + +func TestMultilineNewline(t *testing.T) { + tests := []struct { + name string + filename string + cfg *MultilineConfig + expected []string + }{ + { + name: "do not preserve newline", + cfg: &MultilineConfig{ + Pattern: `\[[0-9]{2}/[A-Za-z]{3}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} \+[0-9]{4}\]`, + InvertMatch: true, + }, + filename: "test_multiline.log", + expected: []string{ + `[04/Jun/2016:12:41:45 +0100] DEBUG HelloExample: This is debug`, + `[04/Jun/2016:12:41:48 +0100] INFO HelloExample: This is info`, + "[04/Jun/2016:12:41:46 +0100] ERROR HelloExample: Sorry, something wrong! " + + "java.lang.ArithmeticException: / by zero" + + "\tat com.foo.HelloExample2.divide(HelloExample2.java:24)" + + "\tat com.foo.HelloExample2.main(HelloExample2.java:14)", + `[04/Jun/2016:12:41:48 +0100] WARN HelloExample: This is warn`, + }, + }, + { + name: "preserve newline", + cfg: &MultilineConfig{ + Pattern: `\[[0-9]{2}/[A-Za-z]{3}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} \+[0-9]{4}\]`, + InvertMatch: true, + PreserveNewline: true, + }, + filename: "test_multiline.log", + expected: []string{ + `[04/Jun/2016:12:41:45 +0100] DEBUG HelloExample: This is debug`, + `[04/Jun/2016:12:41:48 +0100] INFO HelloExample: This is info`, + `[04/Jun/2016:12:41:46 +0100] ERROR HelloExample: Sorry, something wrong!` + ` ` + ` +java.lang.ArithmeticException: / by zero + at com.foo.HelloExample2.divide(HelloExample2.java:24) + at com.foo.HelloExample2.main(HelloExample2.java:14)`, + `[04/Jun/2016:12:41:48 +0100] WARN HelloExample: This is warn`, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, err := tt.cfg.NewMultiline() + require.NoError(t, err) + + f, err := os.Open(filepath.Join("testdata", tt.filename)) + require.NoError(t, err) + + scanner := bufio.NewScanner(f) + + var buffer bytes.Buffer + var result []string + for scanner.Scan() { + line := scanner.Text() + + text := m.ProcessLine(line, &buffer) + if text == "" { + continue + } + result = append(result, text) + } + if text := m.Flush(&buffer); text != "" { + result = append(result, text) + } + require.EqualValues(t, tt.expected, result) }) } @@ -368,9 +451,10 @@ func TestMultiLineQuotedError(t *testing.T) { func TestMultiLineQuotedAndPattern(t *testing.T) { c := &MultilineConfig{ - Pattern: "=>$", - MatchWhichLine: Next, - Quotation: "double-quotes", + Pattern: "=>$", + MatchWhichLine: Next, + Quotation: "double-quotes", + PreserveNewline: true, } m, err := c.NewMultiline() require.NoError(t, err, "Configuration was OK.") @@ -397,7 +481,7 @@ func TestMultiLineQuotedAndPattern(t *testing.T) { require.NotZero(t, buffer.Len()) text = m.ProcessLine("4", &buffer) - require.Equal(t, "1=>2=>\"a quoted\nmultiline string\"=>3=>4", text) + require.Equal(t, "1=>\n2=>\n\"a quoted\nmultiline string\"=>\n3=>\n4", text) require.Zero(t, buffer.Len()) text = m.ProcessLine("5", &buffer) diff --git a/plugins/inputs/tail/sample.conf b/plugins/inputs/tail/sample.conf index 4a71cb846..28d3daa50 100644 --- a/plugins/inputs/tail/sample.conf +++ b/plugins/inputs/tail/sample.conf @@ -71,5 +71,10 @@ ## When handling quotes, escaped quotes (e.g. \") are handled correctly. #quotation = "ignore" + ## The preserve_newline option can be true or false (defaults to false). + ## If true, the newline character is preserved for multiline elements, + ## this is useful to preserve message-structure e.g. for logging outputs. + #preserver_newline = false + #After the specified timeout, this plugin sends the multiline event even if no new pattern is found to start a new event. The default is 5s. #timeout = 5s