chore: Wrap long lines in plugins/parsers (#12180)

This commit is contained in:
Paweł Żak 2022-11-08 18:47:35 +01:00 committed by GitHub
parent 674bea70fb
commit 7807847356
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 207 additions and 44 deletions

View File

@ -1139,7 +1139,11 @@ timestamp,type,name,status
// This should fail when not resetting but reading again due to the header etc // This should fail when not resetting but reading again due to the header etc
_, err = p.Parse([]byte(testCSV)) _, err = p.Parse([]byte(testCSV))
require.Error(t, err, `parsing time "garbage nonsense that needs be skipped" as "2006-01-02T15:04:05Z07:00": cannot parse "garbage nonsense that needs be skipped" as "2006"`) require.Error(
t,
err,
`parsing time "garbage nonsense that needs be skipped" as "2006-01-02T15:04:05Z07:00": cannot parse "garbage nonsense that needs be skipped" as "2006"`,
)
} }
func TestParseCSVLinewiseResetModeNone(t *testing.T) { func TestParseCSVLinewiseResetModeNone(t *testing.T) {
@ -1243,7 +1247,11 @@ func TestParseCSVLinewiseResetModeNone(t *testing.T) {
// This should fail when not resetting but reading again due to the header etc // This should fail when not resetting but reading again due to the header etc
_, err = p.ParseLine(testCSV[0]) _, err = p.ParseLine(testCSV[0])
require.Error(t, err, `parsing time "garbage nonsense that needs be skipped" as "2006-01-02T15:04:05Z07:00": cannot parse "garbage nonsense that needs be skipped" as "2006"`) require.Error(
t,
err,
`parsing time "garbage nonsense that needs be skipped" as "2006-01-02T15:04:05Z07:00": cannot parse "garbage nonsense that needs be skipped" as "2006"`,
)
} }
func TestParseCSVResetModeAlways(t *testing.T) { func TestParseCSVResetModeAlways(t *testing.T) {
@ -1467,5 +1475,9 @@ func TestParseCSVLinewiseResetModeAlways(t *testing.T) {
// This should fail as reset-mode "always" is ignored in line-wise parsing // This should fail as reset-mode "always" is ignored in line-wise parsing
_, err = p.ParseLine(testCSV[0]) _, err = p.ParseLine(testCSV[0])
require.Error(t, err, `parsing time "garbage nonsense that needs be skipped" as "2006-01-02T15:04:05Z07:00": cannot parse "garbage nonsense that needs be skipped" as "2006"`) require.Error(
t,
err,
`parsing time "garbage nonsense that needs be skipped" as "2006-01-02T15:04:05Z07:00": cannot parse "garbage nonsense that needs be skipped" as "2006"`,
)
} }

View File

@ -1,5 +1,6 @@
package grok package grok
//nolint:lll,revive // conditionally long lines allowed
const DefaultPatterns = ` const DefaultPatterns = `
# Example log file pattern, example log looks like this: # Example log file pattern, example log looks like this:
# [04/Jun/2016:12:41:45 +0100] 1.25 200 192.168.1.1 5.432µs # [04/Jun/2016:12:41:45 +0100] 1.25 200 192.168.1.1 5.432µs

View File

@ -104,7 +104,10 @@ func TestCustomInfluxdbHttpd(t *testing.T) {
require.NoError(t, p.Compile()) require.NoError(t, p.Compile())
// Parse an influxdb POST request // Parse an influxdb POST request
m, err := p.ParseLine(`[httpd] ::1 - - [14/Jun/2016:11:33:29 +0100] "POST /write?consistency=any&db=telegraf&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" 6f61bc44-321b-11e6-8050-000000000000 2513`) m, err := p.ParseLine(
`[httpd] ::1 - - [14/Jun/2016:11:33:29 +0100] ` +
`"POST /write?consistency=any&db=telegraf&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" 6f61bc44-321b-11e6-8050-000000000000 2513`,
)
require.NotNil(t, m) require.NotNil(t, m)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, require.Equal(t,
@ -123,18 +126,25 @@ func TestCustomInfluxdbHttpd(t *testing.T) {
require.Equal(t, map[string]string{"verb": "POST", "resp_code": "204"}, m.Tags()) require.Equal(t, map[string]string{"verb": "POST", "resp_code": "204"}, m.Tags())
// Parse an influxdb GET request // Parse an influxdb GET request
m, err = p.ParseLine(`[httpd] ::1 - - [14/Jun/2016:12:10:02 +0100] "GET /query?db=telegraf&q=SELECT+bytes%2Cresponse_time_us+FROM+logparser_grok+WHERE+http_method+%3D+%27GET%27+AND+response_time_us+%3E+0+AND+time+%3E+now%28%29+-+1h HTTP/1.1" 200 578 "http://localhost:8083/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36" 8a3806f1-3220-11e6-8006-000000000000 988`) m, err = p.ParseLine(
`[httpd] ::1 - - [14/Jun/2016:12:10:02 +0100] ` +
`"GET /query?db=telegraf&q=SELECT+bytes%2Cresponse_time_us+FROM+logparser_grok+` +
`WHERE+http_method+%3D+%27GET%27+AND+response_time_us+%3E+0+AND+time+%3E+now%28%29+-+1h HTTP/1.1" 200 578 ` +
`"http://localhost:8083/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) ` +
`Chrome/51.0.2704.84 Safari/537.36" 8a3806f1-3220-11e6-8006-000000000000 988`,
)
require.NotNil(t, m) require.NotNil(t, m)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, require.Equal(t,
map[string]interface{}{ map[string]interface{}{
"resp_bytes": int64(578), "resp_bytes": int64(578),
"auth": "-", "auth": "-",
"client_ip": "::1", "client_ip": "::1",
"http_version": float64(1.1), "http_version": float64(1.1),
"ident": "-", "ident": "-",
"referrer": "http://localhost:8083/", "referrer": "http://localhost:8083/",
"request": "/query?db=telegraf&q=SELECT+bytes%2Cresponse_time_us+FROM+logparser_grok+WHERE+http_method+%3D+%27GET%27+AND+response_time_us+%3E+0+AND+time+%3E+now%28%29+-+1h", "request": "/query?db=telegraf&q=" +
"SELECT+bytes%2Cresponse_time_us+FROM+logparser_grok+WHERE+http_method+%3D+%27GET%27+AND+response_time_us+%3E+0+AND+time+%3E+now%28%29+-+1h",
"response_time_us": int64(988), "response_time_us": int64(988),
"agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36", "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36",
}, },
@ -1092,7 +1102,9 @@ func TestMeasurementModifierNoName(t *testing.T) {
func TestEmptyYearInTimestamp(t *testing.T) { func TestEmptyYearInTimestamp(t *testing.T) {
p := &Parser{ p := &Parser{
Patterns: []string{`%{APPLE_SYSLOG_TIME_SHORT:timestamp:ts-"Jan 2 15:04:05"} %{HOSTNAME} %{APP_NAME:app_name}\[%{NUMBER:pid:int}\]%{GREEDYDATA:message}`}, Patterns: []string{
`%{APPLE_SYSLOG_TIME_SHORT:timestamp:ts-"Jan 2 15:04:05"} %{HOSTNAME} %{APP_NAME:app_name}\[%{NUMBER:pid:int}\]%{GREEDYDATA:message}`,
},
CustomPatterns: ` CustomPatterns: `
APPLE_SYSLOG_TIME_SHORT %{MONTH} +%{MONTHDAY} %{TIME} APPLE_SYSLOG_TIME_SHORT %{MONTH} +%{MONTHDAY} %{TIME}
APP_NAME [a-zA-Z0-9\.]+ APP_NAME [a-zA-Z0-9\.]+

View File

@ -9,7 +9,6 @@ import (
"time" "time"
"github.com/influxdata/line-protocol/v2/lineprotocol" "github.com/influxdata/line-protocol/v2/lineprotocol"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
@ -520,8 +519,21 @@ func parseTests(stream bool) []parseTest {
}, },
}, },
{ {
name: "procstat", name: "procstat",
input: []byte("procstat,exe=bash,process_name=bash voluntary_context_switches=42i,memory_rss=5103616i,rlimit_memory_data_hard=2147483647i,cpu_time_user=0.02,rlimit_file_locks_soft=2147483647i,pid=29417i,cpu_time_nice=0,rlimit_memory_locked_soft=65536i,read_count=259i,rlimit_memory_vms_hard=2147483647i,memory_swap=0i,rlimit_num_fds_soft=1024i,rlimit_nice_priority_hard=0i,cpu_time_soft_irq=0,cpu_time=0i,rlimit_memory_locked_hard=65536i,realtime_priority=0i,signals_pending=0i,nice_priority=20i,cpu_time_idle=0,memory_stack=139264i,memory_locked=0i,rlimit_memory_stack_soft=8388608i,cpu_time_iowait=0,cpu_time_guest=0,cpu_time_guest_nice=0,rlimit_memory_data_soft=2147483647i,read_bytes=0i,rlimit_cpu_time_soft=2147483647i,involuntary_context_switches=2i,write_bytes=106496i,cpu_time_system=0,cpu_time_irq=0,cpu_usage=0,memory_vms=21659648i,memory_data=1576960i,rlimit_memory_stack_hard=2147483647i,num_threads=1i,rlimit_memory_rss_soft=2147483647i,rlimit_realtime_priority_soft=0i,num_fds=4i,write_count=35i,rlimit_signals_pending_soft=78994i,cpu_time_steal=0,rlimit_num_fds_hard=4096i,rlimit_file_locks_hard=2147483647i,rlimit_cpu_time_hard=2147483647i,rlimit_signals_pending_hard=78994i,rlimit_nice_priority_soft=0i,rlimit_memory_rss_hard=2147483647i,rlimit_memory_vms_soft=2147483647i,rlimit_realtime_priority_hard=0i 1517620624000000000"), input: []byte(
"procstat,exe=bash,process_name=bash voluntary_context_switches=42i,memory_rss=5103616i,rlimit_memory_data_hard=2147483647i," +
"cpu_time_user=0.02,rlimit_file_locks_soft=2147483647i,pid=29417i,cpu_time_nice=0,rlimit_memory_locked_soft=65536i," +
"read_count=259i,rlimit_memory_vms_hard=2147483647i,memory_swap=0i,rlimit_num_fds_soft=1024i,rlimit_nice_priority_hard=0i," +
"cpu_time_soft_irq=0,cpu_time=0i,rlimit_memory_locked_hard=65536i,realtime_priority=0i,signals_pending=0i,nice_priority=20i," +
"cpu_time_idle=0,memory_stack=139264i,memory_locked=0i,rlimit_memory_stack_soft=8388608i,cpu_time_iowait=0,cpu_time_guest=0," +
"cpu_time_guest_nice=0,rlimit_memory_data_soft=2147483647i,read_bytes=0i,rlimit_cpu_time_soft=2147483647i," +
"involuntary_context_switches=2i,write_bytes=106496i,cpu_time_system=0,cpu_time_irq=0,cpu_usage=0,memory_vms=21659648i," +
"memory_data=1576960i,rlimit_memory_stack_hard=2147483647i,num_threads=1i,rlimit_memory_rss_soft=2147483647i," +
"rlimit_realtime_priority_soft=0i,num_fds=4i,write_count=35i,rlimit_signals_pending_soft=78994i,cpu_time_steal=0," +
"rlimit_num_fds_hard=4096i,rlimit_file_locks_hard=2147483647i,rlimit_cpu_time_hard=2147483647i,rlimit_signals_pending_hard=78994i," +
"rlimit_nice_priority_soft=0i,rlimit_memory_rss_hard=2147483647i,rlimit_memory_vms_soft=2147483647i,rlimit_realtime_priority_hard=0i " +
"1517620624000000000",
),
metrics: []telegraf.Metric{ metrics: []telegraf.Metric{
metric.New( metric.New(
"procstat", "procstat",
@ -765,9 +777,10 @@ func TestParserErrorString(t *testing.T) {
errString: `metric parse error: field value has unrecognized type at 2:11: "cpu value=invalid"`, errString: `metric parse error: field value has unrecognized type at 2:11: "cpu value=invalid"`,
}, },
{ {
name: "handler error", name: "handler error",
input: []byte("cpu value=9223372036854775808i\ncpu value=42"), input: []byte("cpu value=9223372036854775808i\ncpu value=42"),
errString: `metric parse error: cannot parse value for field key "value": line-protocol value out of range at 1:11: "cpu value=9223372036854775808i"`, errString: `metric parse error: cannot parse value for field key "value": ` +
`line-protocol value out of range at 1:11: "cpu value=9223372036854775808i"`,
}, },
{ {
name: "buffer too long", name: "buffer too long",

View File

@ -1834,7 +1834,18 @@ func BenchmarkMachine(b *testing.B) {
} }
func TestMachineProcstat(_ *testing.T) { func TestMachineProcstat(_ *testing.T) {
input := []byte("procstat,exe=bash,process_name=bash voluntary_context_switches=42i,memory_rss=5103616i,rlimit_memory_data_hard=2147483647i,cpu_time_user=0.02,rlimit_file_locks_soft=2147483647i,pid=29417i,cpu_time_nice=0,rlimit_memory_locked_soft=65536i,read_count=259i,rlimit_memory_vms_hard=2147483647i,memory_swap=0i,rlimit_num_fds_soft=1024i,rlimit_nice_priority_hard=0i,cpu_time_soft_irq=0,cpu_time=0i,rlimit_memory_locked_hard=65536i,realtime_priority=0i,signals_pending=0i,nice_priority=20i,cpu_time_idle=0,memory_stack=139264i,memory_locked=0i,rlimit_memory_stack_soft=8388608i,cpu_time_iowait=0,cpu_time_guest=0,cpu_time_guest_nice=0,rlimit_memory_data_soft=2147483647i,read_bytes=0i,rlimit_cpu_time_soft=2147483647i,involuntary_context_switches=2i,write_bytes=106496i,cpu_time_system=0,cpu_time_irq=0,cpu_usage=0,memory_vms=21659648i,memory_data=1576960i,rlimit_memory_stack_hard=2147483647i,num_threads=1i,rlimit_memory_rss_soft=2147483647i,rlimit_realtime_priority_soft=0i,num_fds=4i,write_count=35i,rlimit_signals_pending_soft=78994i,cpu_time_steal=0,rlimit_num_fds_hard=4096i,rlimit_file_locks_hard=2147483647i,rlimit_cpu_time_hard=2147483647i,rlimit_signals_pending_hard=78994i,rlimit_nice_priority_soft=0i,rlimit_memory_rss_hard=2147483647i,rlimit_memory_vms_soft=2147483647i,rlimit_realtime_priority_hard=0i 1517620624000000000") input := []byte(
"procstat,exe=bash,process_name=bash voluntary_context_switches=42i,memory_rss=5103616i,rlimit_memory_data_hard=2147483647i,cpu_time_user=0.02," +
"rlimit_file_locks_soft=2147483647i,pid=29417i,cpu_time_nice=0,rlimit_memory_locked_soft=65536i,read_count=259i," +
"rlimit_memory_vms_hard=2147483647i,memory_swap=0i,rlimit_num_fds_soft=1024i,rlimit_nice_priority_hard=0i,cpu_time_soft_irq=0,cpu_time=0i," +
"rlimit_memory_locked_hard=65536i,realtime_priority=0i,signals_pending=0i,nice_priority=20i,cpu_time_idle=0,memory_stack=139264i," +
"memory_locked=0i,rlimit_memory_stack_soft=8388608i,cpu_time_iowait=0,cpu_time_guest=0,cpu_time_guest_nice=0,rlimit_memory_data_soft=2147483647i," +
"read_bytes=0i,rlimit_cpu_time_soft=2147483647i,involuntary_context_switches=2i,write_bytes=106496i,cpu_time_system=0,cpu_time_irq=0,cpu_usage=0," +
"memory_vms=21659648i,memory_data=1576960i,rlimit_memory_stack_hard=2147483647i,num_threads=1i,rlimit_memory_rss_soft=2147483647i," +
"rlimit_realtime_priority_soft=0i,num_fds=4i,write_count=35i,rlimit_signals_pending_soft=78994i,cpu_time_steal=0,rlimit_num_fds_hard=4096i," +
"rlimit_file_locks_hard=2147483647i,rlimit_cpu_time_hard=2147483647i,rlimit_signals_pending_hard=78994i,rlimit_nice_priority_soft=0i," +
"rlimit_memory_rss_hard=2147483647i,rlimit_memory_vms_soft=2147483647i,rlimit_realtime_priority_hard=0i 1517620624000000000",
)
handler := &TestingHandler{} handler := &TestingHandler{}
fsm := influx.NewMachine(handler) fsm := influx.NewMachine(handler)
fsm.SetData(input) fsm.SetData(input)
@ -1847,7 +1858,18 @@ func TestMachineProcstat(_ *testing.T) {
} }
func BenchmarkMachineProcstat(b *testing.B) { func BenchmarkMachineProcstat(b *testing.B) {
input := []byte("procstat,exe=bash,process_name=bash voluntary_context_switches=42i,memory_rss=5103616i,rlimit_memory_data_hard=2147483647i,cpu_time_user=0.02,rlimit_file_locks_soft=2147483647i,pid=29417i,cpu_time_nice=0,rlimit_memory_locked_soft=65536i,read_count=259i,rlimit_memory_vms_hard=2147483647i,memory_swap=0i,rlimit_num_fds_soft=1024i,rlimit_nice_priority_hard=0i,cpu_time_soft_irq=0,cpu_time=0i,rlimit_memory_locked_hard=65536i,realtime_priority=0i,signals_pending=0i,nice_priority=20i,cpu_time_idle=0,memory_stack=139264i,memory_locked=0i,rlimit_memory_stack_soft=8388608i,cpu_time_iowait=0,cpu_time_guest=0,cpu_time_guest_nice=0,rlimit_memory_data_soft=2147483647i,read_bytes=0i,rlimit_cpu_time_soft=2147483647i,involuntary_context_switches=2i,write_bytes=106496i,cpu_time_system=0,cpu_time_irq=0,cpu_usage=0,memory_vms=21659648i,memory_data=1576960i,rlimit_memory_stack_hard=2147483647i,num_threads=1i,rlimit_memory_rss_soft=2147483647i,rlimit_realtime_priority_soft=0i,num_fds=4i,write_count=35i,rlimit_signals_pending_soft=78994i,cpu_time_steal=0,rlimit_num_fds_hard=4096i,rlimit_file_locks_hard=2147483647i,rlimit_cpu_time_hard=2147483647i,rlimit_signals_pending_hard=78994i,rlimit_nice_priority_soft=0i,rlimit_memory_rss_hard=2147483647i,rlimit_memory_vms_soft=2147483647i,rlimit_realtime_priority_hard=0i 1517620624000000000") input := []byte(
"procstat,exe=bash,process_name=bash voluntary_context_switches=42i,memory_rss=5103616i,rlimit_memory_data_hard=2147483647i,cpu_time_user=0.02," +
"rlimit_file_locks_soft=2147483647i,pid=29417i,cpu_time_nice=0,rlimit_memory_locked_soft=65536i,read_count=259i," +
"rlimit_memory_vms_hard=2147483647i,memory_swap=0i,rlimit_num_fds_soft=1024i,rlimit_nice_priority_hard=0i,cpu_time_soft_irq=0,cpu_time=0i," +
"rlimit_memory_locked_hard=65536i,realtime_priority=0i,signals_pending=0i,nice_priority=20i,cpu_time_idle=0,memory_stack=139264i," +
"memory_locked=0i,rlimit_memory_stack_soft=8388608i,cpu_time_iowait=0,cpu_time_guest=0,cpu_time_guest_nice=0,rlimit_memory_data_soft=2147483647i," +
"read_bytes=0i,rlimit_cpu_time_soft=2147483647i,involuntary_context_switches=2i,write_bytes=106496i,cpu_time_system=0,cpu_time_irq=0,cpu_usage=0," +
"memory_vms=21659648i,memory_data=1576960i,rlimit_memory_stack_hard=2147483647i,num_threads=1i,rlimit_memory_rss_soft=2147483647i," +
"rlimit_realtime_priority_soft=0i,num_fds=4i,write_count=35i,rlimit_signals_pending_soft=78994i,cpu_time_steal=0,rlimit_num_fds_hard=4096i," +
"rlimit_file_locks_hard=2147483647i,rlimit_cpu_time_hard=2147483647i,rlimit_signals_pending_hard=78994i,rlimit_nice_priority_soft=0i," +
"rlimit_memory_rss_hard=2147483647i,rlimit_memory_vms_soft=2147483647i,rlimit_realtime_priority_hard=0i 1517620624000000000",
)
handler := &BenchmarkingHandler{} handler := &BenchmarkingHandler{}
fsm := influx.NewMachine(handler) fsm := influx.NewMachine(handler)
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {

View File

@ -495,8 +495,20 @@ var ptests = []struct {
}, },
}, },
{ {
name: "procstat", name: "procstat",
input: []byte("procstat,exe=bash,process_name=bash voluntary_context_switches=42i,memory_rss=5103616i,rlimit_memory_data_hard=2147483647i,cpu_time_user=0.02,rlimit_file_locks_soft=2147483647i,pid=29417i,cpu_time_nice=0,rlimit_memory_locked_soft=65536i,read_count=259i,rlimit_memory_vms_hard=2147483647i,memory_swap=0i,rlimit_num_fds_soft=1024i,rlimit_nice_priority_hard=0i,cpu_time_soft_irq=0,cpu_time=0i,rlimit_memory_locked_hard=65536i,realtime_priority=0i,signals_pending=0i,nice_priority=20i,cpu_time_idle=0,memory_stack=139264i,memory_locked=0i,rlimit_memory_stack_soft=8388608i,cpu_time_iowait=0,cpu_time_guest=0,cpu_time_guest_nice=0,rlimit_memory_data_soft=2147483647i,read_bytes=0i,rlimit_cpu_time_soft=2147483647i,involuntary_context_switches=2i,write_bytes=106496i,cpu_time_system=0,cpu_time_irq=0,cpu_usage=0,memory_vms=21659648i,memory_data=1576960i,rlimit_memory_stack_hard=2147483647i,num_threads=1i,rlimit_memory_rss_soft=2147483647i,rlimit_realtime_priority_soft=0i,num_fds=4i,write_count=35i,rlimit_signals_pending_soft=78994i,cpu_time_steal=0,rlimit_num_fds_hard=4096i,rlimit_file_locks_hard=2147483647i,rlimit_cpu_time_hard=2147483647i,rlimit_signals_pending_hard=78994i,rlimit_nice_priority_soft=0i,rlimit_memory_rss_hard=2147483647i,rlimit_memory_vms_soft=2147483647i,rlimit_realtime_priority_hard=0i 1517620624000000000"), input: []byte(
"procstat,exe=bash,process_name=bash voluntary_context_switches=42i,memory_rss=5103616i,rlimit_memory_data_hard=2147483647i,cpu_time_user=0.02," +
"rlimit_file_locks_soft=2147483647i,pid=29417i,cpu_time_nice=0,rlimit_memory_locked_soft=65536i,read_count=259i," +
"rlimit_memory_vms_hard=2147483647i,memory_swap=0i,rlimit_num_fds_soft=1024i,rlimit_nice_priority_hard=0i,cpu_time_soft_irq=0,cpu_time=0i," +
"rlimit_memory_locked_hard=65536i,realtime_priority=0i,signals_pending=0i,nice_priority=20i,cpu_time_idle=0,memory_stack=139264i," +
"memory_locked=0i,rlimit_memory_stack_soft=8388608i,cpu_time_iowait=0,cpu_time_guest=0,cpu_time_guest_nice=0," +
"rlimit_memory_data_soft=2147483647i,read_bytes=0i,rlimit_cpu_time_soft=2147483647i,involuntary_context_switches=2i,write_bytes=106496i," +
"cpu_time_system=0,cpu_time_irq=0,cpu_usage=0,memory_vms=21659648i,memory_data=1576960i,rlimit_memory_stack_hard=2147483647i,num_threads=1i," +
"rlimit_memory_rss_soft=2147483647i,rlimit_realtime_priority_soft=0i,num_fds=4i,write_count=35i,rlimit_signals_pending_soft=78994i," +
"cpu_time_steal=0,rlimit_num_fds_hard=4096i,rlimit_file_locks_hard=2147483647i,rlimit_cpu_time_hard=2147483647i," +
"rlimit_signals_pending_hard=78994i,rlimit_nice_priority_soft=0i,rlimit_memory_rss_hard=2147483647i,rlimit_memory_vms_soft=2147483647i," +
"rlimit_realtime_priority_hard=0i 1517620624000000000",
),
metrics: []telegraf.Metric{ metrics: []telegraf.Metric{
metric.New( metric.New(
"procstat", "procstat",

View File

@ -4,9 +4,10 @@ import (
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/testutil" "github.com/influxdata/telegraf/testutil"
"github.com/stretchr/testify/require"
) )
func TestParse(t *testing.T) { func TestParse(t *testing.T) {
@ -70,8 +71,10 @@ func TestParse(t *testing.T) {
}, },
}, },
{ {
name: "logfmt parser parses every line", name: "logfmt parser parses every line",
bytes: []byte("ts=2018-07-24T19:43:40.275Z lvl=info msg=\"http request\" method=POST\nparent_id=088876RL000 duration=7.45 log_id=09R4e4Rl000"), bytes: []byte(
"ts=2018-07-24T19:43:40.275Z lvl=info msg=\"http request\" method=POST\nparent_id=088876RL000 duration=7.45 log_id=09R4e4Rl000",
),
measurement: "testlog", measurement: "testlog",
want: []telegraf.Metric{ want: []telegraf.Metric{
testutil.MustMetric( testutil.MustMetric(
@ -172,7 +175,8 @@ func TestParseLine(t *testing.T) {
{ {
name: "ParseLine only returns metrics from first string", name: "ParseLine only returns metrics from first string",
measurement: "testlog", measurement: "testlog",
s: "ts=2018-07-24T19:43:35.207268Z lvl=5 msg=\"Write failed\" log_id=09R4e4Rl000\nmethod=POST parent_id=088876RL000 duration=7.45 log_id=09R4e4Rl000", s: "ts=2018-07-24T19:43:35.207268Z lvl=5 msg=\"Write failed\" log_id=09R4e4Rl000\nmethod=POST " +
"parent_id=088876RL000 duration=7.45 log_id=09R4e4Rl000",
want: testutil.MustMetric( want: testutil.MustMetric(
"testlog", "testlog",
map[string]string{}, map[string]string{},

View File

@ -97,7 +97,9 @@ type Parser struct {
// https://github.com/Alignak-monitoring/alignak/blob/develop/alignak/misc/perfdata.py // https://github.com/Alignak-monitoring/alignak/blob/develop/alignak/misc/perfdata.py
var ( var (
perfSplitRegExp = regexp.MustCompile(`([^=]+=\S+)`) perfSplitRegExp = regexp.MustCompile(`([^=]+=\S+)`)
nagiosRegExp = regexp.MustCompile(`^([^=]+)=([\d\.\-\+eE]+)([\w\/%]*);?([\d\.\-\+eE:~@]+)?;?([\d\.\-\+eE:~@]+)?;?([\d\.\-\+eE]+)?;?([\d\.\-\+eE]+)?;?\s*`) nagiosRegExp = regexp.MustCompile(
`^([^=]+)=([\d\.\-\+eE]+)([\w\/%]*);?([\d\.\-\+eE:~@]+)?;?([\d\.\-\+eE:~@]+)?;?([\d\.\-\+eE]+)?;?([\d\.\-\+eE]+)?;?\s*`,
)
) )
func (p *Parser) ParseLine(line string) (telegraf.Metric, error) { func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {

View File

@ -18,6 +18,7 @@ import (
) )
const ( const (
//nolint:lll,revive // conditionally long lines allowed
validUniqueGauge = `# HELP cadvisor_version_info A metric with a constant '1' value labeled by kernel version, OS version, docker version, cadvisor version & cadvisor revision. validUniqueGauge = `# HELP cadvisor_version_info A metric with a constant '1' value labeled by kernel version, OS version, docker version, cadvisor version & cadvisor revision.
# TYPE cadvisor_version_info gauge # TYPE cadvisor_version_info gauge
cadvisor_version_info{cadvisorRevision="",cadvisorVersion="",dockerVersion="1.8.2",kernelVersion="3.10.0-229.20.1.el7.x86_64",osVersion="CentOS Linux 7 (Core)"} 1 cadvisor_version_info{cadvisorRevision="",cadvisorVersion="",dockerVersion="1.8.2",kernelVersion="3.10.0-229.20.1.el7.x86_64",osVersion="CentOS Linux 7 (Core)"} 1
@ -474,7 +475,19 @@ func TestParserProtobufHeader(t *testing.T) {
2, 2,
), ),
} }
sampleProtoBufData := []uint8{67, 10, 9, 115, 119, 97, 112, 95, 102, 114, 101, 101, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108, 101, 99, 116, 101, 100, 32, 109, 101, 116, 114, 105, 99, 24, 1, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 18, 9, 9, 0, 0, 0, 0, 224, 36, 205, 65, 65, 10, 7, 115, 119, 97, 112, 95, 105, 110, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108, 101, 99, 116, 101, 100, 32, 109, 101, 116, 114, 105, 99, 24, 0, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 26, 9, 9, 0, 0, 0, 0, 0, 0, 63, 65, 66, 10, 8, 115, 119, 97, 112, 95, 111, 117, 116, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108, 101, 99, 116, 101, 100, 32, 109, 101, 116, 114, 105, 99, 24, 0, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 26, 9, 9, 0, 0, 0, 0, 0, 30, 110, 65, 68, 10, 10, 115, 119, 97, 112, 95, 116, 111, 116, 97, 108, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108, 101, 99, 116, 101, 100, 32, 109, 101, 116, 114, 105, 99, 24, 1, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 18, 9, 9, 0, 0, 0, 0, 104, 153, 205, 65, 67, 10, 9, 115, 119, 97, 112, 95, 117, 115, 101, 100, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108, 101, 99, 116, 101, 100, 32, 109, 101, 116, 114, 105, 99, 24, 1, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 18, 9, 9, 0, 0, 0, 0, 0, 34, 109, 65, 75, 10, 17, 115, 119, 97, 112, 95, 117, 115, 101, 100, 95, 112, 101, 114, 99, 101, 110, 116, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108, 101, 99, 116, 101, 100, 32, 109, 101, 116, 114, 105, 99, 24, 1, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 18, 9, 9, 109, 234, 180, 197, 37, 155, 248, 63} sampleProtoBufData := []uint8{67, 10, 9, 115, 119, 97, 112, 95, 102, 114, 101, 101, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108,
101, 99, 116, 101, 100, 32, 109, 101, 116, 114, 105, 99, 24, 1, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 18, 9, 9, 0, 0,
0, 0, 224, 36, 205, 65, 65, 10, 7, 115, 119, 97, 112, 95, 105, 110, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108, 101, 99, 116,
101, 100, 32, 109, 101, 116, 114, 105, 99, 24, 0, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 26, 9, 9, 0, 0, 0, 0, 0, 0,
63, 65, 66, 10, 8, 115, 119, 97, 112, 95, 111, 117, 116, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108, 101, 99, 116, 101, 100,
32, 109, 101, 116, 114, 105, 99, 24, 0, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 26, 9, 9, 0, 0, 0, 0, 0, 30, 110, 65,
68, 10, 10, 115, 119, 97, 112, 95, 116, 111, 116, 97, 108, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108, 101, 99, 116, 101,
100, 32, 109, 101, 116, 114, 105, 99, 24, 1, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 18, 9, 9, 0, 0, 0, 0, 104, 153,
205, 65, 67, 10, 9, 115, 119, 97, 112, 95, 117, 115, 101, 100, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102, 32, 99, 111, 108, 108, 101, 99, 116,
101, 100, 32, 109, 101, 116, 114, 105, 99, 24, 1, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109, 115, 107, 18, 9, 9, 0, 0, 0, 0, 0, 34,
109, 65, 75, 10, 17, 115, 119, 97, 112, 95, 117, 115, 101, 100, 95, 112, 101, 114, 99, 101, 110, 116, 18, 25, 84, 101, 108, 101, 103, 114, 97, 102,
32, 99, 111, 108, 108, 101, 99, 116, 101, 100, 32, 109, 101, 116, 114, 105, 99, 24, 1, 34, 25, 10, 12, 10, 4, 104, 111, 115, 116, 18, 4, 111, 109,
115, 107, 18, 9, 9, 109, 234, 180, 197, 37, 155, 248, 63}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited") w.Header().Set("Content-Type", "application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited")
_, err := w.Write(sampleProtoBufData) _, err := w.Write(sampleProtoBufData)

View File

@ -34,7 +34,12 @@ func TestParse(t *testing.T) {
parsedMetrics, err = parser.Parse([]byte("\u0394test.delta 1.234 1530939936 source=\"mysource\" tag2=value2")) parsedMetrics, err = parser.Parse([]byte("\u0394test.delta 1.234 1530939936 source=\"mysource\" tag2=value2"))
require.NoError(t, err) require.NoError(t, err)
testMetric = metric.New("\u0394test.delta", map[string]string{"source": "mysource", "tag2": "value2"}, map[string]interface{}{"value": 1.234}, time.Unix(1530939936, 0)) testMetric = metric.New(
"\u0394test.delta",
map[string]string{"source": "mysource", "tag2": "value2"},
map[string]interface{}{"value": 1.234},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetrics[0], testMetric) require.EqualValues(t, parsedMetrics[0], testMetric)
parsedMetrics, err = parser.Parse([]byte("test.metric 1 1530939936")) parsedMetrics, err = parser.Parse([]byte("test.metric 1 1530939936"))
@ -54,27 +59,52 @@ func TestParse(t *testing.T) {
parsedMetrics, err = parser.Parse([]byte("\"test.metric\" 1.1234 1530939936 \"source\"=\"mysource\" tag2=value2")) parsedMetrics, err = parser.Parse([]byte("\"test.metric\" 1.1234 1530939936 \"source\"=\"mysource\" tag2=value2"))
require.NoError(t, err) require.NoError(t, err)
testMetric = metric.New("test.metric", map[string]string{"source": "mysource", "tag2": "value2"}, map[string]interface{}{"value": 1.1234}, time.Unix(1530939936, 0)) testMetric = metric.New(
"test.metric",
map[string]string{"source": "mysource", "tag2": "value2"},
map[string]interface{}{"value": 1.1234},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetrics[0], testMetric) require.EqualValues(t, parsedMetrics[0], testMetric)
parsedMetrics, err = parser.Parse([]byte("\"test.metric\" -1.1234 1530939936 \"source\"=\"mysource\" tag2=value2")) parsedMetrics, err = parser.Parse([]byte("\"test.metric\" -1.1234 1530939936 \"source\"=\"mysource\" tag2=value2"))
require.NoError(t, err) require.NoError(t, err)
testMetric = metric.New("test.metric", map[string]string{"source": "mysource", "tag2": "value2"}, map[string]interface{}{"value": -1.1234}, time.Unix(1530939936, 0)) testMetric = metric.New(
"test.metric",
map[string]string{"source": "mysource", "tag2": "value2"},
map[string]interface{}{"value": -1.1234},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetrics[0], testMetric) require.EqualValues(t, parsedMetrics[0], testMetric)
parsedMetrics, err = parser.Parse([]byte("\"test.metric\" 1.1234e04 1530939936 \"source\"=\"mysource\" tag2=value2")) parsedMetrics, err = parser.Parse([]byte("\"test.metric\" 1.1234e04 1530939936 \"source\"=\"mysource\" tag2=value2"))
require.NoError(t, err) require.NoError(t, err)
testMetric = metric.New("test.metric", map[string]string{"source": "mysource", "tag2": "value2"}, map[string]interface{}{"value": 1.1234e04}, time.Unix(1530939936, 0)) testMetric = metric.New(
"test.metric",
map[string]string{"source": "mysource", "tag2": "value2"},
map[string]interface{}{"value": 1.1234e04},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetrics[0], testMetric) require.EqualValues(t, parsedMetrics[0], testMetric)
parsedMetrics, err = parser.Parse([]byte("\"test.metric\" 1.1234e-04 1530939936 \"source\"=\"mysource\" tag2=value2")) parsedMetrics, err = parser.Parse([]byte("\"test.metric\" 1.1234e-04 1530939936 \"source\"=\"mysource\" tag2=value2"))
require.NoError(t, err) require.NoError(t, err)
testMetric = metric.New("test.metric", map[string]string{"source": "mysource", "tag2": "value2"}, map[string]interface{}{"value": 1.1234e-04}, time.Unix(1530939936, 0)) testMetric = metric.New(
"test.metric",
map[string]string{"source": "mysource", "tag2": "value2"},
map[string]interface{}{"value": 1.1234e-04},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetrics[0], testMetric) require.EqualValues(t, parsedMetrics[0], testMetric)
parsedMetrics, err = parser.Parse([]byte("test.metric 1.1234 1530939936 source=\"mysource\" tag2=value2 ")) parsedMetrics, err = parser.Parse([]byte("test.metric 1.1234 1530939936 source=\"mysource\" tag2=value2 "))
require.NoError(t, err) require.NoError(t, err)
testMetric = metric.New("test.metric", map[string]string{"source": "mysource", "tag2": "value2"}, map[string]interface{}{"value": 1.1234}, time.Unix(1530939936, 0)) testMetric = metric.New(
"test.metric",
map[string]string{"source": "mysource", "tag2": "value2"},
map[string]interface{}{"value": 1.1234},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetrics[0], testMetric) require.EqualValues(t, parsedMetrics[0], testMetric)
} }
@ -105,12 +135,22 @@ func TestParseLine(t *testing.T) {
parsedMetric, err = parser.ParseLine("\"test.metric\" 1.1234 1530939936 \"source\"=\"mysource\" tag2=value2") parsedMetric, err = parser.ParseLine("\"test.metric\" 1.1234 1530939936 \"source\"=\"mysource\" tag2=value2")
require.NoError(t, err) require.NoError(t, err)
testMetric = metric.New("test.metric", map[string]string{"source": "mysource", "tag2": "value2"}, map[string]interface{}{"value": 1.1234}, time.Unix(1530939936, 0)) testMetric = metric.New(
"test.metric",
map[string]string{"source": "mysource", "tag2": "value2"},
map[string]interface{}{"value": 1.1234},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetric, testMetric) require.EqualValues(t, parsedMetric, testMetric)
parsedMetric, err = parser.ParseLine("test.metric 1.1234 1530939936 source=\"mysource\" tag2=value2 ") parsedMetric, err = parser.ParseLine("test.metric 1.1234 1530939936 source=\"mysource\" tag2=value2 ")
require.NoError(t, err) require.NoError(t, err)
testMetric = metric.New("test.metric", map[string]string{"source": "mysource", "tag2": "value2"}, map[string]interface{}{"value": 1.1234}, time.Unix(1530939936, 0)) testMetric = metric.New(
"test.metric",
map[string]string{"source": "mysource", "tag2": "value2"},
map[string]interface{}{"value": 1.1234},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetric, testMetric) require.EqualValues(t, parsedMetric, testMetric)
} }
@ -134,14 +174,31 @@ func TestParseMultiple(t *testing.T) {
testMetrics = []telegraf.Metric{testMetric1, testMetric2} testMetrics = []telegraf.Metric{testMetric1, testMetric2}
require.EqualValues(t, parsedMetrics, testMetrics) require.EqualValues(t, parsedMetrics, testMetrics)
parsedMetrics, err = parser.Parse([]byte("\"test.metric\" 1.1234 1530939936 \"source\"=\"mysource\" tag2=value2\ntest.metric 1.1234 1530939936 source=\"mysource\" tag2=value2 ")) parsedMetrics, err = parser.Parse(
[]byte(
"\"test.metric\" 1.1234 1530939936 \"source\"=\"mysource\" tag2=value2\n" +
"test.metric 1.1234 1530939936 source=\"mysource\" tag2=value2 ",
),
)
require.NoError(t, err) require.NoError(t, err)
testMetric1 = metric.New("test.metric", map[string]string{"source": "mysource", "tag2": "value2"}, map[string]interface{}{"value": 1.1234}, time.Unix(1530939936, 0)) testMetric1 = metric.New(
testMetric2 = metric.New("test.metric", map[string]string{"source": "mysource", "tag2": "value2"}, map[string]interface{}{"value": 1.1234}, time.Unix(1530939936, 0)) "test.metric",
map[string]string{"source": "mysource", "tag2": "value2"},
map[string]interface{}{"value": 1.1234},
time.Unix(1530939936, 0),
)
testMetric2 = metric.New(
"test.metric",
map[string]string{"source": "mysource", "tag2": "value2"},
map[string]interface{}{"value": 1.1234},
time.Unix(1530939936, 0),
)
testMetrics = []telegraf.Metric{testMetric1, testMetric2} testMetrics = []telegraf.Metric{testMetric1, testMetric2}
require.EqualValues(t, parsedMetrics, testMetrics) require.EqualValues(t, parsedMetrics, testMetrics)
parsedMetrics, err = parser.Parse([]byte("test.metric 1 1530939936 source=mysource\n\"test.metric\" 1.1234 1530939936 source=\"mysource\"\ntest.metric3 333 1530939936 tagit=valueit")) parsedMetrics, err = parser.Parse(
[]byte("test.metric 1 1530939936 source=mysource\n\"test.metric\" 1.1234 1530939936 source=\"mysource\"\ntest.metric3 333 1530939936 tagit=valueit"),
)
require.NoError(t, err) require.NoError(t, err)
testMetric1 = metric.New("test.metric", map[string]string{"source": "mysource"}, map[string]interface{}{"value": 1.}, time.Unix(1530939936, 0)) testMetric1 = metric.New("test.metric", map[string]string{"source": "mysource"}, map[string]interface{}{"value": 1.}, time.Unix(1530939936, 0))
testMetric2 = metric.New("test.metric", map[string]string{"source": "mysource"}, map[string]interface{}{"value": 1.1234}, time.Unix(1530939936, 0)) testMetric2 = metric.New("test.metric", map[string]string{"source": "mysource"}, map[string]interface{}{"value": 1.1234}, time.Unix(1530939936, 0))
@ -204,16 +261,31 @@ func TestParseDefaultTags(t *testing.T) {
parsedMetrics, err := parser.Parse([]byte("test.metric 1 1530939936")) parsedMetrics, err := parser.Parse([]byte("test.metric 1 1530939936"))
require.NoError(t, err) require.NoError(t, err)
testMetric := metric.New("test.metric", map[string]string{"myDefault": "value1", "another": "test2"}, map[string]interface{}{"value": 1.}, time.Unix(1530939936, 0)) testMetric := metric.New(
"test.metric",
map[string]string{"myDefault": "value1", "another": "test2"},
map[string]interface{}{"value": 1.},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetrics[0], testMetric) require.EqualValues(t, parsedMetrics[0], testMetric)
parsedMetrics, err = parser.Parse([]byte("test.metric 1 1530939936 source=mysource")) parsedMetrics, err = parser.Parse([]byte("test.metric 1 1530939936 source=mysource"))
require.NoError(t, err) require.NoError(t, err)
testMetric = metric.New("test.metric", map[string]string{"myDefault": "value1", "another": "test2", "source": "mysource"}, map[string]interface{}{"value": 1.}, time.Unix(1530939936, 0)) testMetric = metric.New(
"test.metric",
map[string]string{"myDefault": "value1", "another": "test2", "source": "mysource"},
map[string]interface{}{"value": 1.},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetrics[0], testMetric) require.EqualValues(t, parsedMetrics[0], testMetric)
parsedMetrics, err = parser.Parse([]byte("\"test.metric\" 1.1234 1530939936 another=\"test3\"")) parsedMetrics, err = parser.Parse([]byte("\"test.metric\" 1.1234 1530939936 another=\"test3\""))
require.NoError(t, err) require.NoError(t, err)
testMetric = metric.New("test.metric", map[string]string{"myDefault": "value1", "another": "test2"}, map[string]interface{}{"value": 1.1234}, time.Unix(1530939936, 0)) testMetric = metric.New(
"test.metric",
map[string]string{"myDefault": "value1", "another": "test2"},
map[string]interface{}{"value": 1.1234},
time.Unix(1530939936, 0),
)
require.EqualValues(t, parsedMetrics[0], testMetric) require.EqualValues(t, parsedMetrics[0], testMetric)
} }