diff --git a/agent/agent.go b/agent/agent.go index 9f9c54714..adf0a545c 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -589,6 +589,7 @@ func (a *Agent) gatherOnce( case <-slowWarning.C: log.Printf("W! [%s] Collection took longer than expected; not complete after interval of %s", input.LogName(), interval) + input.IncrGatherTimeouts() case <-ticker.Elapsed(): log.Printf("D! [%s] Previous collection has not completed; scheduled collection skipped", input.LogName()) diff --git a/models/running_input.go b/models/running_input.go index 34fda9a9a..2f6ea8eae 100644 --- a/models/running_input.go +++ b/models/running_input.go @@ -10,6 +10,7 @@ import ( var ( GlobalMetricsGathered = selfstat.Register("agent", "metrics_gathered", map[string]string{}) GlobalGatherErrors = selfstat.Register("agent", "gather_errors", map[string]string{}) + GlobalGatherTimeouts = selfstat.Register("agent", "gather_timeouts", map[string]string{}) ) type RunningInput struct { @@ -21,6 +22,7 @@ type RunningInput struct { MetricsGathered selfstat.Stat GatherTime selfstat.Stat + GatherTimeouts selfstat.Stat } func NewRunningInput(input telegraf.Input, config *InputConfig) *RunningInput { @@ -50,6 +52,11 @@ func NewRunningInput(input telegraf.Input, config *InputConfig) *RunningInput { "gather_time_ns", tags, ), + GatherTimeouts: selfstat.Register( + "gather", + "gather_timeouts", + tags, + ), log: logger, } } @@ -154,3 +161,8 @@ func (r *RunningInput) SetDefaultTags(tags map[string]string) { func (r *RunningInput) Log() telegraf.Logger { return r.log } + +func (r *RunningInput) IncrGatherTimeouts() { + GlobalGatherTimeouts.Incr(1) + r.GatherTimeouts.Incr(1) +} diff --git a/plugins/inputs/internal/README.md b/plugins/inputs/internal/README.md index d83926eb4..c837eb832 100644 --- a/plugins/inputs/internal/README.md +++ b/plugins/inputs/internal/README.md @@ -47,6 +47,7 @@ agent stats collect aggregate stats on all telegraf plugins. - internal_agent - gather_errors + - gather_timeouts - metrics_dropped - metrics_gathered - metrics_written @@ -58,6 +59,7 @@ that are of the same input type. They are tagged with `input=` - internal_gather - gather_time_ns - metrics_gathered + - gather_timeouts internal_write stats collect aggregate stats on all output plugins that are of the same input type. They are tagged with `output=` @@ -88,10 +90,10 @@ to each particular plugin and with `version=`. ```text internal_memstats,host=tyrion alloc_bytes=4457408i,sys_bytes=10590456i,pointer_lookups=7i,mallocs=17642i,frees=7473i,heap_sys_bytes=6848512i,heap_idle_bytes=1368064i,heap_in_use_bytes=5480448i,heap_released_bytes=0i,total_alloc_bytes=6875560i,heap_alloc_bytes=4457408i,heap_objects_bytes=10169i,num_gc=2i 1480682800000000000 -internal_agent,host=tyrion,go_version=1.12.7,version=1.99.0 metrics_written=18i,metrics_dropped=0i,metrics_gathered=19i,gather_errors=0i 1480682800000000000 +internal_agent,host=tyrion,go_version=1.12.7,version=1.99.0 metrics_written=18i,metrics_dropped=0i,metrics_gathered=19i,gather_errors=0i,gather_timeouts=0i 1480682800000000000 internal_write,output=file,host=tyrion,version=1.99.0 buffer_limit=10000i,write_time_ns=636609i,metrics_added=18i,metrics_written=18i,buffer_size=0i 1480682800000000000 -internal_gather,input=internal,host=tyrion,version=1.99.0 metrics_gathered=19i,gather_time_ns=442114i 1480682800000000000 -internal_gather,input=http_listener,host=tyrion,version=1.99.0 metrics_gathered=0i,gather_time_ns=167285i 1480682800000000000 +internal_gather,input=internal,host=tyrion,version=1.99.0 metrics_gathered=19i,gather_time_ns=442114i,gather_timeouts=0i 1480682800000000000 +internal_gather,input=http_listener,host=tyrion,version=1.99.0 metrics_gathered=0i,gather_time_ns=167285i,gather_timeouts=0i 1480682800000000000 internal_http_listener,address=:8186,host=tyrion,version=1.99.0 queries_received=0i,writes_received=0i,requests_received=0i,buffers_created=0i,requests_served=0i,pings_received=0i,bytes_received=0i,not_founds_served=0i,pings_served=0i,queries_served=0i,writes_served=0i 1480682800000000000 internal_mqtt_consumer,host=tyrion,version=1.99.0 messages_received=622i,payload_size=37942i 1657282270000000000 ```