feat(internal): Add gather_timeouts metric (#13471)

agent/agent.go

@@ -589,6 +589,7 @@ func (a *Agent) gatherOnce(
 		case <-slowWarning.C:
 			log.Printf("W! [%s] Collection took longer than expected; not complete after interval of %s",
 				input.LogName(), interval)
+			input.IncrGatherTimeouts()
 		case <-ticker.Elapsed():
 			log.Printf("D! [%s] Previous collection has not completed; scheduled collection skipped",
 				input.LogName())

models/running_input.go

@@ -10,6 +10,7 @@ import (
 var (
 	GlobalMetricsGathered = selfstat.Register("agent", "metrics_gathered", map[string]string{})
 	GlobalGatherErrors    = selfstat.Register("agent", "gather_errors", map[string]string{})
+	GlobalGatherTimeouts  = selfstat.Register("agent", "gather_timeouts", map[string]string{})
 )
 
 type RunningInput struct {
@@ -21,6 +22,7 @@ type RunningInput struct {
 
 	MetricsGathered selfstat.Stat
 	GatherTime      selfstat.Stat
+	GatherTimeouts  selfstat.Stat
 }
 
 func NewRunningInput(input telegraf.Input, config *InputConfig) *RunningInput {
@@ -50,6 +52,11 @@ func NewRunningInput(input telegraf.Input, config *InputConfig) *RunningInput {
 			"gather_time_ns",
 			tags,
 		),
+		GatherTimeouts: selfstat.Register(
+			"gather",
+			"gather_timeouts",
+			tags,
+		),
 		log: logger,
 	}
 }
@@ -154,3 +161,8 @@ func (r *RunningInput) SetDefaultTags(tags map[string]string) {
 func (r *RunningInput) Log() telegraf.Logger {
 	return r.log
 }
+
+func (r *RunningInput) IncrGatherTimeouts() {
+	GlobalGatherTimeouts.Incr(1)
+	r.GatherTimeouts.Incr(1)
+}

plugins/inputs/internal/README.md

@@ -47,6 +47,7 @@ agent stats collect aggregate stats on all telegraf plugins.
 
 - internal_agent
   - gather_errors
+  - gather_timeouts
   - metrics_dropped
   - metrics_gathered
   - metrics_written
@@ -58,6 +59,7 @@ that are of the same input type. They are tagged with `input=<plugin_name>`
 - internal_gather
   - gather_time_ns
   - metrics_gathered
+  - gather_timeouts
 
 internal_write stats collect aggregate stats on all output plugins
 that are of the same input type. They are tagged with `output=<plugin_name>`
@@ -88,10 +90,10 @@ to each particular plugin and with `version=<telegraf_version>`.
 
 ```text
 internal_memstats,host=tyrion alloc_bytes=4457408i,sys_bytes=10590456i,pointer_lookups=7i,mallocs=17642i,frees=7473i,heap_sys_bytes=6848512i,heap_idle_bytes=1368064i,heap_in_use_bytes=5480448i,heap_released_bytes=0i,total_alloc_bytes=6875560i,heap_alloc_bytes=4457408i,heap_objects_bytes=10169i,num_gc=2i 1480682800000000000
-internal_agent,host=tyrion,go_version=1.12.7,version=1.99.0 metrics_written=18i,metrics_dropped=0i,metrics_gathered=19i,gather_errors=0i 1480682800000000000
+internal_agent,host=tyrion,go_version=1.12.7,version=1.99.0 metrics_written=18i,metrics_dropped=0i,metrics_gathered=19i,gather_errors=0i,gather_timeouts=0i 1480682800000000000
 internal_write,output=file,host=tyrion,version=1.99.0 buffer_limit=10000i,write_time_ns=636609i,metrics_added=18i,metrics_written=18i,buffer_size=0i 1480682800000000000
-internal_gather,input=internal,host=tyrion,version=1.99.0 metrics_gathered=19i,gather_time_ns=442114i 1480682800000000000
-internal_gather,input=http_listener,host=tyrion,version=1.99.0 metrics_gathered=0i,gather_time_ns=167285i 1480682800000000000
+internal_gather,input=internal,host=tyrion,version=1.99.0 metrics_gathered=19i,gather_time_ns=442114i,gather_timeouts=0i 1480682800000000000
+internal_gather,input=http_listener,host=tyrion,version=1.99.0 metrics_gathered=0i,gather_time_ns=167285i,gather_timeouts=0i 1480682800000000000
 internal_http_listener,address=:8186,host=tyrion,version=1.99.0 queries_received=0i,writes_received=0i,requests_received=0i,buffers_created=0i,requests_served=0i,pings_received=0i,bytes_received=0i,not_founds_served=0i,pings_served=0i,queries_served=0i,writes_served=0i 1480682800000000000
 internal_mqtt_consumer,host=tyrion,version=1.99.0 messages_received=622i,payload_size=37942i 1657282270000000000
 ```