Add state and readiness to kube_inventory pod metrics (#7691)

plugins/inputs/kube_inventory/README.md

@@ -66,6 +66,9 @@ avoid cardinality issues:
   # tls_key = "/path/to/keyfile"
   ## Use TLS but skip chain & host verification
   # insecure_skip_verify = false
+
+  ## Uncomment to remove deprecated metrics.
+  # fielddrop = ["terminated_reason"]
 ```
 
 #### Kubernetes Permissions
@@ -206,10 +209,13 @@ subjects:
     - namespace
     - node_name
     - pod_name
+    - state
+    - readiness
   - fields:
     - restarts_total
-    - state
+    - state_code
-    - terminated_reason
+    - state_reason
+    - terminated_reason (string, deprecated in 1.15: use `state_reason` instead)
     - resource_requests_cpu_units
     - resource_requests_memory_bytes
     - resource_limits_cpu_units
@@ -267,7 +273,6 @@ The persistentvolumeclaim "phase" is saved in the `phase` tag with a correlated
 | pending   | 2                         |
 | unknown   | 3                         |
 
-
 ### Example Output:
 
 ```
@@ -278,7 +283,7 @@ kubernetes_node,node_name=ip-172-17-0-2.internal allocatable_pods=110i,capacity_
 kubernetes_persistentvolume,phase=Released,pv_name=pvc-aaaaaaaa-bbbb-cccc-1111-222222222222,storageclass=ebs-1-retain phase_type=3i 1547597616000000000
 kubernetes_persistentvolumeclaim,namespace=default,phase=Bound,pvc_name=data-etcd-0,storageclass=ebs-1-retain phase_type=0i 1547597615000000000
 kubernetes_pod,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1 last_transition_time=1547578322000000000i,ready="false" 1547597616000000000
-kubernetes_pod_container,container_name=telegraf,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1,state=running resource_requests_cpu_units=0.1,resource_limits_memory_bytes=524288000,resource_limits_cpu_units=0.5,restarts_total=0i,state_code=0i,terminated_reason="",resource_requests_memory_bytes=524288000 1547597616000000000
+kubernetes_pod_container,container_name=telegraf,namespace=default,node_name=ip-172-17-0-2.internal,pod_name=tick1,state=running,readiness=ready resource_requests_cpu_units=0.1,resource_limits_memory_bytes=524288000,resource_limits_cpu_units=0.5,restarts_total=0i,state_code=0i,state_reason="",resource_requests_memory_bytes=524288000 1547597616000000000
 kubernetes_statefulset,namespace=default,statefulset_name=etcd replicas_updated=3i,spec_replicas=3i,observed_generation=1i,created=1544101669000000000i,generation=1i,replicas=3i,replicas_current=3i,replicas_ready=3i 1547597616000000000
 ```
 

plugins/inputs/kube_inventory/pod.go

@@ -3,7 +3,7 @@ package kube_inventory
 import (
 	"context"
 
-	"github.com/ericchiang/k8s/apis/core/v1"
+	v1 "github.com/ericchiang/k8s/apis/core/v1"
 
 	"github.com/influxdata/telegraf"
 )
@@ -37,7 +37,9 @@ func (ki *KubernetesInventory) gatherPod(p v1.Pod, acc telegraf.Accumulator) err
 
 func gatherPodContainer(nodeName string, p v1.Pod, cs v1.ContainerStatus, c v1.Container, acc telegraf.Accumulator) {
 	stateCode := 3
+	stateReason := ""
 	state := "unknown"
+
 	switch {
 	case cs.State.Running != nil:
 		stateCode = 0
@@ -45,9 +47,16 @@ func gatherPodContainer(nodeName string, p v1.Pod, cs v1.ContainerStatus, c v1.C
 	case cs.State.Terminated != nil:
 		stateCode = 1
 		state = "terminated"
+		stateReason = cs.State.Terminated.GetReason()
 	case cs.State.Waiting != nil:
 		stateCode = 2
 		state = "waiting"
+		stateReason = cs.State.Waiting.GetReason()
+	}
+
+	readiness := "unready"
+	if cs.GetReady() {
+		readiness = "ready"
 	}
 
 	fields := map[string]interface{}{
@@ -55,12 +64,18 @@ func gatherPodContainer(nodeName string, p v1.Pod, cs v1.ContainerStatus, c v1.C
 		"state_code":        stateCode,
 		"terminated_reason": cs.State.Terminated.GetReason(),
 	}
+
+	if stateReason != "" {
+		fields["state_reason"] = stateReason
+	}
+
 	tags := map[string]string{
 		"container_name": *c.Name,
 		"namespace":      *p.Metadata.Namespace,
 		"node_name":      *p.Spec.NodeName,
 		"pod_name":       *p.Metadata.Name,
 		"state":          state,
+		"readiness":      readiness,
 	}
 
 	req := c.Resources.Requests

plugins/inputs/kube_inventory/pod_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/ericchiang/k8s/apis/core/v1"
+	v1 "github.com/ericchiang/k8s/apis/core/v1"
 	metav1 "github.com/ericchiang/k8s/apis/meta/v1"
 	"github.com/ericchiang/k8s/apis/resource"
 	"github.com/influxdata/telegraf/testutil"
@@ -44,7 +44,43 @@ func TestPod(t *testing.T) {
 									NodeName: toStrPtr("node1"),
 									Containers: []*v1.Container{
 										{
-											Name:  toStrPtr("forwarder"),
+											Name:  toStrPtr("running"),
+											Image: toStrPtr("image1"),
+											Ports: []*v1.ContainerPort{
+												{
+													ContainerPort: toInt32Ptr(8080),
+													Protocol:      toStrPtr("TCP"),
+												},
+											},
+											Resources: &v1.ResourceRequirements{
+												Limits: map[string]*resource.Quantity{
+													"cpu": {String_: toStrPtr("100m")},
+												},
+												Requests: map[string]*resource.Quantity{
+													"cpu": {String_: toStrPtr("100m")},
+												},
+											},
+										},
+										{
+											Name:  toStrPtr("completed"),
+											Image: toStrPtr("image1"),
+											Ports: []*v1.ContainerPort{
+												{
+													ContainerPort: toInt32Ptr(8080),
+													Protocol:      toStrPtr("TCP"),
+												},
+											},
+											Resources: &v1.ResourceRequirements{
+												Limits: map[string]*resource.Quantity{
+													"cpu": {String_: toStrPtr("100m")},
+												},
+												Requests: map[string]*resource.Quantity{
+													"cpu": {String_: toStrPtr("100m")},
+												},
+											},
+										},
+										{
+											Name:  toStrPtr("waiting"),
 											Image: toStrPtr("image1"),
 											Ports: []*v1.ContainerPort{
 												{
@@ -101,7 +137,7 @@ func TestPod(t *testing.T) {
 									},
 									ContainerStatuses: []*v1.ContainerStatus{
 										{
-											Name: toStrPtr("forwarder"),
+											Name: toStrPtr("running"),
 											State: &v1.ContainerState{
 												Running: &v1.ContainerStateRunning{
 													StartedAt: &metav1.Time{Seconds: toInt64Ptr(cond2.Unix())},
@@ -113,6 +149,34 @@ func TestPod(t *testing.T) {
 											ImageID:      toStrPtr("image_id1"),
 											ContainerID:  toStrPtr("docker://54abe32d0094479d3d"),
 										},
+										{
+											Name: toStrPtr("completed"),
+											State: &v1.ContainerState{
+												Terminated: &v1.ContainerStateTerminated{
+													StartedAt: &metav1.Time{Seconds: toInt64Ptr(cond2.Unix())},
+													ExitCode:  toInt32Ptr(0),
+													Reason:    toStrPtr("Completed"),
+												},
+											},
+											Ready:        toBoolPtr(false),
+											RestartCount: toInt32Ptr(3),
+											Image:        toStrPtr("image1"),
+											ImageID:      toStrPtr("image_id1"),
+											ContainerID:  toStrPtr("docker://54abe32d0094479d3d"),
+										},
+										{
+											Name: toStrPtr("waiting"),
+											State: &v1.ContainerState{
+												Waiting: &v1.ContainerStateWaiting{
+													Reason: toStrPtr("PodUninitialized"),
+												},
+											},
+											Ready:        toBoolPtr(false),
+											RestartCount: toInt32Ptr(3),
+											Image:        toStrPtr("image1"),
+											ImageID:      toStrPtr("image_id1"),
+											ContainerID:  toStrPtr("docker://54abe32d0094479d3d"),
+										},
 									},
 								},
 								Metadata: &metav1.ObjectMeta{
@@ -150,10 +214,47 @@ func TestPod(t *testing.T) {
 						},
 						Tags: map[string]string{
 							"namespace":      "ns1",
-							"container_name": "forwarder",
+							"container_name": "running",
 							"node_name":      "node1",
 							"pod_name":       "pod1",
 							"state":          "running",
+							"readiness":      "ready",
+						},
+					},
+					{
+						Measurement: podContainerMeasurement,
+						Fields: map[string]interface{}{
+							"restarts_total":                   int32(3),
+							"state_code":                       1,
+							"state_reason":                     "Completed",
+							"resource_requests_millicpu_units": int64(100),
+							"resource_limits_millicpu_units":   int64(100),
+						},
+						Tags: map[string]string{
+							"namespace":      "ns1",
+							"container_name": "completed",
+							"node_name":      "node1",
+							"pod_name":       "pod1",
+							"state":          "terminated",
+							"readiness":      "unready",
+						},
+					},
+					{
+						Measurement: podContainerMeasurement,
+						Fields: map[string]interface{}{
+							"restarts_total":                   int32(3),
+							"state_code":                       2,
+							"state_reason":                     "PodUninitialized",
+							"resource_requests_millicpu_units": int64(100),
+							"resource_limits_millicpu_units":   int64(100),
+						},
+						Tags: map[string]string{
+							"namespace":      "ns1",
+							"container_name": "waiting",
+							"node_name":      "node1",
+							"pod_name":       "pod1",
+							"state":          "waiting",
+							"readiness":      "unready",
 						},
 					},
 				},