From fe144e7c990e03bb4f32e8ae2a3eee24919eafd9 Mon Sep 17 00:00:00 2001
From: Grace Wehner <grace.wehner@microsoft.com>
Date: Tue, 17 Aug 2021 14:54:55 -0700
Subject: [PATCH] fix: issues with prometheus kubernetes pod discovery (#9605)

---
 plugins/inputs/prometheus/kubernetes.go | 50 ++++++++++++++-----------
 1 file changed, 29 insertions(+), 21 deletions(-)

diff --git a/plugins/inputs/prometheus/kubernetes.go b/plugins/inputs/prometheus/kubernetes.go
index e78c64af3..0e658003a 100644
--- a/plugins/inputs/prometheus/kubernetes.go
+++ b/plugins/inputs/prometheus/kubernetes.go
@@ -111,35 +111,43 @@ func (p *Prometheus) watchPod(ctx context.Context, client *kubernetes.Clientset)
 		LabelSelector: p.KubernetesLabelSelector,
 		FieldSelector: p.KubernetesFieldSelector,
 	})
+	defer watcher.Stop()
 	if err != nil {
 		return err
 	}
-	pod := &corev1.Pod{}
-	go func() {
-		for event := range watcher.ResultChan() {
-			pod = &corev1.Pod{}
-			// If the pod is not "ready", there will be no ip associated with it.
-			if pod.Annotations["prometheus.io/scrape"] != "true" ||
-				!podReady(pod.Status.ContainerStatuses) {
-				continue
-			}
 
-			switch event.Type {
-			case watch.Added:
-				registerPod(pod, p)
-			case watch.Modified:
-				// To avoid multiple actions for each event, unregister on the first event
-				// in the delete sequence, when the containers are still "ready".
-				if pod.GetDeletionTimestamp() != nil {
-					unregisterPod(pod, p)
-				} else {
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			for event := range watcher.ResultChan() {
+				pod, ok := event.Object.(*corev1.Pod)
+				if !ok {
+					return fmt.Errorf("Unexpected object when getting pods")
+				}
+
+				// If the pod is not "ready", there will be no ip associated with it.
+				if pod.Annotations["prometheus.io/scrape"] != "true" ||
+					!podReady(pod.Status.ContainerStatuses) {
+					continue
+				}
+
+				switch event.Type {
+				case watch.Added:
 					registerPod(pod, p)
+				case watch.Modified:
+					// To avoid multiple actions for each event, unregister on the first event
+					// in the delete sequence, when the containers are still "ready".
+					if pod.GetDeletionTimestamp() != nil {
+						unregisterPod(pod, p)
+					} else {
+						registerPod(pod, p)
+					}
 				}
 			}
 		}
-	}()
-
-	return nil
+	}
 }
 
 func (p *Prometheus) cAdvisor(ctx context.Context, bearerToken string) error {