fix: issues with prometheus kubernetes pod discovery (#9605)

This commit is contained in:
Grace Wehner 2021-08-17 14:54:55 -07:00 committed by GitHub
parent 02ccbec348
commit fe144e7c99
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 29 additions and 21 deletions

View File

@ -111,35 +111,43 @@ func (p *Prometheus) watchPod(ctx context.Context, client *kubernetes.Clientset)
LabelSelector: p.KubernetesLabelSelector,
FieldSelector: p.KubernetesFieldSelector,
})
defer watcher.Stop()
if err != nil {
return err
}
pod := &corev1.Pod{}
go func() {
for event := range watcher.ResultChan() {
pod = &corev1.Pod{}
// If the pod is not "ready", there will be no ip associated with it.
if pod.Annotations["prometheus.io/scrape"] != "true" ||
!podReady(pod.Status.ContainerStatuses) {
continue
}
switch event.Type {
case watch.Added:
registerPod(pod, p)
case watch.Modified:
// To avoid multiple actions for each event, unregister on the first event
// in the delete sequence, when the containers are still "ready".
if pod.GetDeletionTimestamp() != nil {
unregisterPod(pod, p)
} else {
for {
select {
case <-ctx.Done():
return nil
default:
for event := range watcher.ResultChan() {
pod, ok := event.Object.(*corev1.Pod)
if !ok {
return fmt.Errorf("Unexpected object when getting pods")
}
// If the pod is not "ready", there will be no ip associated with it.
if pod.Annotations["prometheus.io/scrape"] != "true" ||
!podReady(pod.Status.ContainerStatuses) {
continue
}
switch event.Type {
case watch.Added:
registerPod(pod, p)
case watch.Modified:
// To avoid multiple actions for each event, unregister on the first event
// in the delete sequence, when the containers are still "ready".
if pod.GetDeletionTimestamp() != nil {
unregisterPod(pod, p)
} else {
registerPod(pod, p)
}
}
}
}
}()
return nil
}
}
func (p *Prometheus) cAdvisor(ctx context.Context, bearerToken string) error {