CL-Softwares
/
telegraf
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(inputs.prometheus): Allow explicit scrape configuration without annotations (#11962)

This commit is contained in:
Maxim Ivanov 2022-11-22 17:04:52 +00:00 committed by GitHub
parent d4eda21742
commit c3562ae8b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 207 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
plugins/inputs/prometheus/README.md
View File

@ -38,13 +38,37 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
## Kubernetes config file to create client from.
# kube_config = "/path/to/kubernetes.config"
## Scrape Kubernetes pods for the following prometheus annotations:
## - prometheus.io/scrape: Enable scraping for this pod
## - prometheus.io/scheme: If the metrics endpoint is secured then you will need to
## set this to 'https' & most likely set the tls config.
## - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation.
## Scrape Pods
## Enable scraping of k8s pods. Further settings as to which pods to scape
## are determiend by the 'method' option below. When enabled, the default is
## to use annotations to determine whether to scrape or not.
# monitor_kubernetes_pods = false
## Scrape Pods Method
## annotations: default, looks for specific pod annotations documented below
## settings: only look for pods matching the settings provided, not
## annotations
## settings+annotations: looks at pods that match annotations using the user
## defined settings
# monitor_kubernetes_pods_method = "annotations"
## Scrape Pods 'annotations' method options
## If set method is set to 'annotations' or 'settings+annotations', these
## annotation flags are looked for:
## - prometheus.io/scrape: Required to enable scraping for this pod. Can also
## use 'prometheus.io/scrape=false' annotation to opt-out entirely.
## - prometheus.io/scheme: If the metrics endpoint is secured then you will
## need to set this to 'https' & most likely set the tls config
## - prometheus.io/path: If the metrics path is not /metrics, define it with
## this annotation
## - prometheus.io/port: If port is not 9102 use this annotation
# monitor_kubernetes_pods = true
## Scrape Pods 'settings' method options
## When using 'settings' or 'settings+annotations', the default values for
## annotations can be modified using with the following options:
# monitor_kubernetes_pods_scheme = "http"
# monitor_kubernetes_pods_port = "9102"
# monitor_kubernetes_pods_path = "/metrics"
## Get the list of pods to scrape with either the scope of
## - cluster: the kubernetes watch api (default, no need to specify)

83
plugins/inputs/prometheus/kubernetes.go
View File

@ -10,6 +10,7 @@ import (
"net/url"
"os/user"
"path/filepath"
"strconv"
"time"
corev1 "k8s.io/api/core/v1"
@ -96,6 +97,25 @@ func (p *Prometheus) startK8s(ctx context.Context) error {
return nil
}
func shouldScrapePod(pod *corev1.Pod, p *Prometheus) bool {
isCandidate := podReady(pod.Status.ContainerStatuses) &&
podHasMatchingNamespace(pod, p) &&
podHasMatchingLabelSelector(pod, p.podLabelSelector) &&
podHasMatchingFieldSelector(pod, p.podFieldSelector)
var shouldScrape bool
switch p.MonitorKubernetesPodsMethod {
case MonitorMethodAnnotations: // must have 'true' annotation to be scraped
shouldScrape = pod.Annotations != nil && pod.Annotations["prometheus.io/scrape"] == "true"
case MonitorMethodSettings: // will be scraped regardless of annotation
shouldScrape = true
case MonitorMethodSettingsAndAnnotations: // will be scraped unless opts out with 'false' annotation
shouldScrape = pod.Annotations == nil || pod.Annotations["prometheus.io/scrape"] != "false"
}
return isCandidate && shouldScrape
}
// An edge case exists if a pod goes offline at the same time a new pod is created
// (without the scrape annotations). K8s may re-assign the old pod ip to the non-scrape
// pod, causing errors in the logs. This is only true if the pod going offline is not
@ -126,11 +146,7 @@ func (p *Prometheus) watchPod(ctx context.Context, clientset *kubernetes.Clients
pod, _ := clientset.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
if pod.Annotations["prometheus.io/scrape"] == "true" &&
podReady(pod.Status.ContainerStatuses) &&
podHasMatchingNamespace(pod, p) &&
podHasMatchingLabelSelector(pod, p.podLabelSelector) &&
podHasMatchingFieldSelector(pod, p.podFieldSelector) {
if shouldScrapePod(pod, p) {
registerPod(pod, p)
}
},
@ -147,11 +163,7 @@ func (p *Prometheus) watchPod(ctx context.Context, clientset *kubernetes.Clients
newPod, _ := clientset.CoreV1().Pods(newNamespace).Get(ctx, newName, metav1.GetOptions{})
if newPod.Annotations["prometheus.io/scrape"] == "true" &&
podReady(newPod.Status.ContainerStatuses) &&
podHasMatchingNamespace(newPod, p) &&
podHasMatchingLabelSelector(newPod, p.podLabelSelector) &&
podHasMatchingFieldSelector(newPod, p.podFieldSelector) {
if shouldScrapePod(newPod, p) {
if newPod.GetDeletionTimestamp() == nil {
registerPod(newPod, p)
}
@ -169,11 +181,7 @@ func (p *Prometheus) watchPod(ctx context.Context, clientset *kubernetes.Clients
oldPod, _ := clientset.CoreV1().Pods(oldNamespace).Get(ctx, oldName, metav1.GetOptions{})
if oldPod.Annotations["prometheus.io/scrape"] == "true" &&
podReady(oldPod.Status.ContainerStatuses) &&
podHasMatchingNamespace(oldPod, p) &&
podHasMatchingLabelSelector(oldPod, p.podLabelSelector) &&
podHasMatchingFieldSelector(oldPod, p.podFieldSelector) {
if shouldScrapePod(oldPod, p) {
if oldPod.GetDeletionTimestamp() != nil {
unregisterPod(oldPod, p)
}
@ -192,11 +200,7 @@ func (p *Prometheus) watchPod(ctx context.Context, clientset *kubernetes.Clients
pod, _ := clientset.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
if pod.Annotations["prometheus.io/scrape"] == "true" &&
podReady(pod.Status.ContainerStatuses) &&
podHasMatchingNamespace(pod, p) &&
podHasMatchingLabelSelector(pod, p.podLabelSelector) &&
podHasMatchingFieldSelector(pod, p.podFieldSelector) {
if shouldScrapePod(pod, p) {
if pod.GetDeletionTimestamp() != nil {
unregisterPod(pod, p)
}
@ -276,12 +280,7 @@ func updateCadvisorPodList(p *Prometheus, req *http.Request) error {
// Register pod only if it has an annotation to scrape, if it is ready,
// and if namespace and selectors are specified and match
for _, pod := range pods {
if necessaryPodFieldsArePresent(pod) &&
pod.Annotations["prometheus.io/scrape"] == "true" &&
podReady(pod.Status.ContainerStatuses) &&
podHasMatchingNamespace(pod, p) &&
podHasMatchingLabelSelector(pod, p.podLabelSelector) &&
podHasMatchingFieldSelector(pod, p.podFieldSelector) {
if necessaryPodFieldsArePresent(pod) && shouldScrapePod(pod, p) {
registerPod(pod, p)
}
}
@ -355,7 +354,7 @@ func registerPod(pod *corev1.Pod, p *Prometheus) {
if p.kubernetesPods == nil {
p.kubernetesPods = map[string]URLAndAddress{}
}
targetURL, err := getScrapeURL(pod)
targetURL, err := getScrapeURL(pod, p)
if err != nil {
p.Log.Errorf("could not parse URL: %s", err)
return
@ -397,7 +396,7 @@ func registerPod(pod *corev1.Pod, p *Prometheus) {
}
}
func getScrapeURL(pod *corev1.Pod) (*url.URL, error) {
func getScrapeURL(pod *corev1.Pod, p *Prometheus) (*url.URL, error) {
ip := pod.Status.PodIP
if ip == "" {
// return as if scrape was disabled, we will be notified again once the pod
@ -405,16 +404,36 @@ func getScrapeURL(pod *corev1.Pod) (*url.URL, error) {
return nil, nil
}
scheme := pod.Annotations["prometheus.io/scheme"]
pathAndQuery := pod.Annotations["prometheus.io/path"]
port := pod.Annotations["prometheus.io/port"]
var scheme, pathAndQuery, port string
if p.MonitorKubernetesPodsMethod == MonitorMethodSettings ||
p.MonitorKubernetesPodsMethod == MonitorMethodSettingsAndAnnotations {
scheme = p.MonitorKubernetesPodsScheme
pathAndQuery = p.MonitorKubernetesPodsPath
port = strconv.Itoa(p.MonitorKubernetesPodsPort)
}
if p.MonitorKubernetesPodsMethod == MonitorMethodAnnotations ||
p.MonitorKubernetesPodsMethod == MonitorMethodSettingsAndAnnotations {
if ann := pod.Annotations["prometheus.io/scheme"]; ann != "" {
scheme = ann
}
if ann := pod.Annotations["prometheus.io/path"]; ann != "" {
pathAndQuery = ann
}
if ann := pod.Annotations["prometheus.io/port"]; ann != "" {
port = ann
}
}
if scheme == "" {
scheme = "http"
}
if port == "" {
port = "9102"
}
if pathAndQuery == "" {
pathAndQuery = "/metrics"
}
@ -431,7 +450,7 @@ func getScrapeURL(pod *corev1.Pod) (*url.URL, error) {
}
func unregisterPod(pod *corev1.Pod, p *Prometheus) {
targetURL, err := getScrapeURL(pod)
targetURL, err := getScrapeURL(pod, p)
if err != nil {
p.Log.Errorf("failed to parse url: %s", err)
return

97
plugins/inputs/prometheus/kubernetes_test.go
View File

@ -12,67 +12,116 @@ import (
"github.com/influxdata/telegraf/testutil"
)
func initPrometheus() *Prometheus {
prom := &Prometheus{Log: testutil.Logger{}}
prom.MonitorKubernetesPodsScheme = "http"
prom.MonitorKubernetesPodsPort = 9102
prom.MonitorKubernetesPodsPath = "/metrics"
prom.MonitorKubernetesPodsMethod = MonitorMethodAnnotations
return prom
}
func TestScrapeURLNoAnnotations(t *testing.T) {
prom := &Prometheus{Log: testutil.Logger{}}
p := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{}}
p.Annotations = map[string]string{}
url, err := getScrapeURL(p)
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Nil(t, url)
}
func TestScrapeURLAnnotationsNoScrape(t *testing.T) {
p := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{}}
p.Name = "myPod"
p.Annotations = map[string]string{"prometheus.io/scrape": "false"}
url, err := getScrapeURL(p)
func TestScrapeURLNoAnnotationsScrapeConfig(t *testing.T) {
prom := initPrometheus()
prom.MonitorKubernetesPodsMethod = MonitorMethodSettingsAndAnnotations
p := pod()
p.Annotations = map[string]string{}
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Nil(t, url)
require.Equal(t, "http://127.0.0.1:9102/metrics", url.String())
}
func TestScrapeURLScrapeConfigCustom(t *testing.T) {
prom := initPrometheus()
prom.MonitorKubernetesPodsMethod = MonitorMethodSettingsAndAnnotations
prom.MonitorKubernetesPodsScheme = "https"
prom.MonitorKubernetesPodsPort = 9999
prom.MonitorKubernetesPodsPath = "/svc/metrics"
p := pod()
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Equal(t, "https://127.0.0.1:9999/svc/metrics", url.String())
}
func TestScrapeURLAnnotations(t *testing.T) {
prom := &Prometheus{Log: testutil.Logger{}}
p := pod()
p.Annotations = map[string]string{"prometheus.io/scrape": "true"}
url, err := getScrapeURL(p)
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Equal(t, "http://127.0.0.1:9102/metrics", url.String())
}
func TestScrapeURLAnnotationsScrapeConfig(t *testing.T) {
prom := initPrometheus()
prom.MonitorKubernetesPodsMethod = MonitorMethodSettingsAndAnnotations
p := pod()
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Equal(t, "http://127.0.0.1:9102/metrics", url.String())
}
func TestScrapeURLAnnotationsCustomPort(t *testing.T) {
prom := initPrometheus()
p := pod()
p.Annotations = map[string]string{"prometheus.io/scrape": "true", "prometheus.io/port": "9000"}
url, err := getScrapeURL(p)
p.Annotations = map[string]string{"prometheus.io/port": "9000"}
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Equal(t, "http://127.0.0.1:9000/metrics", url.String())
}
func TestScrapeURLAnnotationsCustomPortScrapeConfig(t *testing.T) {
prom := initPrometheus()
prom.MonitorKubernetesPodsMethod = MonitorMethodSettingsAndAnnotations
p := pod()
p.Annotations = map[string]string{"prometheus.io/port": "9000"}
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Equal(t, "http://127.0.0.1:9000/metrics", url.String())
}
func TestScrapeURLAnnotationsCustomPath(t *testing.T) {
prom := initPrometheus()
p := pod()
p.Annotations = map[string]string{"prometheus.io/scrape": "true", "prometheus.io/path": "mymetrics"}
url, err := getScrapeURL(p)
p.Annotations = map[string]string{"prometheus.io/path": "mymetrics"}
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Equal(t, "http://127.0.0.1:9102/mymetrics", url.String())
}
func TestScrapeURLAnnotationsCustomPathWithSep(t *testing.T) {
prom := initPrometheus()
p := pod()
p.Annotations = map[string]string{"prometheus.io/scrape": "true", "prometheus.io/path": "/mymetrics"}
url, err := getScrapeURL(p)
p.Annotations = map[string]string{"prometheus.io/path": "/mymetrics"}
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Equal(t, "http://127.0.0.1:9102/mymetrics", url.String())
}
func TestScrapeURLAnnotationsCustomPathWithQueryParameters(t *testing.T) {
prom := initPrometheus()
p := pod()
p.Annotations = map[string]string{"prometheus.io/scrape": "true", "prometheus.io/path": "/v1/agent/metrics?format=prometheus"}
url, err := getScrapeURL(p)
p.Annotations = map[string]string{"prometheus.io/path": "/v1/agent/metrics?format=prometheus"}
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Equal(t, "http://127.0.0.1:9102/v1/agent/metrics?format=prometheus", url.String())
}
func TestScrapeURLAnnotationsCustomPathWithFragment(t *testing.T) {
prom := initPrometheus()
p := pod()
p.Annotations = map[string]string{"prometheus.io/scrape": "true", "prometheus.io/path": "/v1/agent/metrics#prometheus"}
url, err := getScrapeURL(p)
p.Annotations = map[string]string{"prometheus.io/path": "/v1/agent/metrics#prometheus"}
url, err := getScrapeURL(p, prom)
require.NoError(t, err)
require.Equal(t, "http://127.0.0.1:9102/v1/agent/metrics#prometheus", url.String())
}
@ -86,6 +135,16 @@ func TestAddPod(t *testing.T) {
require.Equal(t, 1, len(prom.kubernetesPods))
}
func TestAddPodScrapeConfig(t *testing.T) {
prom := initPrometheus()
prom.MonitorKubernetesPodsMethod = MonitorMethodSettingsAndAnnotations
p := pod()
p.Annotations = map[string]string{}
registerPod(p, prom)
require.Equal(t, 1, len(prom.kubernetesPods))
}
func TestAddMultipleDuplicatePods(t *testing.T) {
prom := &Prometheus{Log: testutil.Logger{}}

18
plugins/inputs/prometheus/prometheus.go
View File

@ -31,6 +31,15 @@ var sampleConfig string
const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3`
type MonitorMethod string
const (
MonitorMethodNone MonitorMethod = ""
MonitorMethodAnnotations MonitorMethod = "annotations"
MonitorMethodSettings MonitorMethod = "settings"
MonitorMethodSettingsAndAnnotations MonitorMethod = "settings+annotations"
)
type Prometheus struct {
// An array of urls to scrape metrics from.
URLs []string `toml:"urls"`
@ -92,6 +101,11 @@ type Prometheus struct {
podFieldSelector fields.Selector
isNodeScrapeScope bool
MonitorKubernetesPodsMethod MonitorMethod `toml:"monitor_kubernetes_pods_method"`
MonitorKubernetesPodsScheme string `toml:"monitor_kubernetes_pods_scheme"`
MonitorKubernetesPodsPath string `toml:"monitor_kubernetes_pods_path"`
MonitorKubernetesPodsPort int `toml:"monitor_kubernetes_pods_port"`
// Only for monitor_kubernetes_pods=true
CacheRefreshInterval int `toml:"cache_refresh_interval"`
@ -140,6 +154,10 @@ func (p *Prometheus) Init() error {
p.Log.Infof("Using the label selector: %v and field selector: %v", p.podLabelSelector, p.podFieldSelector)
}
if p.MonitorKubernetesPodsMethod == MonitorMethodNone {
p.MonitorKubernetesPodsMethod = MonitorMethodAnnotations
}
ctx := context.Background()
client, err := p.HTTPClientConfig.CreateClient(ctx, p.Log)
if err != nil {

36
plugins/inputs/prometheus/sample.conf
View File

@ -21,13 +21,37 @@
## Kubernetes config file to create client from.
# kube_config = "/path/to/kubernetes.config"
## Scrape Kubernetes pods for the following prometheus annotations:
## - prometheus.io/scrape: Enable scraping for this pod
## - prometheus.io/scheme: If the metrics endpoint is secured then you will need to
## set this to 'https' & most likely set the tls config.
## - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation.
## Scrape Pods
## Enable scraping of k8s pods. Further settings as to which pods to scape
## are determiend by the 'method' option below. When enabled, the default is
## to use annotations to determine whether to scrape or not.
# monitor_kubernetes_pods = false
## Scrape Pods Method
## annotations: default, looks for specific pod annotations documented below
## settings: only look for pods matching the settings provided, not
## annotations
## settings+annotations: looks at pods that match annotations using the user
## defined settings
# monitor_kubernetes_pods_method = "annotations"
## Scrape Pods 'annotations' method options
## If set method is set to 'annotations' or 'settings+annotations', these
## annotation flags are looked for:
## - prometheus.io/scrape: Required to enable scraping for this pod. Can also
## use 'prometheus.io/scrape=false' annotation to opt-out entirely.
## - prometheus.io/scheme: If the metrics endpoint is secured then you will
## need to set this to 'https' & most likely set the tls config
## - prometheus.io/path: If the metrics path is not /metrics, define it with
## this annotation
## - prometheus.io/port: If port is not 9102 use this annotation
# monitor_kubernetes_pods = true
## Scrape Pods 'settings' method options
## When using 'settings' or 'settings+annotations', the default values for
## annotations can be modified using with the following options:
# monitor_kubernetes_pods_scheme = "http"
# monitor_kubernetes_pods_port = "9102"
# monitor_kubernetes_pods_path = "/metrics"
## Get the list of pods to scrape with either the scope of
## - cluster: the kubernetes watch api (default, no need to specify)