feat (inputs/prometheus): add setting to set custom namespace label name to avoid conflicts (#11538)

plugins/inputs/prometheus/README.md

@@ -53,6 +53,9 @@ in Prometheus format.
   ## Restricts Kubernetes monitoring to a single namespace
   ##   ex: monitor_kubernetes_pods_namespace = "default"
   # monitor_kubernetes_pods_namespace = ""
+  ## The name of the label for the pod that is being scraped.
+  ## Default is 'namespace' but this can conflict with metrics that have the label 'namespace'
+  # pod_namespace_label_name = "namespace"
   # label selector to target pods which have the label
   # kubernetes_label_selector = "env=dev,app=nginx"
   # field selector to target pods
@@ -159,6 +162,10 @@ the following annotation are supported:
 Using the `monitor_kubernetes_pods_namespace` option allows you to limit which
 pods you are scraping.
 
+The setting `pod_namespace_label_name` allows you to change the label name for
+the namespace of the pod you are scraping. The default is `namespace`, but this
+will overwrite a label with the name `namespace` from a metric scraped.
+
 Using `pod_scrape_scope = "node"` allows more scalable scraping for pods which
 will scrape pods only in the node that telegraf is running. It will fetch the
 pod list locally from the node's kubelet. This will require running Telegraf in

plugins/inputs/prometheus/kubernetes.go

@@ -373,8 +373,14 @@ func registerPod(pod *corev1.Pod, p *Prometheus) {
 	if tags == nil {
 		tags = map[string]string{}
 	}
+
 	tags["pod_name"] = pod.Name
-	tags["namespace"] = pod.Namespace
+	podNamespace := "namespace"
+	if p.PodNamespaceLabelName != "" {
+		podNamespace = p.PodNamespaceLabelName
+	}
+	tags[podNamespace] = pod.Namespace
+
 	// add labels as metrics tags
 	for k, v := range pod.Labels {
 		tags[k] = v

plugins/inputs/prometheus/kubernetes_test.go

@@ -119,6 +119,27 @@ func TestDeletePods(t *testing.T) {
 	require.Equal(t, 0, len(prom.kubernetesPods))
 }
 
+func TestKeepDefaultNamespaceLabelName(t *testing.T) {
+	prom := &Prometheus{Log: testutil.Logger{}}
+
+	p := pod()
+	p.Annotations = map[string]string{"prometheus.io/scrape": "true"}
+	registerPod(p, prom)
+	tags := prom.kubernetesPods["http://127.0.0.1:9102/metrics"].Tags
+	require.Equal(t, "default", tags["namespace"])
+}
+
+func TestChangeNamespaceLabelName(t *testing.T) {
+	prom := &Prometheus{Log: testutil.Logger{}, PodNamespaceLabelName: "pod_namespace"}
+
+	p := pod()
+	p.Annotations = map[string]string{"prometheus.io/scrape": "true"}
+	registerPod(p, prom)
+	tags := prom.kubernetesPods["http://127.0.0.1:9102/metrics"].Tags
+	require.Equal(t, "default", tags["pod_namespace"])
+	require.Equal(t, "", tags["namespace"])
+}
+
 func TestPodHasMatchingNamespace(t *testing.T) {
 	prom := &Prometheus{Log: testutil.Logger{}, PodNamespace: "default"}
 

plugins/inputs/prometheus/prometheus.go

@@ -75,15 +75,16 @@ type Prometheus struct {
 	headers map[string]string
 
 	// Should we scrape Kubernetes services for prometheus annotations
-	MonitorPods       bool   `toml:"monitor_kubernetes_pods"`
+	MonitorPods           bool   `toml:"monitor_kubernetes_pods"`
-	PodScrapeScope    string `toml:"pod_scrape_scope"`
+	PodScrapeScope        string `toml:"pod_scrape_scope"`
-	NodeIP            string `toml:"node_ip"`
+	NodeIP                string `toml:"node_ip"`
-	PodScrapeInterval int    `toml:"pod_scrape_interval"`
+	PodScrapeInterval     int    `toml:"pod_scrape_interval"`
-	PodNamespace      string `toml:"monitor_kubernetes_pods_namespace"`
+	PodNamespace          string `toml:"monitor_kubernetes_pods_namespace"`
-	lock              sync.Mutex
+	PodNamespaceLabelName string `toml:"pod_namespace_label_name"`
-	kubernetesPods    map[string]URLAndAddress
+	lock                  sync.Mutex
-	cancel            context.CancelFunc
+	kubernetesPods        map[string]URLAndAddress
-	wg                sync.WaitGroup
+	cancel                context.CancelFunc
+	wg                    sync.WaitGroup
 
 	// Only for monitor_kubernetes_pods=true and pod_scrape_scope="node"
 	podLabelSelector  labels.Selector

plugins/inputs/prometheus/sample.conf

@@ -45,6 +45,9 @@
   ## Restricts Kubernetes monitoring to a single namespace
   ##   ex: monitor_kubernetes_pods_namespace = "default"
   # monitor_kubernetes_pods_namespace = ""
+  ## The name of the label for the pod that is being scraped.
+  ## Default is 'namespace' but this can conflict with metrics that have the label 'namespace'
+  # pod_namespace_label_name = "namespace"
   # label selector to target pods which have the label
   # kubernetes_label_selector = "env=dev,app=nginx"
   # field selector to target pods