feat (inputs/prometheus): add setting to set custom namespace label name to avoid conflicts (#11538)

plugins/inputs/prometheus/README.md

@@ -53,6 +53,9 @@ in Prometheus format.
   ## Restricts Kubernetes monitoring to a single namespace
   ##   ex: monitor_kubernetes_pods_namespace = "default"
   # monitor_kubernetes_pods_namespace = ""
+  ## The name of the label for the pod that is being scraped.
+  ## Default is 'namespace' but this can conflict with metrics that have the label 'namespace'
+  # pod_namespace_label_name = "namespace"
   # label selector to target pods which have the label
   # kubernetes_label_selector = "env=dev,app=nginx"
   # field selector to target pods
@@ -159,6 +162,10 @@ the following annotation are supported:
 Using the `monitor_kubernetes_pods_namespace` option allows you to limit which
 pods you are scraping.
 
+The setting `pod_namespace_label_name` allows you to change the label name for
+the namespace of the pod you are scraping. The default is `namespace`, but this
+will overwrite a label with the name `namespace` from a metric scraped.
+
 Using `pod_scrape_scope = "node"` allows more scalable scraping for pods which
 will scrape pods only in the node that telegraf is running. It will fetch the
 pod list locally from the node's kubelet. This will require running Telegraf in

plugins/inputs/prometheus/kubernetes.go

@@ -373,8 +373,14 @@ func registerPod(pod *corev1.Pod, p *Prometheus) {
 	if tags == nil {
 		tags = map[string]string{}
 	}
+
 	tags["pod_name"] = pod.Name
-	tags["namespace"] = pod.Namespace
+	podNamespace := "namespace"
+	if p.PodNamespaceLabelName != "" {
+		podNamespace = p.PodNamespaceLabelName
+	}
+	tags[podNamespace] = pod.Namespace
+
 	// add labels as metrics tags
 	for k, v := range pod.Labels {
 		tags[k] = v

plugins/inputs/prometheus/kubernetes_test.go

@@ -119,6 +119,27 @@ func TestDeletePods(t *testing.T) {
 	require.Equal(t, 0, len(prom.kubernetesPods))
 }
 
+func TestKeepDefaultNamespaceLabelName(t *testing.T) {
+	prom := &Prometheus{Log: testutil.Logger{}}
+
+	p := pod()
+	p.Annotations = map[string]string{"prometheus.io/scrape": "true"}
+	registerPod(p, prom)
+	tags := prom.kubernetesPods["http://127.0.0.1:9102/metrics"].Tags
+	require.Equal(t, "default", tags["namespace"])
+}
+
+func TestChangeNamespaceLabelName(t *testing.T) {
+	prom := &Prometheus{Log: testutil.Logger{}, PodNamespaceLabelName: "pod_namespace"}
+
+	p := pod()
+	p.Annotations = map[string]string{"prometheus.io/scrape": "true"}
+	registerPod(p, prom)
+	tags := prom.kubernetesPods["http://127.0.0.1:9102/metrics"].Tags
+	require.Equal(t, "default", tags["pod_namespace"])
+	require.Equal(t, "", tags["namespace"])
+}
+
 func TestPodHasMatchingNamespace(t *testing.T) {
 	prom := &Prometheus{Log: testutil.Logger{}, PodNamespace: "default"}
 

plugins/inputs/prometheus/prometheus.go

@@ -80,6 +80,7 @@ type Prometheus struct {
 	NodeIP                string `toml:"node_ip"`
 	PodScrapeInterval     int    `toml:"pod_scrape_interval"`
 	PodNamespace          string `toml:"monitor_kubernetes_pods_namespace"`
+	PodNamespaceLabelName string `toml:"pod_namespace_label_name"`
 	lock                  sync.Mutex
 	kubernetesPods        map[string]URLAndAddress
 	cancel                context.CancelFunc

plugins/inputs/prometheus/sample.conf

@@ -45,6 +45,9 @@
   ## Restricts Kubernetes monitoring to a single namespace
   ##   ex: monitor_kubernetes_pods_namespace = "default"
   # monitor_kubernetes_pods_namespace = ""
+  ## The name of the label for the pod that is being scraped.
+  ## Default is 'namespace' but this can conflict with metrics that have the label 'namespace'
+  # pod_namespace_label_name = "namespace"
   # label selector to target pods which have the label
   # kubernetes_label_selector = "env=dev,app=nginx"
   # field selector to target pods