feat (inputs/prometheus): add setting to set custom namespace label name to avoid conflicts (#11538)

plugins/inputs/prometheus/README.md

@@ -53,6 +53,9 @@ in Prometheus format.
   ## Restricts Kubernetes monitoring to a single namespace
   ##   ex: monitor_kubernetes_pods_namespace = "default"
   # monitor_kubernetes_pods_namespace = ""
+  ## The name of the label for the pod that is being scraped.
+  ## Default is 'namespace' but this can conflict with metrics that have the label 'namespace'
+  # pod_namespace_label_name = "namespace"
   # label selector to target pods which have the label
   # kubernetes_label_selector = "env=dev,app=nginx"
   # field selector to target pods
@@ -159,6 +162,10 @@ the following annotation are supported:
 Using the `monitor_kubernetes_pods_namespace` option allows you to limit which
 pods you are scraping.
 
+The setting `pod_namespace_label_name` allows you to change the label name for
+the namespace of the pod you are scraping. The default is `namespace`, but this
+will overwrite a label with the name `namespace` from a metric scraped.
+
 Using `pod_scrape_scope = "node"` allows more scalable scraping for pods which
 will scrape pods only in the node that telegraf is running. It will fetch the
 pod list locally from the node's kubelet. This will require running Telegraf in

plugins/inputs/prometheus/kubernetes.go

@@ -373,8 +373,14 @@ func registerPod(pod *corev1.Pod, p *Prometheus) {
 	if tags == nil {
 		tags = map[string]string{}
 	}
+
 	tags["pod_name"] = pod.Name
-	tags["namespace"] = pod.Namespace
+	podNamespace := "namespace"
+	if p.PodNamespaceLabelName != "" {
+		podNamespace = p.PodNamespaceLabelName
+	}
+	tags[podNamespace] = pod.Namespace
+
 	// add labels as metrics tags
 	for k, v := range pod.Labels {
 		tags[k] = v

plugins/inputs/prometheus/kubernetes_test.go

@@ -119,6 +119,27 @@ func TestDeletePods(t *testing.T) {
 	require.Equal(t, 0, len(prom.kubernetesPods))
 }
 
+func TestKeepDefaultNamespaceLabelName(t *testing.T) {
+	prom := &Prometheus{Log: testutil.Logger{}}
+
+	p := pod()
+	p.Annotations = map[string]string{"prometheus.io/scrape": "true"}
+	registerPod(p, prom)
+	tags := prom.kubernetesPods["http://127.0.0.1:9102/metrics"].Tags
+	require.Equal(t, "default", tags["namespace"])
+}
+
+func TestChangeNamespaceLabelName(t *testing.T) {
+	prom := &Prometheus{Log: testutil.Logger{}, PodNamespaceLabelName: "pod_namespace"}
+
+	p := pod()
+	p.Annotations = map[string]string{"prometheus.io/scrape": "true"}
+	registerPod(p, prom)
+	tags := prom.kubernetesPods["http://127.0.0.1:9102/metrics"].Tags
+	require.Equal(t, "default", tags["pod_namespace"])
+	require.Equal(t, "", tags["namespace"])
+}
+
 func TestPodHasMatchingNamespace(t *testing.T) {
 	prom := &Prometheus{Log: testutil.Logger{}, PodNamespace: "default"}
 

plugins/inputs/prometheus/prometheus.go

@@ -75,15 +75,16 @@ type Prometheus struct {
 	headers map[string]string
 
 	// Should we scrape Kubernetes services for prometheus annotations
-	MonitorPods       bool   `toml:"monitor_kubernetes_pods"`
-	PodScrapeScope    string `toml:"pod_scrape_scope"`
-	NodeIP            string `toml:"node_ip"`
-	PodScrapeInterval int    `toml:"pod_scrape_interval"`
-	PodNamespace      string `toml:"monitor_kubernetes_pods_namespace"`
-	lock              sync.Mutex
-	kubernetesPods    map[string]URLAndAddress
-	cancel            context.CancelFunc
-	wg                sync.WaitGroup
+	MonitorPods           bool   `toml:"monitor_kubernetes_pods"`
+	PodScrapeScope        string `toml:"pod_scrape_scope"`
+	NodeIP                string `toml:"node_ip"`
+	PodScrapeInterval     int    `toml:"pod_scrape_interval"`
+	PodNamespace          string `toml:"monitor_kubernetes_pods_namespace"`
+	PodNamespaceLabelName string `toml:"pod_namespace_label_name"`
+	lock                  sync.Mutex
+	kubernetesPods        map[string]URLAndAddress
+	cancel                context.CancelFunc
+	wg                    sync.WaitGroup
 
 	// Only for monitor_kubernetes_pods=true and pod_scrape_scope="node"
 	podLabelSelector  labels.Selector

plugins/inputs/prometheus/sample.conf

@@ -45,6 +45,9 @@
   ## Restricts Kubernetes monitoring to a single namespace
   ##   ex: monitor_kubernetes_pods_namespace = "default"
   # monitor_kubernetes_pods_namespace = ""
+  ## The name of the label for the pod that is being scraped.
+  ## Default is 'namespace' but this can conflict with metrics that have the label 'namespace'
+  # pod_namespace_label_name = "namespace"
   # label selector to target pods which have the label
   # kubernetes_label_selector = "env=dev,app=nginx"
   # field selector to target pods