Proxmox plugin (#7922)

go.mod

@@ -28,7 +28,7 @@ require (
 	github.com/aws/aws-sdk-go v1.30.9
 	github.com/benbjohnson/clock v1.0.3
 	github.com/bitly/go-hostpool v0.1.0 // indirect
-	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
+	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869
 	github.com/caio/go-tdigest v2.3.0+incompatible // indirect
 	github.com/cenkalti/backoff v2.0.0+incompatible // indirect
 	github.com/cisco-ie/nx-telemetry-proto v0.0.0-20190531143454-82441e232cf6

plugins/inputs/all/all.go

@@ -133,6 +133,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/processes"
 	_ "github.com/influxdata/telegraf/plugins/inputs/procstat"
 	_ "github.com/influxdata/telegraf/plugins/inputs/prometheus"
+	_ "github.com/influxdata/telegraf/plugins/inputs/proxmox"
 	_ "github.com/influxdata/telegraf/plugins/inputs/puppetagent"
 	_ "github.com/influxdata/telegraf/plugins/inputs/rabbitmq"
 	_ "github.com/influxdata/telegraf/plugins/inputs/raindrops"

plugins/inputs/proxmox/README.md

@@ -0,0 +1,62 @@
+# Proxmox Input Plugin
+
+The proxmox plugin gathers metrics about containers and VMs using the Proxmox API.
+
+### Configuration:
+
+```
+[[inputs.proxmox]]
+  ## API connection configuration. The API token was introduced in Proxmox v6.2. Required permissions for user and token: PVEAuditor role on /.
+  base_url = "https://localhost:8006/api2/json"
+  api_token = "USER@REALM!TOKENID=UUID"
+
+  ## Optional TLS Config
+  # tls_ca = "/etc/telegraf/ca.pem"
+  # tls_cert = "/etc/telegraf/cert.pem"
+  # tls_key = "/etc/telegraf/key.pem"
+  ## Use TLS but skip chain & host verification
+  insecure_skip_verify = false
+
+  # HTTP response timeout (default: 5s)
+  response_timeout = "5s"
+```
+
+#### Permissions
+
+The plugin will need to have access to the Proxmox API. An API token
+must be provided with the corresponding user being assigned at least the PVEAuditor
+role on /.
+
+### Measurements & Fields:
+
+- proxmox
+  - status
+  - uptime
+  - cpuload
+  - mem_used
+  - mem_total
+  - mem_free
+  - mem_used_percentage
+  - swap_used
+  - swap_total
+  - swap_free
+  - swap_used_percentage
+  - disk_used
+  - disk_total
+  - disk_free
+  - disk_used_percentage
+
+### Tags:
+
+  - node_fqdn - FQDN of the node telegraf is running on
+  - vm_name - Name of the VM/container
+  - vm_fqdn - FQDN of the VM/container
+  - vm_type - Type of the VM/container (lxc, qemu)
+
+### Example Output:
+
+```
+$ ./telegraf --config telegraf.conf --input-filter proxmox --test
+> proxmox,host=pxnode,node_fqdn=pxnode.example.com,vm_fqdn=vm1.example.com,vm_name=vm1,vm_type=lxc cpuload=0.147998116735236,disk_free=4461129728i,disk_total=5217320960i,disk_used=756191232i,disk_used_percentage=14,mem_free=1046827008i,mem_total=1073741824i,mem_used=26914816i,mem_used_percentage=2,status="running",swap_free=536698880i,swap_total=536870912i,swap_used=172032i,swap_used_percentage=0,uptime=1643793i 1595457277000000000
+> ...
+```

plugins/inputs/proxmox/proxmox.go

@@ -0,0 +1,250 @@
+package proxmox
+
+import (
+	"encoding/json"
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+)
+
+var sampleConfig = `
+  ## API connection configuration. The API token was introduced in Proxmox v6.2. Required permissions for user and token: PVEAuditor role on /.
+  base_url = "https://localhost:8006/api2/json"
+  api_token = "USER@REALM!TOKENID=UUID"
+
+  ## Optional TLS Config
+  # tls_ca = "/etc/telegraf/ca.pem"
+  # tls_cert = "/etc/telegraf/cert.pem"
+  # tls_key = "/etc/telegraf/key.pem"
+  ## Use TLS but skip chain & host verification
+  insecure_skip_verify = false
+
+  # HTTP response timeout (default: 5s)
+  response_timeout = "5s"
+`
+
+func (px *Proxmox) SampleConfig() string {
+	return sampleConfig
+}
+
+func (px *Proxmox) Description() string {
+	return "Provides metrics from Proxmox nodes (Proxmox Virtual Environment > 6.2)."
+}
+
+func (px *Proxmox) Gather(acc telegraf.Accumulator) error {
+	err := getNodeSearchDomain(px)
+	if err != nil {
+		return err
+	}
+
+	gatherLxcData(px, acc)
+	gatherQemuData(px, acc)
+
+	return nil
+}
+
+func (px *Proxmox) Init() error {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return err
+	}
+	px.hostname = hostname
+
+	tlsCfg, err := px.ClientConfig.TLSConfig()
+	if err != nil {
+		return err
+	}
+	px.httpClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: tlsCfg,
+		},
+		Timeout: px.ResponseTimeout.Duration,
+	}
+
+	return nil
+}
+
+func init() {
+	px := Proxmox{
+		requestFunction: performRequest,
+	}
+
+	inputs.Add("proxmox", func() telegraf.Input { return &px })
+}
+
+func getNodeSearchDomain(px *Proxmox) error {
+	apiUrl := "/nodes/" + px.hostname + "/dns"
+	jsonData, err := px.requestFunction(px, apiUrl, http.MethodGet, nil)
+
+	var nodeDns NodeDns
+	err = json.Unmarshal(jsonData, &nodeDns)
+	if err != nil {
+		return err
+	}
+	px.nodeSearchDomain = nodeDns.Data.Searchdomain
+
+	return nil
+}
+
+func performRequest(px *Proxmox, apiUrl string, method string, data url.Values) ([]byte, error) {
+	request, err := http.NewRequest(method, px.BaseURL+apiUrl, strings.NewReader(data.Encode()))
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Add("Authorization", "PVEAPIToken="+px.APIToken)
+
+	resp, err := px.httpClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	responseBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return responseBody, nil
+}
+
+func gatherLxcData(px *Proxmox, acc telegraf.Accumulator) {
+	gatherVmData(px, acc, LXC)
+}
+
+func gatherQemuData(px *Proxmox, acc telegraf.Accumulator) {
+	gatherVmData(px, acc, QEMU)
+}
+
+func gatherVmData(px *Proxmox, acc telegraf.Accumulator, rt ResourceType) {
+	vmStats, err := getVmStats(px, rt)
+	if err != nil {
+		px.Log.Error("Error getting VM stats: %v", err)
+		return
+	}
+
+	// For each VM add metrics to Accumulator
+	for _, vmStat := range vmStats.Data {
+		vmConfig, err := getVmConfig(px, vmStat.ID, rt)
+		if err != nil {
+			px.Log.Error("Error getting VM config: %v", err)
+			return
+		}
+		tags := getTags(px, vmStat.Name, vmConfig, rt)
+		fields, err := getFields(vmStat)
+		if err != nil {
+			px.Log.Error("Error getting VM measurements: %v", err)
+			return
+		}
+		acc.AddFields("proxmox", fields, tags)
+	}
+}
+
+func getVmStats(px *Proxmox, rt ResourceType) (VmStats, error) {
+	apiUrl := "/nodes/" + px.hostname + "/" + string(rt)
+	jsonData, err := px.requestFunction(px, apiUrl, http.MethodGet, nil)
+	if err != nil {
+		return VmStats{}, err
+	}
+
+	var vmStats VmStats
+	err = json.Unmarshal(jsonData, &vmStats)
+	if err != nil {
+		return VmStats{}, err
+	}
+
+	return vmStats, nil
+}
+
+func getVmConfig(px *Proxmox, vmId string, rt ResourceType) (VmConfig, error) {
+	apiUrl := "/nodes/" + px.hostname + "/" + string(rt) + "/" + vmId + "/config"
+	jsonData, err := px.requestFunction(px, apiUrl, http.MethodGet, nil)
+	if err != nil {
+		return VmConfig{}, err
+	}
+
+	var vmConfig VmConfig
+	err = json.Unmarshal(jsonData, &vmConfig)
+	if err != nil {
+		return VmConfig{}, err
+	}
+
+	return vmConfig, nil
+}
+
+func getFields(vmStat VmStat) (map[string]interface{}, error) {
+	mem_total, mem_used, mem_free, mem_used_percentage := getByteMetrics(vmStat.TotalMem, vmStat.UsedMem)
+	swap_total, swap_used, swap_free, swap_used_percentage := getByteMetrics(vmStat.TotalSwap, vmStat.UsedSwap)
+	disk_total, disk_used, disk_free, disk_used_percentage := getByteMetrics(vmStat.TotalDisk, vmStat.UsedDisk)
+
+	return map[string]interface{}{
+		"status":               vmStat.Status,
+		"uptime":               jsonNumberToInt64(vmStat.Uptime),
+		"cpuload":              jsonNumberToFloat64(vmStat.CpuLoad),
+		"mem_used":             mem_used,
+		"mem_total":            mem_total,
+		"mem_free":             mem_free,
+		"mem_used_percentage":  mem_used_percentage,
+		"swap_used":            swap_used,
+		"swap_total":           swap_total,
+		"swap_free":            swap_free,
+		"swap_used_percentage": swap_used_percentage,
+		"disk_used":            disk_used,
+		"disk_total":           disk_total,
+		"disk_free":            disk_free,
+		"disk_used_percentage": disk_used_percentage,
+	}, nil
+}
+
+func getByteMetrics(total json.Number, used json.Number) (int64, int64, int64, float64) {
+	int64Total := jsonNumberToInt64(total)
+	int64Used := jsonNumberToInt64(used)
+	int64Free := int64Total - int64Used
+	usedPercentage := 0.0
+	if int64Total != 0 {
+		usedPercentage = float64(int64Used) * 100 / float64(int64Total)
+	}
+
+	return int64Total, int64Used, int64Free, usedPercentage
+}
+
+func jsonNumberToInt64(value json.Number) int64 {
+	int64Value, err := value.Int64()
+	if err != nil {
+		return 0
+	}
+
+	return int64Value
+}
+
+func jsonNumberToFloat64(value json.Number) float64 {
+	float64Value, err := value.Float64()
+	if err != nil {
+		return 0
+	}
+
+	return float64Value
+}
+
+func getTags(px *Proxmox, name string, vmConfig VmConfig, rt ResourceType) map[string]string {
+	domain := vmConfig.Data.Searchdomain
+	if len(domain) == 0 {
+		domain = px.nodeSearchDomain
+	}
+
+	hostname := vmConfig.Data.Hostname
+	if len(hostname) == 0 {
+		hostname = name
+	}
+	fqdn := hostname + "." + domain
+
+	return map[string]string{
+		"node_fqdn": px.hostname + "." + px.nodeSearchDomain,
+		"vm_name":   name,
+		"vm_fqdn":   fqdn,
+		"vm_type":   string(rt),
+	}
+}

plugins/inputs/proxmox/proxmox_test.go

@@ -0,0 +1,136 @@
+package proxmox
+
+import (
+	"github.com/bmizerany/assert"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/require"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+var nodeSearchDomainTestData = `{"data":{"search":"test.example.com","dns1":"1.0.0.1"}}`
+var qemuTestData = `{"data":[{"name":"qemu1","status":"running","maxdisk":10737418240,"cpu":0.029336643550795,"vmid":"113","uptime":2159739,"disk":0,"maxmem":2147483648,"mem":1722451796}]}`
+var qemuConfigTestData = `{"data":{"hostname":"qemu1","searchdomain":"test.example.com"}}`
+var lxcTestData = `{"data":[{"vmid":"111","type":"lxc","uptime":2078164,"swap":9412608,"disk":"744189952","maxmem":536870912,"mem":98500608,"maxswap":536870912,"cpu":0.00371567669193613,"status":"running","maxdisk":"5217320960","name":"container1"}]}`
+var lxcConfigTestData = `{"data":{"hostname":"container1","searchdomain":"test.example.com"}}`
+
+func performTestRequest(px *Proxmox, apiUrl string, method string, data url.Values) ([]byte, error) {
+	var bytedata = []byte("")
+
+	if strings.HasSuffix(apiUrl, "dns") {
+		bytedata = []byte(nodeSearchDomainTestData)
+	} else if strings.HasSuffix(apiUrl, "qemu") {
+		bytedata = []byte(qemuTestData)
+	} else if strings.HasSuffix(apiUrl, "113/config") {
+		bytedata = []byte(qemuConfigTestData)
+	} else if strings.HasSuffix(apiUrl, "lxc") {
+		bytedata = []byte(lxcTestData)
+	} else if strings.HasSuffix(apiUrl, "111/config") {
+		bytedata = []byte(lxcConfigTestData)
+	}
+
+	return bytedata, nil
+}
+
+func setUp(t *testing.T) *Proxmox {
+	px := &Proxmox{
+		requestFunction: performTestRequest,
+	}
+
+	require.NoError(t, px.Init())
+
+	// Override hostname and logger for test
+	px.hostname = "testnode"
+	px.Log = testutil.Logger{}
+	return px
+}
+
+func TestGetNodeSearchDomain(t *testing.T) {
+	px := setUp(t)
+
+	err := getNodeSearchDomain(px)
+
+	require.NoError(t, err)
+	assert.Equal(t, px.nodeSearchDomain, "test.example.com")
+}
+
+func TestGatherLxcData(t *testing.T) {
+	px := setUp(t)
+	px.nodeSearchDomain = "test.example.com"
+
+	acc := &testutil.Accumulator{}
+	gatherLxcData(px, acc)
+
+	assert.Equal(t, acc.NFields(), 15)
+	testFields := map[string]interface{}{
+		"status":               "running",
+		"uptime":               int64(2078164),
+		"cpuload":              float64(0.00371567669193613),
+		"mem_used":             int64(98500608),
+		"mem_total":            int64(536870912),
+		"mem_free":             int64(438370304),
+		"mem_used_percentage":  float64(18.34716796875),
+		"swap_used":            int64(9412608),
+		"swap_total":           int64(536870912),
+		"swap_free":            int64(527458304),
+		"swap_used_percentage": float64(1.75323486328125),
+		"disk_used":            int64(744189952),
+		"disk_total":           int64(5217320960),
+		"disk_free":            int64(4473131008),
+		"disk_used_percentage": float64(14.26383306117322),
+	}
+	testTags := map[string]string{
+		"node_fqdn": "testnode.test.example.com",
+		"vm_name":   "container1",
+		"vm_fqdn":   "container1.test.example.com",
+		"vm_type":   "lxc",
+	}
+	acc.AssertContainsTaggedFields(t, "proxmox", testFields, testTags)
+}
+
+func TestGatherQemuData(t *testing.T) {
+	px := setUp(t)
+	px.nodeSearchDomain = "test.example.com"
+
+	acc := &testutil.Accumulator{}
+	gatherQemuData(px, acc)
+
+	assert.Equal(t, acc.NFields(), 15)
+	testFields := map[string]interface{}{
+		"status":               "running",
+		"uptime":               int64(2159739),
+		"cpuload":              float64(0.029336643550795),
+		"mem_used":             int64(1722451796),
+		"mem_total":            int64(2147483648),
+		"mem_free":             int64(425031852),
+		"mem_used_percentage":  float64(80.20791206508875),
+		"swap_used":            int64(0),
+		"swap_total":           int64(0),
+		"swap_free":            int64(0),
+		"swap_used_percentage": float64(0),
+		"disk_used":            int64(0),
+		"disk_total":           int64(10737418240),
+		"disk_free":            int64(10737418240),
+		"disk_used_percentage": float64(0),
+	}
+	testTags := map[string]string{
+		"node_fqdn": "testnode.test.example.com",
+		"vm_name":   "qemu1",
+		"vm_fqdn":   "qemu1.test.example.com",
+		"vm_type":   "qemu",
+	}
+	acc.AssertContainsTaggedFields(t, "proxmox", testFields, testTags)
+}
+
+func TestGather(t *testing.T) {
+	px := setUp(t)
+	px.nodeSearchDomain = "test.example.com"
+
+	acc := &testutil.Accumulator{}
+	err := px.Gather(acc)
+	require.NoError(t, err)
+
+	// Results from both tests above
+	assert.Equal(t, acc.NFields(), 30)
+}

plugins/inputs/proxmox/structs.go

@@ -0,0 +1,62 @@
+package proxmox
+
+import (
+	"encoding/json"
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/common/tls"
+	"net/http"
+	"net/url"
+)
+
+type Proxmox struct {
+	BaseURL         string            `toml:"base_url"`
+	APIToken        string            `toml:"api_token"`
+	ResponseTimeout internal.Duration `toml:"response_timeout"`
+	tls.ClientConfig
+
+	hostname         string
+	httpClient       *http.Client
+	nodeSearchDomain string
+
+	requestFunction func(px *Proxmox, apiUrl string, method string, data url.Values) ([]byte, error)
+	Log             telegraf.Logger `toml:"-"`
+}
+
+type ResourceType string
+
+var (
+	QEMU ResourceType = "qemu"
+	LXC  ResourceType = "lxc"
+)
+
+type VmStats struct {
+	Data []VmStat `json:"data"`
+}
+
+type VmStat struct {
+	ID        string      `json:"vmid"`
+	Name      string      `json:"name"`
+	Status    string      `json:"status"`
+	UsedMem   json.Number `json:"mem"`
+	TotalMem  json.Number `json:"maxmem"`
+	UsedDisk  json.Number `json:"disk"`
+	TotalDisk json.Number `json:"maxdisk"`
+	UsedSwap  json.Number `json:"swap"`
+	TotalSwap json.Number `json:"maxswap"`
+	Uptime    json.Number `json:"uptime"`
+	CpuLoad   json.Number `json:"cpu"`
+}
+
+type VmConfig struct {
+	Data struct {
+		Searchdomain string `json:"searchdomain"`
+		Hostname     string `json:"hostname"`
+	} `json:"data"`
+}
+
+type NodeDns struct {
+	Data struct {
+		Searchdomain string `json:"search"`
+	} `json:"data"`
+}