feat(inputs.icinga2): Support collecting hosts,services and endpoint metrics (#12506)

This commit is contained in:
Tomas Barton 2023-01-26 11:16:35 +01:00 committed by GitHub
parent 4b445f6a26
commit d3809956a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 569 additions and 159 deletions

View File

@ -24,9 +24,15 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
[[inputs.icinga2]]
## Required Icinga2 server address
# server = "https://localhost:5665"
## Required Icinga2 object type ("services" or "hosts")
# object_type = "services"
## Collected Icinga2 objects ("services", "hosts")
## Specify at least one object to collect from /v1/objects endpoint.
# objects = ["services"]
## Collect metrics from /v1/status endpoint
## Choose from:
## "ApiListener", "CIB", "IdoMysqlConnection", "IdoPgsqlConnection"
# status = []
## Credentials for basic HTTP authentication
# username = "admin"
@ -43,22 +49,118 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
# insecure_skip_verify = true
```
## Measurements & Fields
## Metrics
- All measurements have the following fields:
- name (string)
- state_code (int)
## Tags
- All measurements have the following tags:
- check_command - The short name of the check command
- display_name - The name of the service or host
- state - The state: UP/DOWN for hosts, OK/WARNING/CRITICAL/UNKNOWN for services
- source - The icinga2 host
- port - The icinga2 port
- scheme - The icinga2 protocol (http/https)
- server - The server the check_command is running for
- `icinga2_hosts`
- tags
- `check_command` - The short name of the check command
- `display_name` - The name of the host
- `state` - The state: UP/DOWN
- `source` - The icinga2 host
- `port` - The icinga2 port
- `scheme` - The icinga2 protocol (http/https)
- `server` - The server the check_command is running for
- fields
- `name` (string)
- `state_code` (int)
- `icinga2_services`
- tags
- `check_command` - The short name of the check command
- `display_name` - The name of the service
- `state` - The state: OK/WARNING/CRITICAL/UNKNOWN for services
- `source` - The icinga2 host
- `port` - The icinga2 port
- `scheme` - The icinga2 protocol (http/https)
- `server` - The server the check_command is running for
- fields
- `name` (string)
- `state_code` (int)
- `icinga2_status`
- component:
- `ApiListener`
- tags
- `component` name
- fields
- `api_num_conn_endpoints`
- `api_num_endpoint`
- `api_num_http_clients`
- `api_num_json_rpc_anonymous_clients`
- `api_num_json_rpc_relay_queue_item_rate`
- `api_num_json_rpc_relay_queue_items`
- `api_num_json_rpc_sync_queue_item_rate`
- `api_num_json_rpc_sync_queue_items`
- `api_num_json_rpc_work_queue_item_rate`
- `api_num_not_conn_endpoints`
- `CIB`
- tags
- `component` name
- fields
- `active_host_checks`
- `active_host_checks_15min`
- `active_host_checks_1min`
- `active_host_checks_5min`
- `active_service_checks`
- `active_service_checks_15min`
- `active_service_checks_1min`
- `active_service_checks_5min`
- `avg_execution_time`
- `avg_latency`
- `current_concurrent_checks`
- `current_pending_callbacks`
- `max_execution_time`
- `max_latency`
- `min_execution_time`
- `min_latency`
- `num_hosts_acknowledged`
- `num_hosts_down`
- `num_hosts_flapping`
- `num_hosts_handled`
- `num_hosts_in_downtime`
- `num_hosts_pending`
- `num_hosts_problem`
- `num_hosts_unreachable`
- `num_hosts_up`
- `num_services_acknowledged`
- `num_services_critical`
- `num_services_flapping`
- `num_services_handled`
- `num_services_in_downtime`
- `num_services_ok`
- `num_services_pending`
- `num_services_problem`
- `num_services_unknown`
- `num_services_unreachable`
- `num_services_warning`
- `passive_host_checks`
- `passive_host_checks_15min`
- `passive_host_checks_1min`
- `passive_host_checks_5min`
- `passive_service_checks`
- `passive_service_checks_15min`
- `passive_service_checks_1min`
- `passive_service_checks_5min`
- `remote_check_queue`
- `uptime`
- `IdoMysqlConnection`
- tags
- `component` name
- fields
- `mysql_queries_1min`
- `mysql_queries_5mins`
- `mysql_queries_15mins`
- `mysql_queries_rate`
- `mysql_query_queue_item_rate`
- `mysql_query_queue_items`
- `IdoPgsqlConnection`
- tags
- `component` name
- fields
- `pgsql_queries_1min`
- `pgsql_queries_5mins`
- `pgsql_queries_15mins`
- `pgsql_queries_rate`
- `pgsql_query_queue_item_rate`
- `pgsql_query_queue_items`
## Sample Queries

View File

@ -4,13 +4,16 @@ package icinga2
import (
_ "embed"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/internal/choice"
"github.com/influxdata/telegraf/plugins/common/tls"
"github.com/influxdata/telegraf/plugins/inputs"
)
@ -20,7 +23,9 @@ var sampleConfig string
type Icinga2 struct {
Server string
ObjectType string
Objects []string
Status []string
ObjectType string `toml:"object_type" deprecated:"1.26.0;2.0.0;use 'objects' instead"`
Username string
Password string
ResponseTimeout config.Duration
@ -31,36 +36,75 @@ type Icinga2 struct {
client *http.Client
}
type Result struct {
Results []Object `json:"results"`
type ResultObject struct {
Results []struct {
Attrs struct {
CheckCommand string `json:"check_command"`
DisplayName string `json:"display_name"`
Name string `json:"name"`
State float64 `json:"state"`
HostName string `json:"host_name"`
} `json:"attrs"`
Name string `json:"name"`
Joins struct{} `json:"joins"`
Meta struct{} `json:"meta"`
Type string `json:"type"`
} `json:"results"`
}
type Object struct {
Attrs Attribute `json:"attrs"`
Name string `json:"name"`
Joins struct{} `json:"joins"`
Meta struct{} `json:"meta"`
Type ObjectType `json:"type"`
type ResultCIB struct {
Results []struct {
Status map[string]interface{} `json:"status"`
} `json:"results"`
}
type Attribute struct {
CheckCommand string `json:"check_command"`
DisplayName string `json:"display_name"`
Name string `json:"name"`
State float64 `json:"state"`
HostName string `json:"host_name"`
type ResultPerfdata struct {
Results []struct {
Perfdata []struct {
Label string `json:"label"`
Value float64 `json:"value"`
} `json:"perfdata"`
} `json:"results"`
}
var levels = []string{"ok", "warning", "critical", "unknown"}
type ObjectType string
func (*Icinga2) SampleConfig() string {
return sampleConfig
}
func (i *Icinga2) GatherStatus(acc telegraf.Accumulator, checks []Object) {
for _, check := range checks {
func (i *Icinga2) Init() error {
statusEndpoints := []string{"ApiListener", "CIB", "IdoMysqlConnection", "IdoPgsqlConnection"}
if err := choice.CheckSlice(i.Status, statusEndpoints); err != nil {
return fmt.Errorf("config option 'status': %w", err)
}
if i.ResponseTimeout < config.Duration(time.Second) {
i.ResponseTimeout = config.Duration(time.Second * 5)
}
client, err := i.createHTTPClient()
if err != nil {
return err
}
i.client = client
// For backward config compatibility
// should be removed in 2.0.0
if i.ObjectType != "" {
i.Objects = []string{i.ObjectType}
}
objectEndpoints := []string{"services", "hosts"}
if err := choice.CheckSlice(i.Objects, objectEndpoints); err != nil {
return fmt.Errorf("config option 'objects': %w", err)
}
return nil
}
func (i *Icinga2) gatherObjects(acc telegraf.Accumulator, checks ResultObject, objectType string) {
for _, check := range checks.Results {
serverURL, err := url.Parse(i.Server)
if err != nil {
i.Log.Error(err.Error())
@ -76,7 +120,7 @@ func (i *Icinga2) GatherStatus(acc telegraf.Accumulator, checks []Object) {
// source is dependent on 'services' or 'hosts' check
source := check.Attrs.Name
if i.ObjectType == "services" {
if objectType == "services" {
source = check.Attrs.HostName
}
@ -90,7 +134,7 @@ func (i *Icinga2) GatherStatus(acc telegraf.Accumulator, checks []Object) {
"port": serverURL.Port(),
}
acc.AddFields(fmt.Sprintf("icinga2_%s", i.ObjectType), fields, tags)
acc.AddFields(fmt.Sprintf("icinga2_%s", objectType), fields, tags)
}
}
@ -110,32 +154,10 @@ func (i *Icinga2) createHTTPClient() (*http.Client, error) {
return client, nil
}
func (i *Icinga2) Gather(acc telegraf.Accumulator) error {
if i.ResponseTimeout < config.Duration(time.Second) {
i.ResponseTimeout = config.Duration(time.Second * 5)
}
if i.client == nil {
client, err := i.createHTTPClient()
if err != nil {
return err
}
i.client = client
}
requestURL := "%s/v1/objects/%s?attrs=name&attrs=display_name&attrs=state&attrs=check_command"
// Note: attrs=host_name is only valid for 'services' requests, using check.Attrs.HostName for the host
// 'hosts' requests will need to use attrs=name only, using check.Attrs.Name for the host
if i.ObjectType == "services" {
requestURL += "&attrs=host_name"
}
address := fmt.Sprintf(requestURL, i.Server, i.ObjectType)
func (i *Icinga2) icingaRequest(address string) (*http.Response, error) {
req, err := http.NewRequest("GET", address, nil)
if err != nil {
return err
return nil, err
}
if i.Username != "" {
@ -144,18 +166,125 @@ func (i *Icinga2) Gather(acc telegraf.Accumulator) error {
resp, err := i.client.Do(req)
if err != nil {
return err
return nil, err
}
defer resp.Body.Close()
return resp, nil
}
result := Result{}
err = json.NewDecoder(resp.Body).Decode(&result)
func (i *Icinga2) parseObjectResponse(resp *http.Response, result *ResultObject) error {
err := json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
return err
}
err = resp.Body.Close()
if err != nil {
return err
}
i.GatherStatus(acc, result.Results)
return nil
}
func (i *Icinga2) parseCIBResponse(resp *http.Response) (map[string]interface{}, error) {
result := ResultCIB{}
err := json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if len(result.Results) == 0 {
return nil, errors.New("no results in Icinga2 API response")
}
return result.Results[0].Status, nil
}
func (i *Icinga2) parsePerfdataResponse(resp *http.Response) (map[string]interface{}, error) {
result := ResultPerfdata{}
err := json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if len(result.Results) == 0 {
return nil, errors.New("no results in Icinga2 API response")
}
fields := make(map[string]interface{})
for _, item := range result.Results[0].Perfdata {
i := strings.Index(item.Label, "-")
if i > 0 {
fields[item.Label[i+1:]] = item.Value
} else {
fields[item.Label] = item.Value
}
}
return fields, nil
}
func (i *Icinga2) Gather(acc telegraf.Accumulator) error {
// Collect /v1/objects
for _, objectType := range i.Objects {
requestURL := "%s/v1/objects/%s?attrs=name&attrs=display_name&attrs=state&attrs=check_command"
// Note: attrs=host_name is only valid for 'services' requests, using check.Attrs.HostName for the host
// 'hosts' requests will need to use attrs=name only, using check.Attrs.Name for the host
if objectType == "services" {
requestURL += "&attrs=host_name"
}
address := fmt.Sprintf(requestURL, i.Server, objectType)
resp, err := i.icingaRequest(address)
if err != nil {
return err
}
result := ResultObject{}
err = i.parseObjectResponse(resp, &result)
if err != nil {
return fmt.Errorf("could not parse object response: %w", err)
}
i.gatherObjects(acc, result, objectType)
}
// Collect /v1/status
for _, statusType := range i.Status {
address := fmt.Sprintf("%s/v1/status/%s", i.Server, statusType)
resp, err := i.icingaRequest(address)
if err != nil {
return err
}
tags := map[string]string{
"component": statusType,
}
var fields map[string]interface{}
switch statusType {
case "ApiListener":
fields, err = i.parsePerfdataResponse(resp)
case "CIB":
fields, err = i.parseCIBResponse(resp)
case "IdoMysqlConnection":
fields, err = i.parsePerfdataResponse(resp)
case "IdoPgsqlConnection":
fields, err = i.parsePerfdataResponse(resp)
}
if err != nil {
return fmt.Errorf("could not parse %s response: %w", statusType, err)
}
acc.AddFields("icinga2_status", fields, tags)
}
return nil
}
@ -164,7 +293,7 @@ func init() {
inputs.Add("icinga2", func() telegraf.Input {
return &Icinga2{
Server: "https://localhost:5665",
ObjectType: "services",
Objects: []string{"services"},
ResponseTimeout: config.Duration(time.Second * 5),
}
})

View File

@ -1,120 +1,293 @@
package icinga2
import (
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/testutil"
"github.com/stretchr/testify/require"
)
func TestGatherServicesStatus(t *testing.T) {
s := `{
"results": [
{
"attrs": {
"check_command": "check-bgp-juniper-netconf",
"display_name": "eq-par.dc2.fr",
"host_name": "someserverfqdn.net",
"name": "ef017af8-c684-4f3f-bb20-0dfe9fcd3dbe",
"state": 0
},
"joins": {},
"meta": {},
"name": "eq-par.dc2.fr!ef017af8-c684-4f3f-bb20-0dfe9fcd3dbe",
"type": "Service"
}
]
func TestIcinga2Default(t *testing.T) {
// This test should succeed with the default initialization.
icinga2 := &Icinga2{
Server: "https://localhost:5665",
Objects: []string{"services"},
ResponseTimeout: config.Duration(time.Second * 5),
}
require.NoError(t, icinga2.Init())
require.Equal(t, config.Duration(5*time.Second), icinga2.ResponseTimeout)
require.Equal(t, "https://localhost:5665", icinga2.Server)
require.Equal(t, []string{"services"}, icinga2.Objects)
}
`
checks := Result{}
require.NoError(t, json.Unmarshal([]byte(s), &checks))
func TestIcinga2DeprecatedHostConfig(t *testing.T) {
icinga2 := &Icinga2{
ObjectType: "hosts", //deprecated
Objects: []string{},
}
require.NoError(t, icinga2.Init())
icinga2 := new(Icinga2)
icinga2.Log = testutil.Logger{}
icinga2.ObjectType = "services"
icinga2.Server = "https://localhost:5665"
require.Equal(t, []string{"hosts"}, icinga2.Objects)
}
var acc testutil.Accumulator
icinga2.GatherStatus(&acc, checks.Results)
func TestIcinga2DeprecatedServicesConfig(t *testing.T) {
icinga2 := &Icinga2{
ObjectType: "services", //deprecated
Objects: []string{},
}
require.NoError(t, icinga2.Init())
expected := []telegraf.Metric{
testutil.MustMetric(
"icinga2_services",
map[string]string{
"display_name": "eq-par.dc2.fr",
require.Equal(t, []string{"services"}, icinga2.Objects)
}
const icinga2ServiceResponse = `{
"results": [
{
"attrs": {
"check_command": "check-bgp-juniper-netconf",
"state": "ok",
"source": "someserverfqdn.net",
"server": "localhost",
"port": "5665",
"scheme": "https",
"display_name": "eq-par.dc2.fr",
"host_name": "someserverfqdn.net",
"name": "ef017af8-c684-4f3f-bb20-0dfe9fcd3dbe",
"state": 0
},
map[string]interface{}{
"name": "ef017af8-c684-4f3f-bb20-0dfe9fcd3dbe",
"state_code": 0,
},
time.Unix(0, 0),
),
"joins": {},
"meta": {},
"name": "eq-par.dc2.fr!ef017af8-c684-4f3f-bb20-0dfe9fcd3dbe",
"type": "Service"
}
]
}`
func TestGatherServicesStatus(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/objects/services" {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
_, err := w.Write([]byte(icinga2ServiceResponse))
require.NoError(t, err)
} else {
w.WriteHeader(http.StatusNotFound)
t.Logf("Req: %s %s\n", r.Host, r.URL.Path)
}
}))
defer ts.Close()
var icinga2 = &Icinga2{
Server: ts.URL,
Objects: []string{"services"},
}
require.NoError(t, icinga2.Init())
var acc testutil.Accumulator
err := icinga2.Gather(&acc)
require.NoError(t, err)
requestURL, err := url.Parse(ts.URL)
require.NoError(t, err)
expectedFields := map[string]interface{}{
"name": "ef017af8-c684-4f3f-bb20-0dfe9fcd3dbe",
"state_code": int64(0),
}
testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics(), testutil.IgnoreTime())
expectedTags := map[string]string{
"display_name": "eq-par.dc2.fr",
"check_command": "check-bgp-juniper-netconf",
"state": "ok",
"source": "someserverfqdn.net",
"server": requestURL.Hostname(),
"port": requestURL.Port(),
"scheme": "http",
}
acc.AssertContainsTaggedFields(t, "icinga2_services", expectedFields, expectedTags)
}
const icinga2HostResponse = `{
"results": [
{
"attrs": {
"address": "192.168.1.1",
"check_command": "ping",
"display_name": "apache",
"name": "webserver",
"state": 2.0
},
"joins": {},
"meta": {},
"name": "webserver",
"type": "Host"
}
]
}
`
func TestGatherHostsStatus(t *testing.T) {
s := `{
"results": [
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/objects/hosts" {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
_, err := w.Write([]byte(icinga2HostResponse))
require.NoError(t, err)
} else {
w.WriteHeader(http.StatusNotFound)
t.Logf("Req: %s %s\n", r.Host, r.URL.Path)
}
}))
defer ts.Close()
var icinga2 = &Icinga2{
Server: ts.URL,
Objects: []string{"hosts"},
}
require.NoError(t, icinga2.Init())
requestURL, err := url.Parse(ts.URL)
require.NoError(t, err)
var acc testutil.Accumulator
err = icinga2.Gather(&acc)
require.NoError(t, err)
expectedFields := map[string]interface{}{
"name": "webserver",
"state_code": int64(2),
}
expectedTags := map[string]string{
"display_name": "apache",
"check_command": "ping",
"state": "critical",
"source": "webserver",
"server": requestURL.Hostname(),
"port": requestURL.Port(),
"scheme": "http",
}
acc.AssertContainsTaggedFields(t, "icinga2_hosts", expectedFields, expectedTags)
}
const icinga2StatusCIB = `{
"results": [
{
"name": "CIB",
"perfdata": [],
"status": {
"active_host_checks": 3.6,
"avg_latency": 2.187678621145969e-06,
"max_latency": 0.001603841781616211
}
}
]
}`
func TestGatherStatusCIB(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/status/CIB" {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
_, err := w.Write([]byte(icinga2StatusCIB))
require.NoError(t, err)
} else {
w.WriteHeader(http.StatusNotFound)
t.Logf("Req: %s %s\n", r.Host, r.URL.Path)
}
}))
defer ts.Close()
var icinga2 = &Icinga2{
Server: ts.URL,
Status: []string{"CIB"},
}
require.NoError(t, icinga2.Init())
var acc testutil.Accumulator
err := icinga2.Gather(&acc)
require.NoError(t, err)
expectedFields := map[string]interface{}{
"active_host_checks": float64(3.6),
"avg_latency": float64(2.187678621145969e-06),
"max_latency": float64(0.001603841781616211),
}
expectedTags := map[string]string{
"component": "CIB",
}
acc.AssertContainsTaggedFields(t, "icinga2_status", expectedFields, expectedTags)
}
const icinga2StatusPgsql = `{
"results": [
{
"name": "IdoPgsqlConnection",
"perfdata": [
{
"attrs": {
"address": "192.168.1.1",
"check_command": "ping",
"display_name": "apache",
"name": "webserver",
"state": 2.0
},
"joins": {},
"meta": {},
"name": "webserver",
"type": "Host"
"counter": false,
"crit": null,
"label": "idopgsqlconnection_ido-pgsql_queries_rate",
"max": null,
"min": null,
"type": "PerfdataValue",
"unit": "",
"value": 649.8666666666667,
"warn": null
},
{
"counter": false,
"crit": null,
"label": "idopgsqlconnection_ido-pgsql_query_queue_item_rate",
"max": null,
"min": null,
"type": "PerfdataValue",
"unit": "",
"value": 1295.1166666666666,
"warn": null
}
]
]
}
]
}
`
checks := Result{}
require.NoError(t, json.Unmarshal([]byte(s), &checks))
func TestGatherStatusPgsql(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/status/IdoPgsqlConnection" {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
_, err := w.Write([]byte(icinga2StatusPgsql))
require.NoError(t, err)
} else {
w.WriteHeader(http.StatusNotFound)
t.Logf("Req: %s %s\n", r.Host, r.URL.Path)
}
}))
defer ts.Close()
var icinga2 = &Icinga2{
Server: ts.URL,
Status: []string{"IdoPgsqlConnection"},
}
require.NoError(t, icinga2.Init())
var acc testutil.Accumulator
err := icinga2.Gather(&acc)
require.NoError(t, err)
icinga2 := new(Icinga2)
icinga2.Log = testutil.Logger{}
icinga2.ObjectType = "hosts"
icinga2.Server = "https://localhost:5665"
icinga2.GatherStatus(&acc, checks.Results)
expected := []telegraf.Metric{
testutil.MustMetric(
"icinga2_hosts",
map[string]string{
"display_name": "apache",
"check_command": "ping",
"state": "critical",
"source": "webserver",
"server": "localhost",
"port": "5665",
"scheme": "https",
},
map[string]interface{}{
"name": "webserver",
"state_code": 2,
},
time.Unix(0, 0),
),
expectedFields := map[string]interface{}{
"pgsql_queries_rate": float64(649.8666666666667),
"pgsql_query_queue_item_rate": float64(1295.1166666666666),
}
testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics(), testutil.IgnoreTime())
expectedTags := map[string]string{
"component": "IdoPgsqlConnection",
}
acc.AssertContainsTaggedFields(t, "icinga2_status", expectedFields, expectedTags)
}

View File

@ -2,9 +2,15 @@
[[inputs.icinga2]]
## Required Icinga2 server address
# server = "https://localhost:5665"
## Required Icinga2 object type ("services" or "hosts")
# object_type = "services"
## Collected Icinga2 objects ("services", "hosts")
## Specify at least one object to collect from /v1/objects endpoint.
# objects = ["services"]
## Collect metrics from /v1/status endpoint
## Choose from:
## "ApiListener", "CIB", "IdoMysqlConnection", "IdoPgsqlConnection"
# status = []
## Credentials for basic HTTP authentication
# username = "admin"