i-prudnikov
GitHub
parent
c0d5af1602
commit
e3ae7caaf0
|
|
@ -36,7 +36,26 @@ In the following order the plugin will attempt to authenticate.
|
|||
# private_key = ""
|
||||
# public_key_id = ""
|
||||
# role_name = ""
|
||||
|
||||
|
||||
## Specify the ali cloud region list to be queried for metrics and objects discovery
|
||||
## If not set, all supported regions (see below) would be covered, it can provide a significant load on API, so the recommendation here
|
||||
## is to limit the list as much as possible. Allowed values: https://www.alibabacloud.com/help/zh/doc-detail/40654.htm
|
||||
## Default supported regions are:
|
||||
## 21 items: cn-qingdao,cn-beijing,cn-zhangjiakou,cn-huhehaote,cn-hangzhou,cn-shanghai,cn-shenzhen,
|
||||
## cn-heyuan,cn-chengdu,cn-hongkong,ap-southeast-1,ap-southeast-2,ap-southeast-3,ap-southeast-5,
|
||||
## ap-south-1,ap-northeast-1,us-west-1,us-east-1,eu-central-1,eu-west-1,me-east-1
|
||||
##
|
||||
## From discovery perspective it set the scope for object discovery, the discovered info can be used to enrich
|
||||
## the metrics with objects attributes/tags. Discovery is supported not for all projects (if not supported, then
|
||||
## it will be reported on the start - for example for 'acs_cdn' project:
|
||||
## 'E! [inputs.aliyuncms] Discovery tool is not activated: no discovery support for project "acs_cdn"' )
|
||||
## Currently, discovery supported for the following projects:
|
||||
## - acs_ecs_dashboard
|
||||
## - acs_rds_dashboard
|
||||
## - acs_slb_dashboard
|
||||
## - acs_vpc_eip
|
||||
regions = ["cn-hongkong"]
|
||||
|
||||
# The minimum period for AliyunCMS metrics is 1 minute (60s). However not all
|
||||
# metrics are made available to the 1 minute period. Some are collected at
|
||||
# 3 minute, 5 minute, or larger intervals.
|
||||
|
|
@ -61,21 +80,7 @@ In the following order the plugin will attempt to authenticate.
|
|||
## Maximum requests per second, default value is 200
|
||||
ratelimit = 200
|
||||
|
||||
## Discovery regions set the scope for object discovery, the discovered info can be used to enrich
|
||||
## the metrics with objects attributes/tags. Discovery is supported not for all projects (if not supported, then
|
||||
## it will be reported on the start - foo example for 'acs_cdn' project:
|
||||
## 'E! [inputs.aliyuncms] Discovery tool is not activated: no discovery support for project "acs_cdn"' )
|
||||
## Currently, discovery supported for the following projects:
|
||||
## - acs_ecs_dashboard
|
||||
## - acs_rds_dashboard
|
||||
## - acs_slb_dashboard
|
||||
## - acs_vpc_eip
|
||||
##
|
||||
## If not set, all regions would be covered, it can provide a significant load on API, so the recommendation here
|
||||
## is to limit the list as much as possible. Allowed values: https://www.alibabacloud.com/help/zh/doc-detail/40654.htm
|
||||
discovery_regions = ["cn-hongkong"]
|
||||
|
||||
## how often the discovery API call executed (default 1m)
|
||||
## How often the discovery API call executed (default 1m)
|
||||
#discovery_interval = "1m"
|
||||
|
||||
## Metrics to Pull (Required)
|
||||
|
|
|
|||
|
|
@ -11,112 +11,116 @@ import (
|
|||
"github.com/aliyun/alibaba-cloud-sdk-go/sdk"
|
||||
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials/providers"
|
||||
"github.com/aliyun/alibaba-cloud-sdk-go/services/cms"
|
||||
"github.com/jmespath/go-jmespath"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/config"
|
||||
"github.com/influxdata/telegraf/internal"
|
||||
"github.com/influxdata/telegraf/internal/limiter"
|
||||
"github.com/influxdata/telegraf/plugins/inputs"
|
||||
"github.com/jmespath/go-jmespath"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
description = "Pull Metric Statistics from Aliyun CMS"
|
||||
sampleConfig = `
|
||||
## Aliyun Credentials
|
||||
## Credentials are loaded in the following order
|
||||
## 1) Ram RoleArn credential
|
||||
## 2) AccessKey STS token credential
|
||||
## 3) AccessKey credential
|
||||
## 4) Ecs Ram Role credential
|
||||
## 5) RSA keypair credential
|
||||
## 6) Environment variables credential
|
||||
## 7) Instance metadata credential
|
||||
## Aliyun Credentials
|
||||
## Credentials are loaded in the following order
|
||||
## 1) Ram RoleArn credential
|
||||
## 2) AccessKey STS token credential
|
||||
## 3) AccessKey credential
|
||||
## 4) Ecs Ram Role credential
|
||||
## 5) RSA keypair credential
|
||||
## 6) Environment variables credential
|
||||
## 7) Instance metadata credential
|
||||
|
||||
# access_key_id = ""
|
||||
# access_key_secret = ""
|
||||
# access_key_sts_token = ""
|
||||
# role_arn = ""
|
||||
# role_session_name = ""
|
||||
# private_key = ""
|
||||
# public_key_id = ""
|
||||
# role_name = ""
|
||||
|
||||
# access_key_id = ""
|
||||
# access_key_secret = ""
|
||||
# access_key_sts_token = ""
|
||||
# role_arn = ""
|
||||
# role_session_name = ""
|
||||
# private_key = ""
|
||||
# public_key_id = ""
|
||||
# role_name = ""
|
||||
## Specify the ali cloud region list to be queried for metrics and objects discovery
|
||||
## If not set, all supported regions (see below) would be covered, it can provide a significant load on API, so the recommendation here
|
||||
## is to limit the list as much as possible. Allowed values: https://www.alibabacloud.com/help/zh/doc-detail/40654.htm
|
||||
## Default supported regions are:
|
||||
## 21 items: cn-qingdao,cn-beijing,cn-zhangjiakou,cn-huhehaote,cn-hangzhou,cn-shanghai,cn-shenzhen,
|
||||
## cn-heyuan,cn-chengdu,cn-hongkong,ap-southeast-1,ap-southeast-2,ap-southeast-3,ap-southeast-5,
|
||||
## ap-south-1,ap-northeast-1,us-west-1,us-east-1,eu-central-1,eu-west-1,me-east-1
|
||||
##
|
||||
## From discovery perspective it set the scope for object discovery, the discovered info can be used to enrich
|
||||
## the metrics with objects attributes/tags. Discovery is supported not for all projects (if not supported, then
|
||||
## it will be reported on the start - for example for 'acs_cdn' project:
|
||||
## 'E! [inputs.aliyuncms] Discovery tool is not activated: no discovery support for project "acs_cdn"' )
|
||||
## Currently, discovery supported for the following projects:
|
||||
## - acs_ecs_dashboard
|
||||
## - acs_rds_dashboard
|
||||
## - acs_slb_dashboard
|
||||
## - acs_vpc_eip
|
||||
regions = ["cn-hongkong"]
|
||||
|
||||
# The minimum period for AliyunCMS metrics is 1 minute (60s). However not all
|
||||
# metrics are made available to the 1 minute period. Some are collected at
|
||||
# 3 minute, 5 minute, or larger intervals.
|
||||
# See: https://help.aliyun.com/document_detail/51936.html?spm=a2c4g.11186623.2.18.2bc1750eeOw1Pv
|
||||
# Note that if a period is configured that is smaller than the minimum for a
|
||||
# particular metric, that metric will not be returned by the Aliyun OpenAPI
|
||||
# and will not be collected by Telegraf.
|
||||
#
|
||||
## Requested AliyunCMS aggregation Period (required - must be a multiple of 60s)
|
||||
period = "5m"
|
||||
|
||||
## Collection Delay (required - must account for metrics availability via AliyunCMS API)
|
||||
delay = "1m"
|
||||
|
||||
## Recommended: use metric 'interval' that is a multiple of 'period' to avoid
|
||||
## gaps or overlap in pulled data
|
||||
interval = "5m"
|
||||
|
||||
## Metric Statistic Project (required)
|
||||
project = "acs_slb_dashboard"
|
||||
|
||||
## Maximum requests per second, default value is 200
|
||||
ratelimit = 200
|
||||
|
||||
## Discovery regions set the scope for object discovery, the discovered info can be used to enrich
|
||||
## the metrics with objects attributes/tags. Discovery is supported not for all projects (if not supported, then
|
||||
## it will be reported on the start - foo example for 'acs_cdn' project:
|
||||
## 'E! [inputs.aliyuncms] Discovery tool is not activated: no discovery support for project "acs_cdn"' )
|
||||
## Currently, discovery supported for the following projects:
|
||||
## - acs_ecs_dashboard
|
||||
## - acs_rds_dashboard
|
||||
## - acs_slb_dashboard
|
||||
## - acs_vpc_eip
|
||||
##
|
||||
## If not set, all regions would be covered, it can provide a significant load on API, so the recommendation here
|
||||
## is to limit the list as much as possible. Allowed values: https://www.alibabacloud.com/help/zh/doc-detail/40654.htm
|
||||
discovery_regions = ["cn-hongkong"]
|
||||
|
||||
## how often the discovery API call executed (default 1m)
|
||||
#discovery_interval = "1m"
|
||||
|
||||
## Metrics to Pull (Required)
|
||||
[[inputs.aliyuncms.metrics]]
|
||||
## Metrics names to be requested,
|
||||
## described here (per project): https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq
|
||||
names = ["InstanceActiveConnection", "InstanceNewConnection"]
|
||||
|
||||
## Dimension filters for Metric (these are optional).
|
||||
## This allows to get additional metric dimension. If dimension is not specified it can be returned or
|
||||
## the data can be aggregated - it depends on particular metric, you can find details here: https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq
|
||||
##
|
||||
## Note, that by default dimension filter includes the list of discovered objects in scope (if discovery is enabled)
|
||||
## Values specified here would be added into the list of discovered objects.
|
||||
## You can specify either single dimension:
|
||||
#dimensions = '{"instanceId": "p-example"}'
|
||||
|
||||
## Or you can specify several dimensions at once:
|
||||
#dimensions = '[{"instanceId": "p-example"},{"instanceId": "q-example"}]'
|
||||
|
||||
## Enrichment tags, can be added from discovery (if supported)
|
||||
## Notation is <measurement_tag_name>:<JMES query path (https://jmespath.org/tutorial.html)>
|
||||
## To figure out which fields are available, consult the Describe<ObjectType> API per project.
|
||||
## For example, for SLB: https://api.aliyun.com/#/?product=Slb&version=2014-05-15&api=DescribeLoadBalancers¶ms={}&tab=MOCK&lang=GO
|
||||
#tag_query_path = [
|
||||
# "address:Address",
|
||||
# "name:LoadBalancerName",
|
||||
# "cluster_owner:Tags.Tag[?TagKey=='cs.cluster.name'].TagValue | [0]"
|
||||
# ]
|
||||
## The following tags added by default: regionId (if discovery enabled), userId, instanceId.
|
||||
|
||||
## Allow metrics without discovery data, if discovery is enabled. If set to true, then metric without discovery
|
||||
## data would be emitted, otherwise dropped. This cane be of help, in case debugging dimension filters, or partial coverage
|
||||
## of discovery scope vs monitoring scope
|
||||
#allow_dps_without_discovery = false
|
||||
# The minimum period for AliyunCMS metrics is 1 minute (60s). However not all
|
||||
# metrics are made available to the 1 minute period. Some are collected at
|
||||
# 3 minute, 5 minute, or larger intervals.
|
||||
# See: https://help.aliyun.com/document_detail/51936.html?spm=a2c4g.11186623.2.18.2bc1750eeOw1Pv
|
||||
# Note that if a period is configured that is smaller than the minimum for a
|
||||
# particular metric, that metric will not be returned by the Aliyun OpenAPI
|
||||
# and will not be collected by Telegraf.
|
||||
#
|
||||
## Requested AliyunCMS aggregation Period (required - must be a multiple of 60s)
|
||||
period = "5m"
|
||||
|
||||
## Collection Delay (required - must account for metrics availability via AliyunCMS API)
|
||||
delay = "1m"
|
||||
|
||||
## Recommended: use metric 'interval' that is a multiple of 'period' to avoid
|
||||
## gaps or overlap in pulled data
|
||||
interval = "5m"
|
||||
|
||||
## Metric Statistic Project (required)
|
||||
project = "acs_slb_dashboard"
|
||||
|
||||
## Maximum requests per second, default value is 200
|
||||
ratelimit = 200
|
||||
|
||||
## How often the discovery API call executed (default 1m)
|
||||
#discovery_interval = "1m"
|
||||
|
||||
## Metrics to Pull (Required)
|
||||
[[inputs.aliyuncms.metrics]]
|
||||
## Metrics names to be requested,
|
||||
## described here (per project): https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq
|
||||
names = ["InstanceActiveConnection", "InstanceNewConnection"]
|
||||
|
||||
## Dimension filters for Metric (these are optional).
|
||||
## This allows to get additional metric dimension. If dimension is not specified it can be returned or
|
||||
## the data can be aggregated - it depends on particular metric, you can find details here: https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq
|
||||
##
|
||||
## Note, that by default dimension filter includes the list of discovered objects in scope (if discovery is enabled)
|
||||
## Values specified here would be added into the list of discovered objects.
|
||||
## You can specify either single dimension:
|
||||
#dimensions = '{"instanceId": "p-example"}'
|
||||
|
||||
## Or you can specify several dimensions at once:
|
||||
#dimensions = '[{"instanceId": "p-example"},{"instanceId": "q-example"}]'
|
||||
|
||||
## Enrichment tags, can be added from discovery (if supported)
|
||||
## Notation is <measurement_tag_name>:<JMES query path (https://jmespath.org/tutorial.html)>
|
||||
## To figure out which fields are available, consult the Describe<ObjectType> API per project.
|
||||
## For example, for SLB: https://api.aliyun.com/#/?product=Slb&version=2014-05-15&api=DescribeLoadBalancers¶ms={}&tab=MOCK&lang=GO
|
||||
#tag_query_path = [
|
||||
# "address:Address",
|
||||
# "name:LoadBalancerName",
|
||||
# "cluster_owner:Tags.Tag[?TagKey=='cs.cluster.name'].TagValue | [0]"
|
||||
# ]
|
||||
## The following tags added by default: regionId (if discovery enabled), userId, instanceId.
|
||||
|
||||
## Allow metrics without discovery data, if discovery is enabled. If set to true, then metric without discovery
|
||||
## data would be emitted, otherwise dropped. This cane be of help, in case debugging dimension filters, or partial coverage
|
||||
## of discovery scope vs monitoring scope
|
||||
#allow_dps_without_discovery = false
|
||||
`
|
||||
)
|
||||
|
||||
|
|
@ -132,7 +136,7 @@ type (
|
|||
PublicKeyID string `toml:"public_key_id"`
|
||||
RoleName string `toml:"role_name"`
|
||||
|
||||
DiscoveryRegions []string `toml:"discovery_regions"`
|
||||
Regions []string `toml:"regions"`
|
||||
DiscoveryInterval config.Duration `toml:"discovery_interval"`
|
||||
Period config.Duration `toml:"period"`
|
||||
Delay config.Duration `toml:"delay"`
|
||||
|
|
@ -162,7 +166,7 @@ type (
|
|||
dtLock sync.Mutex //Guard for discoveryTags & dimensions
|
||||
discoveryTags map[string]map[string]string //Internal data structure that can enrich metrics with tags
|
||||
dimensionsUdObj map[string]string
|
||||
dimensionsUdArr []map[string]string //Parsed Dimensions JSON string (unmarshalled)
|
||||
dimensionsUdArr []map[string]string //Parsed Dimesnsions JSON string (unmarshalled)
|
||||
requestDimensions []map[string]string //this is the actual dimensions list that would be used in API request
|
||||
requestDimensionsStr string //String representation of the above
|
||||
|
||||
|
|
@ -178,6 +182,31 @@ type (
|
|||
}
|
||||
)
|
||||
|
||||
// https://www.alibabacloud.com/help/doc-detail/40654.htm?gclid=Cj0KCQjw4dr0BRCxARIsAKUNjWTAMfyVUn_Y3OevFBV3CMaazrhq0URHsgE7c0m0SeMQRKlhlsJGgIEaAviyEALw_wcB
|
||||
var aliyunRegionList = []string{
|
||||
"cn-qingdao",
|
||||
"cn-beijing",
|
||||
"cn-zhangjiakou",
|
||||
"cn-huhehaote",
|
||||
"cn-hangzhou",
|
||||
"cn-shanghai",
|
||||
"cn-shenzhen",
|
||||
"cn-heyuan",
|
||||
"cn-chengdu",
|
||||
"cn-hongkong",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ap-southeast-3",
|
||||
"ap-southeast-5",
|
||||
"ap-south-1",
|
||||
"ap-northeast-1",
|
||||
"us-west-1",
|
||||
"us-east-1",
|
||||
"eu-central-1",
|
||||
"eu-west-1",
|
||||
"me-east-1",
|
||||
}
|
||||
|
||||
// SampleConfig implements telegraf.Inputs interface
|
||||
func (s *AliyunCMS) SampleConfig() string {
|
||||
return sampleConfig
|
||||
|
|
@ -188,6 +217,7 @@ func (s *AliyunCMS) Description() string {
|
|||
return description
|
||||
}
|
||||
|
||||
// Init perform checks of plugin inputs and initialize internals
|
||||
func (s *AliyunCMS) Init() error {
|
||||
if s.Project == "" {
|
||||
return errors.New("project is not set")
|
||||
|
|
@ -238,9 +268,16 @@ func (s *AliyunCMS) Init() error {
|
|||
|
||||
s.measurement = formatMeasurement(s.Project)
|
||||
|
||||
//Check regions
|
||||
if len(s.Regions) == 0 {
|
||||
s.Regions = aliyunRegionList
|
||||
s.Log.Infof("'regions' is not set. Metrics will be queried across %d regions:\n%s",
|
||||
len(s.Regions), strings.Join(s.Regions, ","))
|
||||
}
|
||||
|
||||
//Init discovery...
|
||||
if s.dt == nil { //Support for tests
|
||||
s.dt, err = newDiscoveryTool(s.DiscoveryRegions, s.Project, s.Log, credential, int(float32(s.RateLimit)*0.2), time.Duration(s.DiscoveryInterval))
|
||||
s.dt, err = newDiscoveryTool(s.Regions, s.Project, s.Log, credential, int(float32(s.RateLimit)*0.2), time.Duration(s.DiscoveryInterval))
|
||||
if err != nil {
|
||||
s.Log.Errorf("Discovery tool is not activated: %v", err)
|
||||
s.dt = nil
|
||||
|
|
@ -248,7 +285,7 @@ func (s *AliyunCMS) Init() error {
|
|||
}
|
||||
}
|
||||
|
||||
s.discoveryData, err = s.dt.getDiscoveryDataAllRegions(nil)
|
||||
s.discoveryData, err = s.dt.getDiscoveryDataAcrossRegions(nil)
|
||||
if err != nil {
|
||||
s.Log.Errorf("Discovery tool is not activated: %v", err)
|
||||
s.dt = nil
|
||||
|
|
@ -265,10 +302,11 @@ func (s *AliyunCMS) Init() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Start plugin discovery loop, metrics are gathered through Gather
|
||||
func (s *AliyunCMS) Start(telegraf.Accumulator) error {
|
||||
//Start periodic discovery process
|
||||
if s.dt != nil {
|
||||
s.dt.Start()
|
||||
s.dt.start()
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -300,9 +338,10 @@ func (s *AliyunCMS) Gather(acc telegraf.Accumulator) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Stop - stops the plugin discovery loop
|
||||
func (s *AliyunCMS) Stop() {
|
||||
if s.dt != nil {
|
||||
s.dt.Stop()
|
||||
s.dt.stop()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -327,78 +366,85 @@ func (s *AliyunCMS) updateWindow(relativeTo time.Time) {
|
|||
|
||||
// Gather given metric and emit error
|
||||
func (s *AliyunCMS) gatherMetric(acc telegraf.Accumulator, metricName string, metric *Metric) error {
|
||||
req := cms.CreateDescribeMetricListRequest()
|
||||
req.Period = strconv.FormatInt(int64(time.Duration(s.Period).Seconds()), 10)
|
||||
req.MetricName = metricName
|
||||
req.Length = "10000"
|
||||
req.Namespace = s.Project
|
||||
req.EndTime = strconv.FormatInt(s.windowEnd.Unix()*1000, 10)
|
||||
req.StartTime = strconv.FormatInt(s.windowStart.Unix()*1000, 10)
|
||||
req.Dimensions = metric.requestDimensionsStr
|
||||
for _, region := range s.Regions {
|
||||
req := cms.CreateDescribeMetricListRequest()
|
||||
req.Period = strconv.FormatInt(int64(time.Duration(s.Period).Seconds()), 10)
|
||||
req.MetricName = metricName
|
||||
req.Length = "10000"
|
||||
req.Namespace = s.Project
|
||||
req.EndTime = strconv.FormatInt(s.windowEnd.Unix()*1000, 10)
|
||||
req.StartTime = strconv.FormatInt(s.windowStart.Unix()*1000, 10)
|
||||
req.Dimensions = metric.requestDimensionsStr
|
||||
req.RegionId = region
|
||||
|
||||
for more := true; more; {
|
||||
resp, err := s.client.DescribeMetricList(req)
|
||||
if err != nil {
|
||||
return errors.Errorf("failed to query metricName list: %v", err)
|
||||
} else if resp.Code != "200" {
|
||||
s.Log.Errorf("failed to query metricName list: %v", resp.Message)
|
||||
break
|
||||
}
|
||||
|
||||
var datapoints []map[string]interface{}
|
||||
if err = json.Unmarshal([]byte(resp.Datapoints), &datapoints); err != nil {
|
||||
return errors.Errorf("failed to decode response datapoints: %v", err)
|
||||
}
|
||||
|
||||
if len(datapoints) == 0 {
|
||||
s.Log.Debugf("No metrics returned from CMS, response msg: %s", resp.Message)
|
||||
break
|
||||
}
|
||||
|
||||
NextDataPoint:
|
||||
for _, datapoint := range datapoints {
|
||||
fields := map[string]interface{}{}
|
||||
datapointTime := int64(0)
|
||||
tags := map[string]string{}
|
||||
for key, value := range datapoint {
|
||||
switch key {
|
||||
case "instanceId", "BucketName":
|
||||
tags[key] = value.(string)
|
||||
if metric.discoveryTags != nil { //discovery can be not activated
|
||||
//Skipping data point if discovery data not exist
|
||||
if _, ok := metric.discoveryTags[value.(string)]; !ok &&
|
||||
!metric.AllowDataPointWODiscoveryData {
|
||||
s.Log.Warnf("Instance %q is not found in discovery, skipping monitoring datapoint...", value.(string))
|
||||
continue NextDataPoint
|
||||
}
|
||||
|
||||
for k, v := range metric.discoveryTags[value.(string)] {
|
||||
tags[k] = v
|
||||
}
|
||||
}
|
||||
case "userId":
|
||||
tags[key] = value.(string)
|
||||
case "timestamp":
|
||||
datapointTime = int64(value.(float64)) / 1000
|
||||
default:
|
||||
fields[formatField(metricName, key)] = value
|
||||
}
|
||||
for more := true; more; {
|
||||
resp, err := s.client.DescribeMetricList(req)
|
||||
if err != nil {
|
||||
return errors.Errorf("failed to query metricName list: %v", err)
|
||||
}
|
||||
//Log.logW("Datapoint time: %s, now: %s", time.Unix(datapointTime, 0).Format(time.RFC3339), time.Now().Format(time.RFC3339))
|
||||
acc.AddFields(s.measurement, fields, tags, time.Unix(datapointTime, 0))
|
||||
if resp.Code != "200" {
|
||||
s.Log.Errorf("failed to query metricName list: %v", resp.Message)
|
||||
break
|
||||
}
|
||||
|
||||
var datapoints []map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(resp.Datapoints), &datapoints); err != nil {
|
||||
return errors.Errorf("failed to decode response datapoints: %v", err)
|
||||
}
|
||||
|
||||
if len(datapoints) == 0 {
|
||||
s.Log.Debugf("No metrics returned from CMS, response msg: %s", resp.Message)
|
||||
break
|
||||
}
|
||||
|
||||
NextDataPoint:
|
||||
for _, datapoint := range datapoints {
|
||||
fields := map[string]interface{}{}
|
||||
datapointTime := int64(0)
|
||||
tags := map[string]string{}
|
||||
for key, value := range datapoint {
|
||||
switch key {
|
||||
case "instanceId", "BucketName":
|
||||
tags[key] = value.(string)
|
||||
if metric.discoveryTags != nil { //discovery can be not activated
|
||||
//Skipping data point if discovery data not exist
|
||||
_, ok := metric.discoveryTags[value.(string)]
|
||||
if !ok &&
|
||||
!metric.AllowDataPointWODiscoveryData {
|
||||
s.Log.Warnf("Instance %q is not found in discovery, skipping monitoring datapoint...", value.(string))
|
||||
continue NextDataPoint
|
||||
}
|
||||
|
||||
for k, v := range metric.discoveryTags[value.(string)] {
|
||||
tags[k] = v
|
||||
}
|
||||
}
|
||||
case "userId":
|
||||
tags[key] = value.(string)
|
||||
case "timestamp":
|
||||
datapointTime = int64(value.(float64)) / 1000
|
||||
default:
|
||||
fields[formatField(metricName, key)] = value
|
||||
}
|
||||
}
|
||||
//Log.logW("Datapoint time: %s, now: %s", time.Unix(datapointTime, 0).Format(time.RFC3339), time.Now().Format(time.RFC3339))
|
||||
acc.AddFields(s.measurement, fields, tags, time.Unix(datapointTime, 0))
|
||||
}
|
||||
|
||||
req.NextToken = resp.NextToken
|
||||
more = req.NextToken != ""
|
||||
}
|
||||
|
||||
req.NextToken = resp.NextToken
|
||||
more = req.NextToken != ""
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//Tag helper
|
||||
//tag helper
|
||||
func parseTag(tagSpec string, data interface{}) (tagKey string, tagValue string, err error) {
|
||||
var (
|
||||
ok bool
|
||||
queryPath = tagSpec
|
||||
)
|
||||
tagKey = tagSpec
|
||||
queryPath := tagSpec
|
||||
|
||||
//Split query path to tagKey and query path
|
||||
if splitted := strings.Split(tagSpec, ":"); len(splitted) == 2 {
|
||||
|
|
@ -416,7 +462,7 @@ func parseTag(tagSpec string, data interface{}) (tagKey string, tagValue string,
|
|||
return "", "", nil
|
||||
}
|
||||
|
||||
tagValue, ok := tagRawValue.(string)
|
||||
tagValue, ok = tagRawValue.(string)
|
||||
if !ok {
|
||||
return "", "", errors.Errorf("Tag value %v parsed by query %q is not a string value",
|
||||
tagRawValue, queryPath)
|
||||
|
|
|
|||
|
|
@ -123,14 +123,13 @@ func TestPluginInitialize(t *testing.T) {
|
|||
var err error
|
||||
|
||||
plugin := new(AliyunCMS)
|
||||
plugin.DiscoveryRegions = []string{"cn-shanghai"}
|
||||
plugin.dt, err = getDiscoveryTool("acs_slb_dashboard", plugin.DiscoveryRegions)
|
||||
plugin.Log = testutil.Logger{Name: inputTitle}
|
||||
plugin.Regions = []string{"cn-shanghai"}
|
||||
plugin.dt, err = getDiscoveryTool("acs_slb_dashboard", plugin.Regions)
|
||||
if err != nil {
|
||||
t.Fatalf("Can't create discovery tool object: %v", err)
|
||||
}
|
||||
|
||||
plugin.Log = testutil.Logger{Name: inputTitle}
|
||||
|
||||
httpResp := &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(bytes.NewBufferString(
|
||||
|
|
@ -150,7 +149,7 @@ func TestPluginInitialize(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("Can't create mock sdk cli: %v", err)
|
||||
}
|
||||
plugin.dt.cli = map[string]aliyunSdkClient{plugin.DiscoveryRegions[0]: &mockCli}
|
||||
plugin.dt.cli = map[string]aliyunSdkClient{plugin.Regions[0]: &mockCli}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
|
|
@ -158,14 +157,24 @@ func TestPluginInitialize(t *testing.T) {
|
|||
accessKeyID string
|
||||
accessKeySecret string
|
||||
expectedErrorString string
|
||||
regions []string
|
||||
discoveryRegions []string
|
||||
}{
|
||||
{
|
||||
name: "Empty project",
|
||||
expectedErrorString: "project is not set",
|
||||
regions: []string{"cn-shanghai"},
|
||||
},
|
||||
{
|
||||
name: "Valid project",
|
||||
project: "acs_slb_dashboard",
|
||||
regions: []string{"cn-shanghai"},
|
||||
accessKeyID: "dummy",
|
||||
accessKeySecret: "dummy",
|
||||
},
|
||||
{
|
||||
name: "'regions' is not set",
|
||||
project: "acs_slb_dashboard",
|
||||
accessKeyID: "dummy",
|
||||
accessKeySecret: "dummy",
|
||||
},
|
||||
|
|
@ -176,12 +185,16 @@ func TestPluginInitialize(t *testing.T) {
|
|||
plugin.Project = tt.project
|
||||
plugin.AccessKeyID = tt.accessKeyID
|
||||
plugin.AccessKeySecret = tt.accessKeySecret
|
||||
plugin.Regions = tt.regions
|
||||
|
||||
if tt.expectedErrorString != "" {
|
||||
require.EqualError(t, plugin.Init(), tt.expectedErrorString)
|
||||
} else {
|
||||
require.Equal(t, nil, plugin.Init())
|
||||
}
|
||||
if len(tt.regions) == 0 { //Check if set to default
|
||||
require.Equal(t, plugin.Regions, aliyunRegionList)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -224,6 +237,7 @@ func TestGatherMetric(t *testing.T) {
|
|||
client: new(mockGatherAliyunCMSClient),
|
||||
measurement: formatMeasurement("acs_slb_dashboard"),
|
||||
Log: testutil.Logger{Name: inputTitle},
|
||||
Regions: []string{"cn-shanghai"},
|
||||
}
|
||||
|
||||
metric := &Metric{
|
||||
|
|
@ -262,15 +276,15 @@ func TestGather(t *testing.T) {
|
|||
Dimensions: `{"instanceId": "i-abcdefgh123456"}`,
|
||||
}
|
||||
plugin := &AliyunCMS{
|
||||
AccessKeyID: "my_access_key_id",
|
||||
AccessKeySecret: "my_access_key_secret",
|
||||
Project: "acs_slb_dashboard",
|
||||
Metrics: []*Metric{metric},
|
||||
RateLimit: 200,
|
||||
measurement: formatMeasurement("acs_slb_dashboard"),
|
||||
DiscoveryRegions: []string{"cn-shanghai"},
|
||||
client: new(mockGatherAliyunCMSClient),
|
||||
Log: testutil.Logger{Name: inputTitle},
|
||||
AccessKeyID: "my_access_key_id",
|
||||
AccessKeySecret: "my_access_key_secret",
|
||||
Project: "acs_slb_dashboard",
|
||||
Metrics: []*Metric{metric},
|
||||
RateLimit: 200,
|
||||
measurement: formatMeasurement("acs_slb_dashboard"),
|
||||
Regions: []string{"cn-shanghai"},
|
||||
client: new(mockGatherAliyunCMSClient),
|
||||
Log: testutil.Logger{Name: inputTitle},
|
||||
}
|
||||
|
||||
//test table:
|
||||
|
|
@ -326,7 +340,7 @@ func TestGather(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestGetDiscoveryDataAllRegions(t *testing.T) {
|
||||
func TestGetDiscoveryDataAcrossRegions(t *testing.T) {
|
||||
//test table:
|
||||
tests := []struct {
|
||||
name string
|
||||
|
|
@ -391,7 +405,7 @@ func TestGetDiscoveryDataAllRegions(t *testing.T) {
|
|||
t.Fatalf("Can't create mock sdk cli: %v", err)
|
||||
}
|
||||
dt.cli = map[string]aliyunSdkClient{tt.region: &mockCli}
|
||||
data, err := dt.getDiscoveryDataAllRegions(nil)
|
||||
data, err := dt.getDiscoveryDataAcrossRegions(nil)
|
||||
|
||||
require.Equal(t, tt.discData, data)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
|
@ -16,37 +17,11 @@ import (
|
|||
"github.com/aliyun/alibaba-cloud-sdk-go/services/rds"
|
||||
"github.com/aliyun/alibaba-cloud-sdk-go/services/slb"
|
||||
"github.com/aliyun/alibaba-cloud-sdk-go/services/vpc"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/internal/limiter"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// https://www.alibabacloud.com/help/doc-detail/40654.htm?gclid=Cj0KCQjw4dr0BRCxARIsAKUNjWTAMfyVUn_Y3OevFBV3CMaazrhq0URHsgE7c0m0SeMQRKlhlsJGgIEaAviyEALw_wcB
|
||||
var aliyunRegionList = []string{
|
||||
"cn-qingdao",
|
||||
"cn-beijing",
|
||||
"cn-zhangjiakou",
|
||||
"cn-huhehaote",
|
||||
"cn-hangzhou",
|
||||
"cn-shanghai",
|
||||
"cn-shenzhen",
|
||||
"cn-heyuan",
|
||||
"cn-chengdu",
|
||||
"cn-hongkong",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ap-southeast-3",
|
||||
"ap-southeast-5",
|
||||
"ap-south-1",
|
||||
"ap-northeast-1",
|
||||
"us-west-1",
|
||||
"us-east-1",
|
||||
"eu-central-1",
|
||||
"eu-west-1",
|
||||
"me-east-1",
|
||||
}
|
||||
|
||||
type discoveryRequest interface {
|
||||
}
|
||||
|
||||
|
|
@ -54,6 +29,7 @@ type aliyunSdkClient interface {
|
|||
ProcessCommonRequest(req *requests.CommonRequest) (response *responses.CommonResponse, err error)
|
||||
}
|
||||
|
||||
// discoveryTool is a object that provides discovery feature
|
||||
type discoveryTool struct {
|
||||
req map[string]discoveryRequest //Discovery request (specific per object type)
|
||||
rateLimit int //Rate limit for API query, as it is limited by API backend
|
||||
|
|
@ -70,8 +46,8 @@ type discoveryTool struct {
|
|||
lg telegraf.Logger //Telegraf logger (should be provided)
|
||||
}
|
||||
|
||||
type response struct {
|
||||
discData []interface{}
|
||||
type parsedDResp struct {
|
||||
data []interface{}
|
||||
totalCount int
|
||||
pageSize int
|
||||
pageNumber int
|
||||
|
|
@ -124,7 +100,8 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred
|
|||
|
||||
if len(regions) == 0 {
|
||||
regions = aliyunRegionList
|
||||
lg.Warnf("Discovery regions are not provided! Data will be queried across %d regions!", len(aliyunRegionList))
|
||||
lg.Infof("'regions' is not provided! Discovery data will be queried across %d regions:\n%s",
|
||||
len(aliyunRegionList), strings.Join(aliyunRegionList, ","))
|
||||
}
|
||||
|
||||
if rateLimit == 0 { //Can be a rounding case
|
||||
|
|
@ -300,21 +277,21 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred
|
|||
}, nil
|
||||
}
|
||||
|
||||
func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse) (discoveryResponse *response, err error) {
|
||||
func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse) (*parsedDResp, error) {
|
||||
var (
|
||||
fullOutput = map[string]interface{}{}
|
||||
foundDataItem, foundRootKey bool
|
||||
discData []interface{}
|
||||
totalCount, pageSize, pageNumber int
|
||||
fullOutput = map[string]interface{}{}
|
||||
data []byte
|
||||
foundDataItem bool
|
||||
foundRootKey bool
|
||||
pdResp = &parsedDResp{}
|
||||
)
|
||||
|
||||
data := resp.GetHttpContentBytes()
|
||||
data = resp.GetHttpContentBytes()
|
||||
if data == nil { //No data
|
||||
return nil, errors.Errorf("No data in response to be parsed")
|
||||
}
|
||||
|
||||
err = json.Unmarshal(data, &fullOutput)
|
||||
if err != nil {
|
||||
if err := json.Unmarshal(data, &fullOutput); err != nil {
|
||||
return nil, errors.Errorf("Can't parse JSON from discovery response: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -329,7 +306,7 @@ func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse)
|
|||
|
||||
//It should contain the array with discovered data
|
||||
for _, item := range rootKeyVal {
|
||||
if discData, foundDataItem = item.([]interface{}); foundDataItem {
|
||||
if pdResp.data, foundDataItem = item.([]interface{}); foundDataItem {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
|
@ -337,70 +314,72 @@ func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse)
|
|||
return nil, errors.Errorf("Didn't find array item in root key %q", key)
|
||||
}
|
||||
case "TotalCount":
|
||||
totalCount = int(val.(float64))
|
||||
pdResp.totalCount = int(val.(float64))
|
||||
case "PageSize":
|
||||
pageSize = int(val.(float64))
|
||||
pdResp.pageSize = int(val.(float64))
|
||||
case "PageNumber":
|
||||
pageNumber = int(val.(float64))
|
||||
pdResp.pageNumber = int(val.(float64))
|
||||
}
|
||||
}
|
||||
if !foundRootKey {
|
||||
return nil, errors.Errorf("Didn't find root key %q in discovery response", dt.respRootKey)
|
||||
}
|
||||
|
||||
return &response{
|
||||
discData: discData,
|
||||
totalCount: totalCount,
|
||||
pageSize: pageSize,
|
||||
pageNumber: pageNumber,
|
||||
}, nil
|
||||
return pdResp, nil
|
||||
}
|
||||
|
||||
func (dt *discoveryTool) getDiscoveryData(cli aliyunSdkClient, req *requests.CommonRequest, limiterChan chan bool) (map[string]interface{}, error) {
|
||||
var discoveryData []interface{}
|
||||
|
||||
func (dt *discoveryTool) getDiscoveryData(cli aliyunSdkClient, req *requests.CommonRequest, lmtr chan bool) (map[string]interface{}, error) {
|
||||
var (
|
||||
err error
|
||||
resp *responses.CommonResponse
|
||||
pDResp *parsedDResp
|
||||
discoveryData []interface{}
|
||||
totalCount int
|
||||
pageNumber int
|
||||
)
|
||||
defer delete(req.QueryParams, "PageNumber")
|
||||
|
||||
for {
|
||||
if limiterChan != nil {
|
||||
<-limiterChan //Rate limiting
|
||||
if lmtr != nil {
|
||||
<-lmtr //Rate limiting
|
||||
}
|
||||
|
||||
resp, err := cli.ProcessCommonRequest(req)
|
||||
resp, err = cli.ProcessCommonRequest(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
discoveryResponse, err := dt.parseDiscoveryResponse(resp)
|
||||
pDResp, err = dt.parseDiscoveryResponse(resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
discoveryData = append(discoveryData, discoveryResponse.discData...)
|
||||
discoveryData = append(discoveryData, pDResp.data...)
|
||||
pageNumber = pDResp.pageNumber
|
||||
totalCount = pDResp.totalCount
|
||||
|
||||
//Pagination
|
||||
discoveryResponse.pageNumber++
|
||||
req.QueryParams["PageNumber"] = strconv.Itoa(discoveryResponse.pageNumber)
|
||||
pageNumber++
|
||||
req.QueryParams["PageNumber"] = strconv.Itoa(pageNumber)
|
||||
|
||||
if len(discoveryData) == discoveryResponse.totalCount { //All data received
|
||||
if len(discoveryData) == totalCount { //All data received
|
||||
//Map data to appropriate shape before return
|
||||
preparedData := map[string]interface{}{}
|
||||
|
||||
for _, raw := range discoveryData {
|
||||
elem, ok := raw.(map[string]interface{})
|
||||
if !ok {
|
||||
return nil, errors.Errorf("Can't parse input data element, not a map[string]interface{} type")
|
||||
return nil, errors.Errorf("can't parse input data element, not a map[string]interface{} type")
|
||||
}
|
||||
if objectID, ok := elem[dt.respObjectIDKey].(string); ok {
|
||||
preparedData[objectID] = elem
|
||||
}
|
||||
}
|
||||
|
||||
return preparedData, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dt *discoveryTool) getDiscoveryDataAllRegions(limiterChan chan bool) (map[string]interface{}, error) {
|
||||
func (dt *discoveryTool) getDiscoveryDataAcrossRegions(lmtr chan bool) (map[string]interface{}, error) {
|
||||
var (
|
||||
data map[string]interface{}
|
||||
resultData = map[string]interface{}{}
|
||||
|
|
@ -431,7 +410,7 @@ func (dt *discoveryTool) getDiscoveryDataAllRegions(limiterChan chan bool) (map[
|
|||
commonRequest.TransToAcsRequest()
|
||||
|
||||
//Get discovery data using common request
|
||||
data, err = dt.getDiscoveryData(cli, commonRequest, limiterChan)
|
||||
data, err = dt.getDiscoveryData(cli, commonRequest, lmtr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -443,7 +422,9 @@ func (dt *discoveryTool) getDiscoveryDataAllRegions(limiterChan chan bool) (map[
|
|||
return resultData, nil
|
||||
}
|
||||
|
||||
func (dt *discoveryTool) Start() {
|
||||
// start the discovery pooling
|
||||
// In case smth. new found it will be reported back through `DataChan`
|
||||
func (dt *discoveryTool) start() {
|
||||
var (
|
||||
err error
|
||||
data map[string]interface{}
|
||||
|
|
@ -468,7 +449,7 @@ func (dt *discoveryTool) Start() {
|
|||
case <-dt.done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
data, err = dt.getDiscoveryDataAllRegions(lmtr.C)
|
||||
data, err = dt.getDiscoveryDataAcrossRegions(lmtr.C)
|
||||
if err != nil {
|
||||
dt.lg.Errorf("Can't get discovery data: %v", err)
|
||||
continue
|
||||
|
|
@ -489,7 +470,9 @@ func (dt *discoveryTool) Start() {
|
|||
}()
|
||||
}
|
||||
|
||||
func (dt *discoveryTool) Stop() {
|
||||
// stop the discovery loop, making sure
|
||||
// all data is read from 'dataChan'
|
||||
func (dt *discoveryTool) stop() {
|
||||
close(dt.done)
|
||||
|
||||
//Shutdown timer
|
||||
|
|
|
|||
Loading…
Reference in New Issue