diff --git a/plugins/inputs/aliyuncms/README.md b/plugins/inputs/aliyuncms/README.md index 4304de593..c239baa63 100644 --- a/plugins/inputs/aliyuncms/README.md +++ b/plugins/inputs/aliyuncms/README.md @@ -36,7 +36,26 @@ In the following order the plugin will attempt to authenticate. # private_key = "" # public_key_id = "" # role_name = "" - + + ## Specify the ali cloud region list to be queried for metrics and objects discovery + ## If not set, all supported regions (see below) would be covered, it can provide a significant load on API, so the recommendation here + ## is to limit the list as much as possible. Allowed values: https://www.alibabacloud.com/help/zh/doc-detail/40654.htm + ## Default supported regions are: + ## 21 items: cn-qingdao,cn-beijing,cn-zhangjiakou,cn-huhehaote,cn-hangzhou,cn-shanghai,cn-shenzhen, + ## cn-heyuan,cn-chengdu,cn-hongkong,ap-southeast-1,ap-southeast-2,ap-southeast-3,ap-southeast-5, + ## ap-south-1,ap-northeast-1,us-west-1,us-east-1,eu-central-1,eu-west-1,me-east-1 + ## + ## From discovery perspective it set the scope for object discovery, the discovered info can be used to enrich + ## the metrics with objects attributes/tags. Discovery is supported not for all projects (if not supported, then + ## it will be reported on the start - for example for 'acs_cdn' project: + ## 'E! [inputs.aliyuncms] Discovery tool is not activated: no discovery support for project "acs_cdn"' ) + ## Currently, discovery supported for the following projects: + ## - acs_ecs_dashboard + ## - acs_rds_dashboard + ## - acs_slb_dashboard + ## - acs_vpc_eip + regions = ["cn-hongkong"] + # The minimum period for AliyunCMS metrics is 1 minute (60s). However not all # metrics are made available to the 1 minute period. Some are collected at # 3 minute, 5 minute, or larger intervals. @@ -61,21 +80,7 @@ In the following order the plugin will attempt to authenticate. ## Maximum requests per second, default value is 200 ratelimit = 200 - ## Discovery regions set the scope for object discovery, the discovered info can be used to enrich - ## the metrics with objects attributes/tags. Discovery is supported not for all projects (if not supported, then - ## it will be reported on the start - foo example for 'acs_cdn' project: - ## 'E! [inputs.aliyuncms] Discovery tool is not activated: no discovery support for project "acs_cdn"' ) - ## Currently, discovery supported for the following projects: - ## - acs_ecs_dashboard - ## - acs_rds_dashboard - ## - acs_slb_dashboard - ## - acs_vpc_eip - ## - ## If not set, all regions would be covered, it can provide a significant load on API, so the recommendation here - ## is to limit the list as much as possible. Allowed values: https://www.alibabacloud.com/help/zh/doc-detail/40654.htm - discovery_regions = ["cn-hongkong"] - - ## how often the discovery API call executed (default 1m) + ## How often the discovery API call executed (default 1m) #discovery_interval = "1m" ## Metrics to Pull (Required) diff --git a/plugins/inputs/aliyuncms/aliyuncms.go b/plugins/inputs/aliyuncms/aliyuncms.go index ac70b9a44..1dc20d718 100644 --- a/plugins/inputs/aliyuncms/aliyuncms.go +++ b/plugins/inputs/aliyuncms/aliyuncms.go @@ -11,112 +11,116 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/sdk" "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials/providers" "github.com/aliyun/alibaba-cloud-sdk-go/services/cms" - "github.com/jmespath/go-jmespath" - "github.com/pkg/errors" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/internal/limiter" "github.com/influxdata/telegraf/plugins/inputs" + "github.com/jmespath/go-jmespath" + "github.com/pkg/errors" ) const ( description = "Pull Metric Statistics from Aliyun CMS" sampleConfig = ` - ## Aliyun Credentials - ## Credentials are loaded in the following order - ## 1) Ram RoleArn credential - ## 2) AccessKey STS token credential - ## 3) AccessKey credential - ## 4) Ecs Ram Role credential - ## 5) RSA keypair credential - ## 6) Environment variables credential - ## 7) Instance metadata credential + ## Aliyun Credentials + ## Credentials are loaded in the following order + ## 1) Ram RoleArn credential + ## 2) AccessKey STS token credential + ## 3) AccessKey credential + ## 4) Ecs Ram Role credential + ## 5) RSA keypair credential + ## 6) Environment variables credential + ## 7) Instance metadata credential + + # access_key_id = "" + # access_key_secret = "" + # access_key_sts_token = "" + # role_arn = "" + # role_session_name = "" + # private_key = "" + # public_key_id = "" + # role_name = "" - # access_key_id = "" - # access_key_secret = "" - # access_key_sts_token = "" - # role_arn = "" - # role_session_name = "" - # private_key = "" - # public_key_id = "" - # role_name = "" + ## Specify the ali cloud region list to be queried for metrics and objects discovery + ## If not set, all supported regions (see below) would be covered, it can provide a significant load on API, so the recommendation here + ## is to limit the list as much as possible. Allowed values: https://www.alibabacloud.com/help/zh/doc-detail/40654.htm + ## Default supported regions are: + ## 21 items: cn-qingdao,cn-beijing,cn-zhangjiakou,cn-huhehaote,cn-hangzhou,cn-shanghai,cn-shenzhen, + ## cn-heyuan,cn-chengdu,cn-hongkong,ap-southeast-1,ap-southeast-2,ap-southeast-3,ap-southeast-5, + ## ap-south-1,ap-northeast-1,us-west-1,us-east-1,eu-central-1,eu-west-1,me-east-1 + ## + ## From discovery perspective it set the scope for object discovery, the discovered info can be used to enrich + ## the metrics with objects attributes/tags. Discovery is supported not for all projects (if not supported, then + ## it will be reported on the start - for example for 'acs_cdn' project: + ## 'E! [inputs.aliyuncms] Discovery tool is not activated: no discovery support for project "acs_cdn"' ) + ## Currently, discovery supported for the following projects: + ## - acs_ecs_dashboard + ## - acs_rds_dashboard + ## - acs_slb_dashboard + ## - acs_vpc_eip + regions = ["cn-hongkong"] - # The minimum period for AliyunCMS metrics is 1 minute (60s). However not all - # metrics are made available to the 1 minute period. Some are collected at - # 3 minute, 5 minute, or larger intervals. - # See: https://help.aliyun.com/document_detail/51936.html?spm=a2c4g.11186623.2.18.2bc1750eeOw1Pv - # Note that if a period is configured that is smaller than the minimum for a - # particular metric, that metric will not be returned by the Aliyun OpenAPI - # and will not be collected by Telegraf. - # - ## Requested AliyunCMS aggregation Period (required - must be a multiple of 60s) - period = "5m" - - ## Collection Delay (required - must account for metrics availability via AliyunCMS API) - delay = "1m" - - ## Recommended: use metric 'interval' that is a multiple of 'period' to avoid - ## gaps or overlap in pulled data - interval = "5m" - - ## Metric Statistic Project (required) - project = "acs_slb_dashboard" - - ## Maximum requests per second, default value is 200 - ratelimit = 200 - - ## Discovery regions set the scope for object discovery, the discovered info can be used to enrich - ## the metrics with objects attributes/tags. Discovery is supported not for all projects (if not supported, then - ## it will be reported on the start - foo example for 'acs_cdn' project: - ## 'E! [inputs.aliyuncms] Discovery tool is not activated: no discovery support for project "acs_cdn"' ) - ## Currently, discovery supported for the following projects: - ## - acs_ecs_dashboard - ## - acs_rds_dashboard - ## - acs_slb_dashboard - ## - acs_vpc_eip - ## - ## If not set, all regions would be covered, it can provide a significant load on API, so the recommendation here - ## is to limit the list as much as possible. Allowed values: https://www.alibabacloud.com/help/zh/doc-detail/40654.htm - discovery_regions = ["cn-hongkong"] - - ## how often the discovery API call executed (default 1m) - #discovery_interval = "1m" - - ## Metrics to Pull (Required) - [[inputs.aliyuncms.metrics]] - ## Metrics names to be requested, - ## described here (per project): https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq - names = ["InstanceActiveConnection", "InstanceNewConnection"] - - ## Dimension filters for Metric (these are optional). - ## This allows to get additional metric dimension. If dimension is not specified it can be returned or - ## the data can be aggregated - it depends on particular metric, you can find details here: https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq - ## - ## Note, that by default dimension filter includes the list of discovered objects in scope (if discovery is enabled) - ## Values specified here would be added into the list of discovered objects. - ## You can specify either single dimension: - #dimensions = '{"instanceId": "p-example"}' - - ## Or you can specify several dimensions at once: - #dimensions = '[{"instanceId": "p-example"},{"instanceId": "q-example"}]' - - ## Enrichment tags, can be added from discovery (if supported) - ## Notation is : - ## To figure out which fields are available, consult the Describe API per project. - ## For example, for SLB: https://api.aliyun.com/#/?product=Slb&version=2014-05-15&api=DescribeLoadBalancers¶ms={}&tab=MOCK&lang=GO - #tag_query_path = [ - # "address:Address", - # "name:LoadBalancerName", - # "cluster_owner:Tags.Tag[?TagKey=='cs.cluster.name'].TagValue | [0]" - # ] - ## The following tags added by default: regionId (if discovery enabled), userId, instanceId. - - ## Allow metrics without discovery data, if discovery is enabled. If set to true, then metric without discovery - ## data would be emitted, otherwise dropped. This cane be of help, in case debugging dimension filters, or partial coverage - ## of discovery scope vs monitoring scope - #allow_dps_without_discovery = false + # The minimum period for AliyunCMS metrics is 1 minute (60s). However not all + # metrics are made available to the 1 minute period. Some are collected at + # 3 minute, 5 minute, or larger intervals. + # See: https://help.aliyun.com/document_detail/51936.html?spm=a2c4g.11186623.2.18.2bc1750eeOw1Pv + # Note that if a period is configured that is smaller than the minimum for a + # particular metric, that metric will not be returned by the Aliyun OpenAPI + # and will not be collected by Telegraf. + # + ## Requested AliyunCMS aggregation Period (required - must be a multiple of 60s) + period = "5m" + + ## Collection Delay (required - must account for metrics availability via AliyunCMS API) + delay = "1m" + + ## Recommended: use metric 'interval' that is a multiple of 'period' to avoid + ## gaps or overlap in pulled data + interval = "5m" + + ## Metric Statistic Project (required) + project = "acs_slb_dashboard" + + ## Maximum requests per second, default value is 200 + ratelimit = 200 + + ## How often the discovery API call executed (default 1m) + #discovery_interval = "1m" + + ## Metrics to Pull (Required) + [[inputs.aliyuncms.metrics]] + ## Metrics names to be requested, + ## described here (per project): https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq + names = ["InstanceActiveConnection", "InstanceNewConnection"] + + ## Dimension filters for Metric (these are optional). + ## This allows to get additional metric dimension. If dimension is not specified it can be returned or + ## the data can be aggregated - it depends on particular metric, you can find details here: https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq + ## + ## Note, that by default dimension filter includes the list of discovered objects in scope (if discovery is enabled) + ## Values specified here would be added into the list of discovered objects. + ## You can specify either single dimension: + #dimensions = '{"instanceId": "p-example"}' + + ## Or you can specify several dimensions at once: + #dimensions = '[{"instanceId": "p-example"},{"instanceId": "q-example"}]' + + ## Enrichment tags, can be added from discovery (if supported) + ## Notation is : + ## To figure out which fields are available, consult the Describe API per project. + ## For example, for SLB: https://api.aliyun.com/#/?product=Slb&version=2014-05-15&api=DescribeLoadBalancers¶ms={}&tab=MOCK&lang=GO + #tag_query_path = [ + # "address:Address", + # "name:LoadBalancerName", + # "cluster_owner:Tags.Tag[?TagKey=='cs.cluster.name'].TagValue | [0]" + # ] + ## The following tags added by default: regionId (if discovery enabled), userId, instanceId. + + ## Allow metrics without discovery data, if discovery is enabled. If set to true, then metric without discovery + ## data would be emitted, otherwise dropped. This cane be of help, in case debugging dimension filters, or partial coverage + ## of discovery scope vs monitoring scope + #allow_dps_without_discovery = false ` ) @@ -132,7 +136,7 @@ type ( PublicKeyID string `toml:"public_key_id"` RoleName string `toml:"role_name"` - DiscoveryRegions []string `toml:"discovery_regions"` + Regions []string `toml:"regions"` DiscoveryInterval config.Duration `toml:"discovery_interval"` Period config.Duration `toml:"period"` Delay config.Duration `toml:"delay"` @@ -162,7 +166,7 @@ type ( dtLock sync.Mutex //Guard for discoveryTags & dimensions discoveryTags map[string]map[string]string //Internal data structure that can enrich metrics with tags dimensionsUdObj map[string]string - dimensionsUdArr []map[string]string //Parsed Dimensions JSON string (unmarshalled) + dimensionsUdArr []map[string]string //Parsed Dimesnsions JSON string (unmarshalled) requestDimensions []map[string]string //this is the actual dimensions list that would be used in API request requestDimensionsStr string //String representation of the above @@ -178,6 +182,31 @@ type ( } ) +// https://www.alibabacloud.com/help/doc-detail/40654.htm?gclid=Cj0KCQjw4dr0BRCxARIsAKUNjWTAMfyVUn_Y3OevFBV3CMaazrhq0URHsgE7c0m0SeMQRKlhlsJGgIEaAviyEALw_wcB +var aliyunRegionList = []string{ + "cn-qingdao", + "cn-beijing", + "cn-zhangjiakou", + "cn-huhehaote", + "cn-hangzhou", + "cn-shanghai", + "cn-shenzhen", + "cn-heyuan", + "cn-chengdu", + "cn-hongkong", + "ap-southeast-1", + "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-5", + "ap-south-1", + "ap-northeast-1", + "us-west-1", + "us-east-1", + "eu-central-1", + "eu-west-1", + "me-east-1", +} + // SampleConfig implements telegraf.Inputs interface func (s *AliyunCMS) SampleConfig() string { return sampleConfig @@ -188,6 +217,7 @@ func (s *AliyunCMS) Description() string { return description } +// Init perform checks of plugin inputs and initialize internals func (s *AliyunCMS) Init() error { if s.Project == "" { return errors.New("project is not set") @@ -238,9 +268,16 @@ func (s *AliyunCMS) Init() error { s.measurement = formatMeasurement(s.Project) + //Check regions + if len(s.Regions) == 0 { + s.Regions = aliyunRegionList + s.Log.Infof("'regions' is not set. Metrics will be queried across %d regions:\n%s", + len(s.Regions), strings.Join(s.Regions, ",")) + } + //Init discovery... if s.dt == nil { //Support for tests - s.dt, err = newDiscoveryTool(s.DiscoveryRegions, s.Project, s.Log, credential, int(float32(s.RateLimit)*0.2), time.Duration(s.DiscoveryInterval)) + s.dt, err = newDiscoveryTool(s.Regions, s.Project, s.Log, credential, int(float32(s.RateLimit)*0.2), time.Duration(s.DiscoveryInterval)) if err != nil { s.Log.Errorf("Discovery tool is not activated: %v", err) s.dt = nil @@ -248,7 +285,7 @@ func (s *AliyunCMS) Init() error { } } - s.discoveryData, err = s.dt.getDiscoveryDataAllRegions(nil) + s.discoveryData, err = s.dt.getDiscoveryDataAcrossRegions(nil) if err != nil { s.Log.Errorf("Discovery tool is not activated: %v", err) s.dt = nil @@ -265,10 +302,11 @@ func (s *AliyunCMS) Init() error { return nil } +// Start plugin discovery loop, metrics are gathered through Gather func (s *AliyunCMS) Start(telegraf.Accumulator) error { //Start periodic discovery process if s.dt != nil { - s.dt.Start() + s.dt.start() } return nil @@ -300,9 +338,10 @@ func (s *AliyunCMS) Gather(acc telegraf.Accumulator) error { return nil } +// Stop - stops the plugin discovery loop func (s *AliyunCMS) Stop() { if s.dt != nil { - s.dt.Stop() + s.dt.stop() } } @@ -327,78 +366,85 @@ func (s *AliyunCMS) updateWindow(relativeTo time.Time) { // Gather given metric and emit error func (s *AliyunCMS) gatherMetric(acc telegraf.Accumulator, metricName string, metric *Metric) error { - req := cms.CreateDescribeMetricListRequest() - req.Period = strconv.FormatInt(int64(time.Duration(s.Period).Seconds()), 10) - req.MetricName = metricName - req.Length = "10000" - req.Namespace = s.Project - req.EndTime = strconv.FormatInt(s.windowEnd.Unix()*1000, 10) - req.StartTime = strconv.FormatInt(s.windowStart.Unix()*1000, 10) - req.Dimensions = metric.requestDimensionsStr + for _, region := range s.Regions { + req := cms.CreateDescribeMetricListRequest() + req.Period = strconv.FormatInt(int64(time.Duration(s.Period).Seconds()), 10) + req.MetricName = metricName + req.Length = "10000" + req.Namespace = s.Project + req.EndTime = strconv.FormatInt(s.windowEnd.Unix()*1000, 10) + req.StartTime = strconv.FormatInt(s.windowStart.Unix()*1000, 10) + req.Dimensions = metric.requestDimensionsStr + req.RegionId = region - for more := true; more; { - resp, err := s.client.DescribeMetricList(req) - if err != nil { - return errors.Errorf("failed to query metricName list: %v", err) - } else if resp.Code != "200" { - s.Log.Errorf("failed to query metricName list: %v", resp.Message) - break - } - - var datapoints []map[string]interface{} - if err = json.Unmarshal([]byte(resp.Datapoints), &datapoints); err != nil { - return errors.Errorf("failed to decode response datapoints: %v", err) - } - - if len(datapoints) == 0 { - s.Log.Debugf("No metrics returned from CMS, response msg: %s", resp.Message) - break - } - - NextDataPoint: - for _, datapoint := range datapoints { - fields := map[string]interface{}{} - datapointTime := int64(0) - tags := map[string]string{} - for key, value := range datapoint { - switch key { - case "instanceId", "BucketName": - tags[key] = value.(string) - if metric.discoveryTags != nil { //discovery can be not activated - //Skipping data point if discovery data not exist - if _, ok := metric.discoveryTags[value.(string)]; !ok && - !metric.AllowDataPointWODiscoveryData { - s.Log.Warnf("Instance %q is not found in discovery, skipping monitoring datapoint...", value.(string)) - continue NextDataPoint - } - - for k, v := range metric.discoveryTags[value.(string)] { - tags[k] = v - } - } - case "userId": - tags[key] = value.(string) - case "timestamp": - datapointTime = int64(value.(float64)) / 1000 - default: - fields[formatField(metricName, key)] = value - } + for more := true; more; { + resp, err := s.client.DescribeMetricList(req) + if err != nil { + return errors.Errorf("failed to query metricName list: %v", err) } - //Log.logW("Datapoint time: %s, now: %s", time.Unix(datapointTime, 0).Format(time.RFC3339), time.Now().Format(time.RFC3339)) - acc.AddFields(s.measurement, fields, tags, time.Unix(datapointTime, 0)) + if resp.Code != "200" { + s.Log.Errorf("failed to query metricName list: %v", resp.Message) + break + } + + var datapoints []map[string]interface{} + if err := json.Unmarshal([]byte(resp.Datapoints), &datapoints); err != nil { + return errors.Errorf("failed to decode response datapoints: %v", err) + } + + if len(datapoints) == 0 { + s.Log.Debugf("No metrics returned from CMS, response msg: %s", resp.Message) + break + } + + NextDataPoint: + for _, datapoint := range datapoints { + fields := map[string]interface{}{} + datapointTime := int64(0) + tags := map[string]string{} + for key, value := range datapoint { + switch key { + case "instanceId", "BucketName": + tags[key] = value.(string) + if metric.discoveryTags != nil { //discovery can be not activated + //Skipping data point if discovery data not exist + _, ok := metric.discoveryTags[value.(string)] + if !ok && + !metric.AllowDataPointWODiscoveryData { + s.Log.Warnf("Instance %q is not found in discovery, skipping monitoring datapoint...", value.(string)) + continue NextDataPoint + } + + for k, v := range metric.discoveryTags[value.(string)] { + tags[k] = v + } + } + case "userId": + tags[key] = value.(string) + case "timestamp": + datapointTime = int64(value.(float64)) / 1000 + default: + fields[formatField(metricName, key)] = value + } + } + //Log.logW("Datapoint time: %s, now: %s", time.Unix(datapointTime, 0).Format(time.RFC3339), time.Now().Format(time.RFC3339)) + acc.AddFields(s.measurement, fields, tags, time.Unix(datapointTime, 0)) + } + + req.NextToken = resp.NextToken + more = req.NextToken != "" } - - req.NextToken = resp.NextToken - more = req.NextToken != "" } - return nil } -//Tag helper +//tag helper func parseTag(tagSpec string, data interface{}) (tagKey string, tagValue string, err error) { + var ( + ok bool + queryPath = tagSpec + ) tagKey = tagSpec - queryPath := tagSpec //Split query path to tagKey and query path if splitted := strings.Split(tagSpec, ":"); len(splitted) == 2 { @@ -416,7 +462,7 @@ func parseTag(tagSpec string, data interface{}) (tagKey string, tagValue string, return "", "", nil } - tagValue, ok := tagRawValue.(string) + tagValue, ok = tagRawValue.(string) if !ok { return "", "", errors.Errorf("Tag value %v parsed by query %q is not a string value", tagRawValue, queryPath) diff --git a/plugins/inputs/aliyuncms/aliyuncms_test.go b/plugins/inputs/aliyuncms/aliyuncms_test.go index a2bae5d0d..22e0acbc5 100644 --- a/plugins/inputs/aliyuncms/aliyuncms_test.go +++ b/plugins/inputs/aliyuncms/aliyuncms_test.go @@ -123,14 +123,13 @@ func TestPluginInitialize(t *testing.T) { var err error plugin := new(AliyunCMS) - plugin.DiscoveryRegions = []string{"cn-shanghai"} - plugin.dt, err = getDiscoveryTool("acs_slb_dashboard", plugin.DiscoveryRegions) + plugin.Log = testutil.Logger{Name: inputTitle} + plugin.Regions = []string{"cn-shanghai"} + plugin.dt, err = getDiscoveryTool("acs_slb_dashboard", plugin.Regions) if err != nil { t.Fatalf("Can't create discovery tool object: %v", err) } - plugin.Log = testutil.Logger{Name: inputTitle} - httpResp := &http.Response{ StatusCode: 200, Body: ioutil.NopCloser(bytes.NewBufferString( @@ -150,7 +149,7 @@ func TestPluginInitialize(t *testing.T) { if err != nil { t.Fatalf("Can't create mock sdk cli: %v", err) } - plugin.dt.cli = map[string]aliyunSdkClient{plugin.DiscoveryRegions[0]: &mockCli} + plugin.dt.cli = map[string]aliyunSdkClient{plugin.Regions[0]: &mockCli} tests := []struct { name string @@ -158,14 +157,24 @@ func TestPluginInitialize(t *testing.T) { accessKeyID string accessKeySecret string expectedErrorString string + regions []string + discoveryRegions []string }{ { name: "Empty project", expectedErrorString: "project is not set", + regions: []string{"cn-shanghai"}, }, { name: "Valid project", project: "acs_slb_dashboard", + regions: []string{"cn-shanghai"}, + accessKeyID: "dummy", + accessKeySecret: "dummy", + }, + { + name: "'regions' is not set", + project: "acs_slb_dashboard", accessKeyID: "dummy", accessKeySecret: "dummy", }, @@ -176,12 +185,16 @@ func TestPluginInitialize(t *testing.T) { plugin.Project = tt.project plugin.AccessKeyID = tt.accessKeyID plugin.AccessKeySecret = tt.accessKeySecret + plugin.Regions = tt.regions if tt.expectedErrorString != "" { require.EqualError(t, plugin.Init(), tt.expectedErrorString) } else { require.Equal(t, nil, plugin.Init()) } + if len(tt.regions) == 0 { //Check if set to default + require.Equal(t, plugin.Regions, aliyunRegionList) + } }) } } @@ -224,6 +237,7 @@ func TestGatherMetric(t *testing.T) { client: new(mockGatherAliyunCMSClient), measurement: formatMeasurement("acs_slb_dashboard"), Log: testutil.Logger{Name: inputTitle}, + Regions: []string{"cn-shanghai"}, } metric := &Metric{ @@ -262,15 +276,15 @@ func TestGather(t *testing.T) { Dimensions: `{"instanceId": "i-abcdefgh123456"}`, } plugin := &AliyunCMS{ - AccessKeyID: "my_access_key_id", - AccessKeySecret: "my_access_key_secret", - Project: "acs_slb_dashboard", - Metrics: []*Metric{metric}, - RateLimit: 200, - measurement: formatMeasurement("acs_slb_dashboard"), - DiscoveryRegions: []string{"cn-shanghai"}, - client: new(mockGatherAliyunCMSClient), - Log: testutil.Logger{Name: inputTitle}, + AccessKeyID: "my_access_key_id", + AccessKeySecret: "my_access_key_secret", + Project: "acs_slb_dashboard", + Metrics: []*Metric{metric}, + RateLimit: 200, + measurement: formatMeasurement("acs_slb_dashboard"), + Regions: []string{"cn-shanghai"}, + client: new(mockGatherAliyunCMSClient), + Log: testutil.Logger{Name: inputTitle}, } //test table: @@ -326,7 +340,7 @@ func TestGather(t *testing.T) { } } -func TestGetDiscoveryDataAllRegions(t *testing.T) { +func TestGetDiscoveryDataAcrossRegions(t *testing.T) { //test table: tests := []struct { name string @@ -391,7 +405,7 @@ func TestGetDiscoveryDataAllRegions(t *testing.T) { t.Fatalf("Can't create mock sdk cli: %v", err) } dt.cli = map[string]aliyunSdkClient{tt.region: &mockCli} - data, err := dt.getDiscoveryDataAllRegions(nil) + data, err := dt.getDiscoveryDataAcrossRegions(nil) require.Equal(t, tt.discData, data) if err != nil { diff --git a/plugins/inputs/aliyuncms/discovery.go b/plugins/inputs/aliyuncms/discovery.go index c3f35c78a..a6fe5471b 100644 --- a/plugins/inputs/aliyuncms/discovery.go +++ b/plugins/inputs/aliyuncms/discovery.go @@ -5,6 +5,7 @@ import ( "reflect" "regexp" "strconv" + "strings" "sync" "time" @@ -16,37 +17,11 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/services/rds" "github.com/aliyun/alibaba-cloud-sdk-go/services/slb" "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" - "github.com/pkg/errors" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal/limiter" + "github.com/pkg/errors" ) -// https://www.alibabacloud.com/help/doc-detail/40654.htm?gclid=Cj0KCQjw4dr0BRCxARIsAKUNjWTAMfyVUn_Y3OevFBV3CMaazrhq0URHsgE7c0m0SeMQRKlhlsJGgIEaAviyEALw_wcB -var aliyunRegionList = []string{ - "cn-qingdao", - "cn-beijing", - "cn-zhangjiakou", - "cn-huhehaote", - "cn-hangzhou", - "cn-shanghai", - "cn-shenzhen", - "cn-heyuan", - "cn-chengdu", - "cn-hongkong", - "ap-southeast-1", - "ap-southeast-2", - "ap-southeast-3", - "ap-southeast-5", - "ap-south-1", - "ap-northeast-1", - "us-west-1", - "us-east-1", - "eu-central-1", - "eu-west-1", - "me-east-1", -} - type discoveryRequest interface { } @@ -54,6 +29,7 @@ type aliyunSdkClient interface { ProcessCommonRequest(req *requests.CommonRequest) (response *responses.CommonResponse, err error) } +// discoveryTool is a object that provides discovery feature type discoveryTool struct { req map[string]discoveryRequest //Discovery request (specific per object type) rateLimit int //Rate limit for API query, as it is limited by API backend @@ -70,8 +46,8 @@ type discoveryTool struct { lg telegraf.Logger //Telegraf logger (should be provided) } -type response struct { - discData []interface{} +type parsedDResp struct { + data []interface{} totalCount int pageSize int pageNumber int @@ -124,7 +100,8 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred if len(regions) == 0 { regions = aliyunRegionList - lg.Warnf("Discovery regions are not provided! Data will be queried across %d regions!", len(aliyunRegionList)) + lg.Infof("'regions' is not provided! Discovery data will be queried across %d regions:\n%s", + len(aliyunRegionList), strings.Join(aliyunRegionList, ",")) } if rateLimit == 0 { //Can be a rounding case @@ -300,21 +277,21 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred }, nil } -func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse) (discoveryResponse *response, err error) { +func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse) (*parsedDResp, error) { var ( - fullOutput = map[string]interface{}{} - foundDataItem, foundRootKey bool - discData []interface{} - totalCount, pageSize, pageNumber int + fullOutput = map[string]interface{}{} + data []byte + foundDataItem bool + foundRootKey bool + pdResp = &parsedDResp{} ) - data := resp.GetHttpContentBytes() + data = resp.GetHttpContentBytes() if data == nil { //No data return nil, errors.Errorf("No data in response to be parsed") } - err = json.Unmarshal(data, &fullOutput) - if err != nil { + if err := json.Unmarshal(data, &fullOutput); err != nil { return nil, errors.Errorf("Can't parse JSON from discovery response: %v", err) } @@ -329,7 +306,7 @@ func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse) //It should contain the array with discovered data for _, item := range rootKeyVal { - if discData, foundDataItem = item.([]interface{}); foundDataItem { + if pdResp.data, foundDataItem = item.([]interface{}); foundDataItem { break } } @@ -337,70 +314,72 @@ func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse) return nil, errors.Errorf("Didn't find array item in root key %q", key) } case "TotalCount": - totalCount = int(val.(float64)) + pdResp.totalCount = int(val.(float64)) case "PageSize": - pageSize = int(val.(float64)) + pdResp.pageSize = int(val.(float64)) case "PageNumber": - pageNumber = int(val.(float64)) + pdResp.pageNumber = int(val.(float64)) } } if !foundRootKey { return nil, errors.Errorf("Didn't find root key %q in discovery response", dt.respRootKey) } - return &response{ - discData: discData, - totalCount: totalCount, - pageSize: pageSize, - pageNumber: pageNumber, - }, nil + return pdResp, nil } -func (dt *discoveryTool) getDiscoveryData(cli aliyunSdkClient, req *requests.CommonRequest, limiterChan chan bool) (map[string]interface{}, error) { - var discoveryData []interface{} - +func (dt *discoveryTool) getDiscoveryData(cli aliyunSdkClient, req *requests.CommonRequest, lmtr chan bool) (map[string]interface{}, error) { + var ( + err error + resp *responses.CommonResponse + pDResp *parsedDResp + discoveryData []interface{} + totalCount int + pageNumber int + ) defer delete(req.QueryParams, "PageNumber") for { - if limiterChan != nil { - <-limiterChan //Rate limiting + if lmtr != nil { + <-lmtr //Rate limiting } - resp, err := cli.ProcessCommonRequest(req) + resp, err = cli.ProcessCommonRequest(req) if err != nil { return nil, err } - discoveryResponse, err := dt.parseDiscoveryResponse(resp) + pDResp, err = dt.parseDiscoveryResponse(resp) if err != nil { return nil, err } - discoveryData = append(discoveryData, discoveryResponse.discData...) + discoveryData = append(discoveryData, pDResp.data...) + pageNumber = pDResp.pageNumber + totalCount = pDResp.totalCount //Pagination - discoveryResponse.pageNumber++ - req.QueryParams["PageNumber"] = strconv.Itoa(discoveryResponse.pageNumber) + pageNumber++ + req.QueryParams["PageNumber"] = strconv.Itoa(pageNumber) - if len(discoveryData) == discoveryResponse.totalCount { //All data received + if len(discoveryData) == totalCount { //All data received //Map data to appropriate shape before return preparedData := map[string]interface{}{} for _, raw := range discoveryData { elem, ok := raw.(map[string]interface{}) if !ok { - return nil, errors.Errorf("Can't parse input data element, not a map[string]interface{} type") + return nil, errors.Errorf("can't parse input data element, not a map[string]interface{} type") } if objectID, ok := elem[dt.respObjectIDKey].(string); ok { preparedData[objectID] = elem } } - return preparedData, nil } } } -func (dt *discoveryTool) getDiscoveryDataAllRegions(limiterChan chan bool) (map[string]interface{}, error) { +func (dt *discoveryTool) getDiscoveryDataAcrossRegions(lmtr chan bool) (map[string]interface{}, error) { var ( data map[string]interface{} resultData = map[string]interface{}{} @@ -431,7 +410,7 @@ func (dt *discoveryTool) getDiscoveryDataAllRegions(limiterChan chan bool) (map[ commonRequest.TransToAcsRequest() //Get discovery data using common request - data, err = dt.getDiscoveryData(cli, commonRequest, limiterChan) + data, err = dt.getDiscoveryData(cli, commonRequest, lmtr) if err != nil { return nil, err } @@ -443,7 +422,9 @@ func (dt *discoveryTool) getDiscoveryDataAllRegions(limiterChan chan bool) (map[ return resultData, nil } -func (dt *discoveryTool) Start() { +// start the discovery pooling +// In case smth. new found it will be reported back through `DataChan` +func (dt *discoveryTool) start() { var ( err error data map[string]interface{} @@ -468,7 +449,7 @@ func (dt *discoveryTool) Start() { case <-dt.done: return case <-ticker.C: - data, err = dt.getDiscoveryDataAllRegions(lmtr.C) + data, err = dt.getDiscoveryDataAcrossRegions(lmtr.C) if err != nil { dt.lg.Errorf("Can't get discovery data: %v", err) continue @@ -489,7 +470,9 @@ func (dt *discoveryTool) Start() { }() } -func (dt *discoveryTool) Stop() { +// stop the discovery loop, making sure +// all data is read from 'dataChan' +func (dt *discoveryTool) stop() { close(dt.done) //Shutdown timer