CL-Softwares
/
telegraf
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: ensure aliyuncms metrics accept array, fix discovery (#10850)

This commit is contained in:
Joshua Powers 2022-04-12 15:52:03 -06:00 committed by GitHub
parent 50d706ecfa
commit 03847d8cac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 124 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -19,7 +19,7 @@ require (
github.com/Shopify/sarama v1.32.0
github.com/aerospike/aerospike-client-go/v5 v5.7.0
github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9
github.com/antchfx/jsonquery v1.1.5
github.com/antchfx/xmlquery v1.3.9

4
go.sum
View File

@ -266,8 +266,8 @@ github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:C
github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE=
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I=
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU=
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529 h1:qAt5MZ3Ukwx/JMAiaagQhNQMBZLcmJbnweBoK3EeHxI=
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU=
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ=
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=

21
plugins/inputs/aliyuncms/aliyuncms.go
View File

@ -144,10 +144,16 @@ func (s *AliyunCMS) Init() error {
if metric.Dimensions != "" {
metric.dimensionsUdObj = map[string]string{}
metric.dimensionsUdArr = []map[string]string{}
// first try to unmarshal as an object
err := json.Unmarshal([]byte(metric.Dimensions), &metric.dimensionsUdObj)
if err != nil {
// then try to unmarshal as an array
err := json.Unmarshal([]byte(metric.Dimensions), &metric.dimensionsUdArr)
return errors.Errorf("Can't parse dimensions (it is neither obj, nor array) %q :%v", metric.Dimensions, err)
if err != nil {
return errors.Errorf("cannot parse dimensions (neither obj, nor array) %q :%v", metric.Dimensions, err)
}
}
}
}
@ -430,14 +436,15 @@ L:
metric.discoveryTags[instanceID][tagKey] = tagValue
}
//Preparing dimensions (first adding dimensions that comes from discovery data)
metric.requestDimensions = append(
metric.requestDimensions,
map[string]string{s.dimensionKey: instanceID})
//if no dimension configured in config file, use discovery data
if len(metric.dimensionsUdArr) == 0 && len(metric.dimensionsUdObj) == 0 {
metric.requestDimensions = append(
metric.requestDimensions,
map[string]string{s.dimensionKey: instanceID})
}
}
//Get final dimension (need to get full lis of
//what was provided in config + what comes from discovery
//add dimensions filter from config file
if len(metric.dimensionsUdArr) != 0 {
metric.requestDimensions = append(metric.requestDimensions, metric.dimensionsUdArr...)
}

101
plugins/inputs/aliyuncms/aliyuncms_test.go
View File

@ -199,6 +199,107 @@ func TestPluginInitialize(t *testing.T) {
}
}
func TestPluginMetricsInitialize(t *testing.T) {
var err error
plugin := new(AliyunCMS)
plugin.Log = testutil.Logger{Name: inputTitle}
plugin.Regions = []string{"cn-shanghai"}
plugin.dt, err = getDiscoveryTool("acs_slb_dashboard", plugin.Regions)
if err != nil {
t.Fatalf("Can't create discovery tool object: %v", err)
}
httpResp := &http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(
`{
"LoadBalancers":
{
"LoadBalancer": [
{"LoadBalancerId":"bla"}
]
},
"TotalCount": 1,
"PageSize": 1,
"PageNumber": 1
}`)),
}
mockCli, err := getMockSdkCli(httpResp)
if err != nil {
t.Fatalf("Can't create mock sdk cli: %v", err)
}
plugin.dt.cli = map[string]aliyunSdkClient{plugin.Regions[0]: &mockCli}
tests := []struct {
name string
project string
accessKeyID string
accessKeySecret string
expectedErrorString string
regions []string
discoveryRegions []string
metrics []*Metric
}{
{
name: "Valid project",
project: "acs_slb_dashboard",
regions: []string{"cn-shanghai"},
accessKeyID: "dummy",
accessKeySecret: "dummy",
metrics: []*Metric{
{
MetricNames: []string{},
Dimensions: `{"instanceId": "i-abcdefgh123456"}`,
},
},
},
{
name: "Valid project",
project: "acs_slb_dashboard",
regions: []string{"cn-shanghai"},
accessKeyID: "dummy",
accessKeySecret: "dummy",
metrics: []*Metric{
{
MetricNames: []string{},
Dimensions: `[{"instanceId": "p-example"},{"instanceId": "q-example"}]`,
},
},
},
{
name: "Valid project",
project: "acs_slb_dashboard",
regions: []string{"cn-shanghai"},
accessKeyID: "dummy",
accessKeySecret: "dummy",
expectedErrorString: `cannot parse dimensions (neither obj, nor array) "[" :unexpected end of JSON input`,
metrics: []*Metric{
{
MetricNames: []string{},
Dimensions: `[`,
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
plugin.Project = tt.project
plugin.AccessKeyID = tt.accessKeyID
plugin.AccessKeySecret = tt.accessKeySecret
plugin.Regions = tt.regions
plugin.Metrics = tt.metrics
if tt.expectedErrorString != "" {
require.EqualError(t, plugin.Init(), tt.expectedErrorString)
} else {
require.Equal(t, nil, plugin.Init())
}
})
}
}
func TestUpdateWindow(t *testing.T) {
duration, _ := time.ParseDuration("1m")
internalDuration := config.Duration(duration)

39
plugins/inputs/aliyuncms/discovery.go
View File

@ -3,7 +3,6 @@ package aliyuncms
import (
"encoding/json"
"reflect"
"regexp"
"strconv"
"strings"
"sync"
@ -91,7 +90,6 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred
var (
dscReq = map[string]discoveryRequest{}
cli = map[string]aliyunSdkClient{}
parseRootKey = regexp.MustCompile(`Describe(.*)`)
responseRootKey string
responseObjectIDKey string
err error
@ -112,12 +110,15 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred
switch project {
case "acs_ecs_dashboard":
dscReq[region] = ecs.CreateDescribeInstancesRequest()
responseRootKey = "Instances"
responseObjectIDKey = "InstanceId"
case "acs_rds_dashboard":
dscReq[region] = rds.CreateDescribeDBInstancesRequest()
responseRootKey = "Items"
responseObjectIDKey = "DBInstanceId"
case "acs_slb_dashboard":
dscReq[region] = slb.CreateDescribeLoadBalancersRequest()
responseRootKey = "LoadBalancers"
responseObjectIDKey = "LoadBalancerId"
case "acs_memcache":
return nil, noDiscoverySupportErr
@ -137,6 +138,7 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred
//req.InitWithApiInfo("oss", "2014-08-15", "DescribeDBInstances", "oss", "openAPI")
case "acs_vpc_eip":
dscReq[region] = vpc.CreateDescribeEipAddressesRequest()
responseRootKey = "EipAddresses"
responseObjectIDKey = "AllocationId"
case "acs_kvstore":
return nil, noDiscoverySupportErr
@ -235,35 +237,6 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred
return nil, errors.Errorf("Can't build discovery request for project: %q,\nregions: %v", project, regions)
}
//Getting response root key (if not set already). This is to be able to parse discovery responses
//As they differ per object type
//Discovery requests are of the same type per every region, so pick the first one
rpcReq, err := getRPCReqFromDiscoveryRequest(dscReq[regions[0]])
//This means that the discovery request is not of proper type/kind
if err != nil {
return nil, errors.Errorf("Can't parse rpc request object from discovery request %v", dscReq[regions[0]])
}
/*
The action name is of the following format Describe<Project related title for managed instances>,
For example: DescribeLoadBalancers -> for SLB project, or DescribeInstances for ECS project
We will use it to construct root key name in the discovery API response.
It follows the following logic: for 'DescribeLoadBalancers' action in discovery request we get the response
in json of the following structure:
{
...
"LoadBalancers": {
"LoadBalancer": [ here comes objects, one per every instance]
}
}
As we can see, the root key is a part of action name, except first word (part) 'Describe'
*/
result := parseRootKey.FindStringSubmatch(rpcReq.GetActionName())
if result == nil || len(result) != 2 {
return nil, errors.Errorf("Can't parse the discovery response root key from request action name %q", rpcReq.GetActionName())
}
responseRootKey = result[1]
return &discoveryTool{
req: dscReq,
cli: cli,
@ -313,9 +286,9 @@ func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse)
if !foundDataItem {
return nil, errors.Errorf("Didn't find array item in root key %q", key)
}
case "TotalCount":
case "TotalCount", "TotalRecordCount":
pdResp.totalCount = int(val.(float64))
case "PageSize":
case "PageSize", "PageRecordCount":
pdResp.pageSize = int(val.(float64))
case "PageNumber":
pdResp.pageNumber = int(val.(float64))