From ba8452d61dcd8c8ce750c4db5f1fcf527d9a4864 Mon Sep 17 00:00:00 2001 From: helenosheaa <38860767+helenosheaa@users.noreply.github.com> Date: Thu, 11 Mar 2021 16:19:23 -0500 Subject: [PATCH] resolved conflicts --- docs/LICENSE_OF_DEPENDENCIES.md | 5 + go.mod | 3 + go.sum | 19 + plugins/inputs/aliyuncms/README.md | 142 ++++++ plugins/inputs/aliyuncms/aliyuncms.go | 566 +++++++++++++++++++++ plugins/inputs/aliyuncms/aliyuncms_test.go | 410 +++++++++++++++ plugins/inputs/aliyuncms/discovery.go | 511 +++++++++++++++++++ plugins/inputs/all/all.go | 1 + 8 files changed, 1657 insertions(+) create mode 100644 plugins/inputs/aliyuncms/README.md create mode 100644 plugins/inputs/aliyuncms/aliyuncms.go create mode 100644 plugins/inputs/aliyuncms/aliyuncms_test.go create mode 100644 plugins/inputs/aliyuncms/discovery.go diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index 0aff4fb29..6b811a5a9 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -20,6 +20,7 @@ following works: - github.com/StackExchange/wmi [MIT License](https://github.com/StackExchange/wmi/blob/master/LICENSE) - github.com/aerospike/aerospike-client-go [Apache License 2.0](https://github.com/aerospike/aerospike-client-go/blob/master/LICENSE) - github.com/alecthomas/units [MIT License](https://github.com/alecthomas/units/blob/master/COPYING) +- github.com/aliyun/alibaba-cloud-sdk-go [Apache License 2.0](https://github.com/aliyun/alibaba-cloud-sdk-go/blob/master/LICENSE) - github.com/amir/raidman [The Unlicense](https://github.com/amir/raidman/blob/master/UNLICENSE) - github.com/antchfx/xmlquery [MIT License](https://github.com/antchfx/xmlquery/blob/master/LICENSE) - github.com/antchfx/xpath [MIT License](https://github.com/antchfx/xpath/blob/master/LICENSE) @@ -105,6 +106,7 @@ following works: - github.com/jcmturner/gofork [BSD 3-Clause "New" or "Revised" License](https://github.com/jcmturner/gofork/blob/master/LICENSE) - github.com/jmespath/go-jmespath [Apache License 2.0](https://github.com/jmespath/go-jmespath/blob/master/LICENSE) - github.com/jpillora/backoff [MIT License](https://github.com/jpillora/backoff/blob/master/LICENSE) +- github.com/json-iterator/go [MIT License](https://github.com/json-iterator/go/blob/master/LICENSE) - github.com/kardianos/service [zlib License](https://github.com/kardianos/service/blob/master/LICENSE) - github.com/karrick/godirwalk [BSD 2-Clause "Simplified" License](https://github.com/karrick/godirwalk/blob/master/LICENSE) - github.com/kballard/go-shellquote [MIT License](https://github.com/kballard/go-shellquote/blob/master/LICENSE) @@ -121,6 +123,8 @@ following works: - github.com/miekg/dns [BSD 3-Clause Clear License](https://github.com/miekg/dns/blob/master/LICENSE) - github.com/mitchellh/go-homedir [MIT License](https://github.com/mitchellh/go-homedir/blob/master/LICENSE) - github.com/mitchellh/mapstructure [MIT License](https://github.com/mitchellh/mapstructure/blob/master/LICENSE) +- github.com/modern-go/concurrent [Apache License 2.0](https://github.com/modern-go/concurrent/blob/master/LICENSE) +- github.com/modern-go/reflect2 [Apache License 2.0](https://github.com/modern-go/reflect2/blob/master/LICENSE) - github.com/multiplay/go-ts3 [BSD 2-Clause "Simplified" License](https://github.com/multiplay/go-ts3/blob/master/LICENSE) - github.com/naoina/go-stringutil [MIT License](https://github.com/naoina/go-stringutil/blob/master/LICENSE) - github.com/nats-io/jwt [Apache License 2.0](https://github.com/nats-io/jwt/blob/master/LICENSE) @@ -193,6 +197,7 @@ following works: - gopkg.in/fsnotify.v1 [BSD 3-Clause "New" or "Revised" License](https://github.com/fsnotify/fsnotify/blob/v1.4.7/LICENSE) - gopkg.in/gorethink/gorethink.v3 [Apache License 2.0](https://github.com/rethinkdb/rethinkdb-go/blob/v3.0.5/LICENSE) - gopkg.in/inf.v0 [BSD 3-Clause "New" or "Revised" License](https://github.com/go-inf/inf/blob/v0.9.1/LICENSE) +- gopkg.in/ini.v1 [Apache License 2.0](https://github.com/go-ini/ini/blob/master/LICENSE) - gopkg.in/jcmturner/aescts.v1 [Apache License 2.0](https://github.com/jcmturner/aescts/blob/v1.0.1/LICENSE) - gopkg.in/jcmturner/dnsutils.v1 [Apache License 2.0](https://github.com/jcmturner/dnsutils/blob/v1.0.1/LICENSE) - gopkg.in/jcmturner/gokrb5.v7 [Apache License 2.0](https://github.com/jcmturner/gokrb5/tree/v7.5.0/LICENSE) diff --git a/go.mod b/go.mod index 705ed742c..ff29eb2ac 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/Shopify/sarama v1.27.2 github.com/aerospike/aerospike-client-go v1.27.0 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 + github.com/aliyun/alibaba-cloud-sdk-go v1.61.785 github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 github.com/antchfx/xmlquery v1.3.3 github.com/antchfx/xpath v1.1.11 @@ -88,6 +89,7 @@ require ( github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect github.com/jackc/pgx v3.6.0+incompatible github.com/james4k/rcon v0.0.0-20120923215419-8fbb8268b60a + github.com/jmespath/go-jmespath v0.4.0 github.com/kardianos/service v1.0.0 github.com/karrick/godirwalk v1.16.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 @@ -128,6 +130,7 @@ require ( github.com/streadway/amqp v0.0.0-20180528204448-e5adc2ada8b8 github.com/stretchr/testify v1.7.0 github.com/tbrandon/mbserver v0.0.0-20170611213546-993e1772cc62 + github.com/tedsuo/ifrit v0.0.0-20191009134036-9a97d0632f00 // indirect github.com/tidwall/gjson v1.6.0 github.com/tinylib/msgp v1.1.5 github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e // indirect diff --git a/go.sum b/go.sum index 6a079a055..7ec88378d 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,7 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+ cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0 h1:RPUcBvDeYgQFMfQu1eBMq6piD1SXmLH+vK3qjewZPus= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= code.cloudfoundry.org/clock v1.0.0 h1:kFXWQM4bxYvdBw2X8BbBeXwQNgfoWv1vqAk2ZZyBN2o= code.cloudfoundry.org/clock v1.0.0/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= @@ -98,6 +99,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.785 h1:3PVbcCSPY0f4timzlCQbDzL/7y/Z0d4YdEl23iAhSTE= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.785/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc= github.com/antchfx/xmlquery v1.3.3 h1:HYmadPG0uz8CySdL68rB4DCLKXz2PurCjS3mnkVF4CQ= @@ -286,6 +289,7 @@ github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5 github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec h1:lJwO/92dFXWeXOZdoGXgptLmNLwynMSHUmU6besqtiw= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= @@ -334,6 +338,7 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -415,6 +420,7 @@ github.com/james4k/rcon v0.0.0-20120923215419-8fbb8268b60a h1:JxcWget6X/VfBMKxPI github.com/james4k/rcon v0.0.0-20120923215419-8fbb8268b60a/go.mod h1:1qNVsDcmNQDsAXYfUuF/Z0rtK5eT8x9D6Pi7S3PjXAg= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -427,6 +433,7 @@ github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+ github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4 h1:nwOc1YaOrYJ37sEBrtWZrdqzK22hiJs3GpDmP3sR2Yw= github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= @@ -438,7 +445,9 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181012004132-a4583d0a56ea h1:g2k+8WR7cHch4g0tBDhfiEvAp7fXxTNBiD1oC1Oxj3E= github.com/juju/errors v0.0.0-20181012004132-a4583d0a56ea/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b h1:Rrp0ByJXEjhREMPGTt3aWYjoIsUGCbt21ekbeJcTWv0= github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= @@ -637,7 +646,11 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4-0.20190306220146-200a235640ff h1:JcVn27VGCEwd33jyNj+3IqEbOmzAX9f9LILt3SoGPHU= github.com/smartystreets/goconvey v1.6.4-0.20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= github.com/soniah/gosnmp v1.25.0 h1:0y8vpjD07NPmnT+wojnUrKkYLX9Fxw1jI4cGTumWugQ= @@ -660,6 +673,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tbrandon/mbserver v0.0.0-20170611213546-993e1772cc62 h1:Oj2e7Sae4XrOsk3ij21QjjEgAcVSeo9nkp0dI//cD2o= github.com/tbrandon/mbserver v0.0.0-20170611213546-993e1772cc62/go.mod h1:qUzPVlSj2UgxJkVbH0ZwuuiR46U8RBMDT5KLY78Ifpw= +github.com/tedsuo/ifrit v0.0.0-20191009134036-9a97d0632f00 h1:mujcChM89zOHwgZBBNr5WZ77mBXP1yR+gLThGCYZgAg= +github.com/tedsuo/ifrit v0.0.0-20191009134036-9a97d0632f00/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= @@ -873,6 +888,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -913,6 +929,7 @@ golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200205215550-e35592f146e4/go.mod h1:U gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= @@ -985,6 +1002,8 @@ gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= diff --git a/plugins/inputs/aliyuncms/README.md b/plugins/inputs/aliyuncms/README.md new file mode 100644 index 000000000..4304de593 --- /dev/null +++ b/plugins/inputs/aliyuncms/README.md @@ -0,0 +1,142 @@ +# Alibaba (aka Aliyun) CloudMonitor Service Statistics Input +Here and after we use `Aliyun` instead `Alibaba` as it is default naming across web console and docs. + +This plugin will pull Metric Statistics from Aliyun CMS. + +### Aliyun Authentication + +This plugin uses an [AccessKey](https://www.alibabacloud.com/help/doc-detail/53045.htm?spm=a2c63.p38356.b99.127.5cba21fdt5MJKr&parentId=28572) credential for Authentication with the Aliyun OpenAPI endpoint. +In the following order the plugin will attempt to authenticate. +1. Ram RoleARN credential if `access_key_id`, `access_key_secret`, `role_arn`, `role_session_name` is specified +2. AccessKey STS token credential if `access_key_id`, `access_key_secret`, `access_key_sts_token` is specified +3. AccessKey credential if `access_key_id`, `access_key_secret` is specified +4. Ecs Ram Role Credential if `role_name` is specified +5. RSA keypair credential if `private_key`, `public_key_id` is specified +6. Environment variables credential +7. Instance metadata credential + +### Configuration: + +```toml + ## Aliyun Credentials + ## Credentials are loaded in the following order + ## 1) Ram RoleArn credential + ## 2) AccessKey STS token credential + ## 3) AccessKey credential + ## 4) Ecs Ram Role credential + ## 5) RSA keypair credential + ## 6) Environment variables credential + ## 7) Instance metadata credential + + # access_key_id = "" + # access_key_secret = "" + # access_key_sts_token = "" + # role_arn = "" + # role_session_name = "" + # private_key = "" + # public_key_id = "" + # role_name = "" + + # The minimum period for AliyunCMS metrics is 1 minute (60s). However not all + # metrics are made available to the 1 minute period. Some are collected at + # 3 minute, 5 minute, or larger intervals. + # See: https://help.aliyun.com/document_detail/51936.html?spm=a2c4g.11186623.2.18.2bc1750eeOw1Pv + # Note that if a period is configured that is smaller than the minimum for a + # particular metric, that metric will not be returned by the Aliyun OpenAPI + # and will not be collected by Telegraf. + # + ## Requested AliyunCMS aggregation Period (required - must be a multiple of 60s) + period = "5m" + + ## Collection Delay (required - must account for metrics availability via AliyunCMS API) + delay = "1m" + + ## Recommended: use metric 'interval' that is a multiple of 'period' to avoid + ## gaps or overlap in pulled data + interval = "5m" + + ## Metric Statistic Project (required) + project = "acs_slb_dashboard" + + ## Maximum requests per second, default value is 200 + ratelimit = 200 + + ## Discovery regions set the scope for object discovery, the discovered info can be used to enrich + ## the metrics with objects attributes/tags. Discovery is supported not for all projects (if not supported, then + ## it will be reported on the start - foo example for 'acs_cdn' project: + ## 'E! [inputs.aliyuncms] Discovery tool is not activated: no discovery support for project "acs_cdn"' ) + ## Currently, discovery supported for the following projects: + ## - acs_ecs_dashboard + ## - acs_rds_dashboard + ## - acs_slb_dashboard + ## - acs_vpc_eip + ## + ## If not set, all regions would be covered, it can provide a significant load on API, so the recommendation here + ## is to limit the list as much as possible. Allowed values: https://www.alibabacloud.com/help/zh/doc-detail/40654.htm + discovery_regions = ["cn-hongkong"] + + ## how often the discovery API call executed (default 1m) + #discovery_interval = "1m" + + ## Metrics to Pull (Required) + [[inputs.aliyuncms.metrics]] + ## Metrics names to be requested, + ## described here (per project): https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq + names = ["InstanceActiveConnection", "InstanceNewConnection"] + + ## Dimension filters for Metric (these are optional). + ## This allows to get additional metric dimension. If dimension is not specified it can be returned or + ## the data can be aggregated - it depends on particular metric, you can find details here: https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq + ## + ## Note, that by default dimension filter includes the list of discovered objects in scope (if discovery is enabled) + ## Values specified here would be added into the list of discovered objects. + ## You can specify either single dimension: + #dimensions = '{"instanceId": "p-example"}' + + ## Or you can specify several dimensions at once: + #dimensions = '[{"instanceId": "p-example"},{"instanceId": "q-example"}]' + + ## Enrichment tags, can be added from discovery (if supported) + ## Notation is : + ## To figure out which fields are available, consult the Describe API per project. + ## For example, for SLB: https://api.aliyun.com/#/?product=Slb&version=2014-05-15&api=DescribeLoadBalancers¶ms={}&tab=MOCK&lang=GO + #tag_query_path = [ + # "address:Address", + # "name:LoadBalancerName", + # "cluster_owner:Tags.Tag[?TagKey=='cs.cluster.name'].TagValue | [0]" + # ] + ## The following tags added by default: regionId (if discovery enabled), userId, instanceId. + + ## Allow metrics without discovery data, if discovery is enabled. If set to true, then metric without discovery + ## data would be emitted, otherwise dropped. This cane be of help, in case debugging dimension filters, or partial coverage + ## of discovery scope vs monitoring scope + #allow_dps_without_discovery = false +``` + +#### Requirements and Terminology + +Plugin Configuration utilizes [preset metric items references](https://www.alibabacloud.com/help/doc-detail/28619.htm?spm=a2c63.p38356.a3.2.389f233d0kPJn0) + +- `discovery_region` must be a valid Aliyun [Region](https://www.alibabacloud.com/help/doc-detail/40654.htm) value +- `period` must be a valid duration value +- `project` must be a preset project value +- `names` must be preset metric names +- `dimensions` must be preset dimension values + +### Measurements & Fields: + +Each Aliyun CMS Project monitored records a measurement with fields for each available Metric Statistic +Project and Metrics are represented in [snake case](https://en.wikipedia.org/wiki/Snake_case) + +- aliyuncms_{project} + - {metric}_average (metric Average value) + - {metric}_minimum (metric Minimum value) + - {metric}_maximum (metric Maximum value) + - {metric}_value (metric Value value) + +### Example Output: + +``` +$ ./telegraf --config telegraf.conf --input-filter aliyuncms --test +> aliyuncms_acs_slb_dashboard,instanceId=p-example,regionId=cn-hangzhou,userId=1234567890 latency_average=0.004810798017284538,latency_maximum=0.1100282669067383,latency_minimum=0.0006084442138671875 +``` \ No newline at end of file diff --git a/plugins/inputs/aliyuncms/aliyuncms.go b/plugins/inputs/aliyuncms/aliyuncms.go new file mode 100644 index 000000000..794f398f7 --- /dev/null +++ b/plugins/inputs/aliyuncms/aliyuncms.go @@ -0,0 +1,566 @@ +package aliyuncms + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials/providers" + "github.com/aliyun/alibaba-cloud-sdk-go/services/cms" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/internal/limiter" + "github.com/influxdata/telegraf/plugins/inputs" + "github.com/jmespath/go-jmespath" + "github.com/pkg/errors" +) + +const ( + description = "Pull Metric Statistics from Aliyun CMS" + sampleConfig = ` + ## Aliyun Credentials + ## Credentials are loaded in the following order + ## 1) Ram RoleArn credential + ## 2) AccessKey STS token credential + ## 3) AccessKey credential + ## 4) Ecs Ram Role credential + ## 5) RSA keypair credential + ## 6) Environment variables credential + ## 7) Instance metadata credential + + # access_key_id = "" + # access_key_secret = "" + # access_key_sts_token = "" + # role_arn = "" + # role_session_name = "" + # private_key = "" + # public_key_id = "" + # role_name = "" + + # The minimum period for AliyunCMS metrics is 1 minute (60s). However not all + # metrics are made available to the 1 minute period. Some are collected at + # 3 minute, 5 minute, or larger intervals. + # See: https://help.aliyun.com/document_detail/51936.html?spm=a2c4g.11186623.2.18.2bc1750eeOw1Pv + # Note that if a period is configured that is smaller than the minimum for a + # particular metric, that metric will not be returned by the Aliyun OpenAPI + # and will not be collected by Telegraf. + # + ## Requested AliyunCMS aggregation Period (required - must be a multiple of 60s) + period = "5m" + + ## Collection Delay (required - must account for metrics availability via AliyunCMS API) + delay = "1m" + + ## Recommended: use metric 'interval' that is a multiple of 'period' to avoid + ## gaps or overlap in pulled data + interval = "5m" + + ## Metric Statistic Project (required) + project = "acs_slb_dashboard" + + ## Maximum requests per second, default value is 200 + ratelimit = 200 + + ## Discovery regions set the scope for object discovery, the discovered info can be used to enrich + ## the metrics with objects attributes/tags. Discovery is supported not for all projects (if not supported, then + ## it will be reported on the start - foo example for 'acs_cdn' project: + ## 'E! [inputs.aliyuncms] Discovery tool is not activated: no discovery support for project "acs_cdn"' ) + ## Currently, discovery supported for the following projects: + ## - acs_ecs_dashboard + ## - acs_rds_dashboard + ## - acs_slb_dashboard + ## - acs_vpc_eip + ## + ## If not set, all regions would be covered, it can provide a significant load on API, so the recommendation here + ## is to limit the list as much as possible. Allowed values: https://www.alibabacloud.com/help/zh/doc-detail/40654.htm + discovery_regions = ["cn-hongkong"] + + ## how often the discovery API call executed (default 1m) + #discovery_interval = "1m" + + ## Metrics to Pull (Required) + [[inputs.aliyuncms.metrics]] + ## Metrics names to be requested, + ## described here (per project): https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq + names = ["InstanceActiveConnection", "InstanceNewConnection"] + + ## Dimension filters for Metric (these are optional). + ## This allows to get additional metric dimension. If dimension is not specified it can be returned or + ## the data can be aggregated - it depends on particular metric, you can find details here: https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11186623.6.690.1938ad41wg8QSq + ## + ## Note, that by default dimension filter includes the list of discovered objects in scope (if discovery is enabled) + ## Values specified here would be added into the list of discovered objects. + ## You can specify either single dimension: + #dimensions = '{"instanceId": "p-example"}' + + ## Or you can specify several dimensions at once: + #dimensions = '[{"instanceId": "p-example"},{"instanceId": "q-example"}]' + + ## Enrichment tags, can be added from discovery (if supported) + ## Notation is : + ## To figure out which fields are available, consult the Describe API per project. + ## For example, for SLB: https://api.aliyun.com/#/?product=Slb&version=2014-05-15&api=DescribeLoadBalancers¶ms={}&tab=MOCK&lang=GO + #tag_query_path = [ + # "address:Address", + # "name:LoadBalancerName", + # "cluster_owner:Tags.Tag[?TagKey=='cs.cluster.name'].TagValue | [0]" + # ] + ## The following tags added by default: regionId (if discovery enabled), userId, instanceId. + + ## Allow metrics without discovery data, if discovery is enabled. If set to true, then metric without discovery + ## data would be emitted, otherwise dropped. This cane be of help, in case debugging dimension filters, or partial coverage + ## of discovery scope vs monitoring scope + #allow_dps_without_discovery = false +` +) + +type ( + // AliyunCMS is aliyun cms config info. + AliyunCMS struct { + AccessKeyID string `toml:"access_key_id"` + AccessKeySecret string `toml:"access_key_secret"` + AccessKeyStsToken string `toml:"access_key_sts_token"` + RoleArn string `toml:"role_arn"` + RoleSessionName string `toml:"role_session_name"` + PrivateKey string `toml:"private_key"` + PublicKeyID string `toml:"public_key_id"` + RoleName string `toml:"role_name"` + + DiscoveryRegions []string `toml:"discovery_regions"` + DiscoveryInterval internal.Duration `toml:"discovery_interval"` + Period internal.Duration `toml:"period"` + Delay internal.Duration `toml:"delay"` + Project string `toml:"project"` + Metrics []*Metric `toml:"metrics"` + RateLimit int `toml:"ratelimit"` + + Log telegraf.Logger `toml:"-"` + + client aliyuncmsClient + windowStart time.Time + windowEnd time.Time + dt *discoveryTool + dimensionKey string + discoveryData map[string]interface{} + measurement string + } + + // Metric describes what metrics to get + Metric struct { + ObjectsFilter string `toml:"objects_filter"` + MetricNames []string `toml:"names"` + Dimensions string `toml:"dimensions"` //String representation of JSON dimensions + TagsQueryPath []string `toml:"tag_query_path"` + AllowDataPointWODiscoveryData bool `toml:"allow_dps_without_discovery"` //Allow data points without discovery data (if no discovery data found) + + dtLock sync.Mutex //Guard for discoveryTags & dimensions + discoveryTags map[string]map[string]string //Internal data structure that can enrich metrics with tags + dimensionsUdObj map[string]string + dimensionsUdArr []map[string]string //Parsed Dimesnsions JSON string (unmarshalled) + requestDimensions []map[string]string //this is the actual dimensions list that would be used in API request + requestDimensionsStr string //String representation of the above + + } + + // Dimension describe how to get metrics + Dimension struct { + Value string `toml:"value"` + } + + aliyuncmsClient interface { + DescribeMetricList(request *cms.DescribeMetricListRequest) (response *cms.DescribeMetricListResponse, err error) + } +) + +// SampleConfig implements telegraf.Inputs interface +func (s *AliyunCMS) SampleConfig() string { + return sampleConfig +} + +// Description implements telegraf.Inputs interface +func (s *AliyunCMS) Description() string { + return description +} + +func (s *AliyunCMS) Init() error { + + if s.Project == "" { + return errors.New("project is not set") + } + + var ( + roleSessionExpiration = 600 + sessionExpiration = 600 + ) + configuration := &providers.Configuration{ + AccessKeyID: s.AccessKeyID, + AccessKeySecret: s.AccessKeySecret, + AccessKeyStsToken: s.AccessKeyStsToken, + RoleArn: s.RoleArn, + RoleSessionName: s.RoleSessionName, + RoleSessionExpiration: &roleSessionExpiration, + PrivateKey: s.PrivateKey, + PublicKeyID: s.PublicKeyID, + SessionExpiration: &sessionExpiration, + RoleName: s.RoleName, + } + credentialProviders := []providers.Provider{ + providers.NewConfigurationCredentialProvider(configuration), + providers.NewEnvCredentialProvider(), + providers.NewInstanceMetadataProvider(), + } + credential, err := providers.NewChainProvider(credentialProviders).Retrieve() + if err != nil { + return errors.Errorf("failed to retrieve credential: %v", err) + } + s.client, err = cms.NewClientWithOptions("", sdk.NewConfig(), credential) + if err != nil { + return errors.Errorf("failed to create cms client: %v", err) + } + + //check metrics dimensions consistency + for _, metric := range s.Metrics { + if metric.Dimensions != "" { + metric.dimensionsUdObj = map[string]string{} + metric.dimensionsUdArr = []map[string]string{} + err := json.Unmarshal([]byte(metric.Dimensions), &metric.dimensionsUdObj) + if err != nil { + err := json.Unmarshal([]byte(metric.Dimensions), &metric.dimensionsUdArr) + return errors.Errorf("Can't parse dimensions (it is neither obj, nor array) %q :%v", metric.Dimensions, err) + } + } + } + + s.measurement = formatMeasurement(s.Project) + + //Init discovery... + if s.dt == nil { //Support for tests + s.dt, err = NewDiscoveryTool(s.DiscoveryRegions, s.Project, s.Log, credential, int(float32(s.RateLimit)*0.2), s.DiscoveryInterval.Duration) + if err != nil { + s.Log.Errorf("Discovery tool is not activated: %v", err) + s.dt = nil + return nil + } + } + + s.discoveryData, err = s.dt.getDiscoveryDataAllRegions(nil) + if err != nil { + s.Log.Errorf("Discovery tool is not activated: %v", err) + s.dt = nil + return nil + } + + s.Log.Infof("%d object(s) discovered...", len(s.discoveryData)) + + //Special setting for acs_oss project since the API differs + if s.Project == "acs_oss" { + s.dimensionKey = "BucketName" + } + + return nil +} + +func (s *AliyunCMS) Start(telegraf.Accumulator) error { + //Start periodic discovery process + if s.dt != nil { + s.dt.Start() + } + + return nil +} + +// Gather implements telegraf.Inputs interface +func (s *AliyunCMS) Gather(acc telegraf.Accumulator) error { + + s.updateWindow(time.Now()) + + // limit concurrency or we can easily exhaust user connection limit + lmtr := limiter.NewRateLimiter(s.RateLimit, time.Second) + defer lmtr.Stop() + + var wg sync.WaitGroup + for _, metric := range s.Metrics { + //Prepare internal structure with data from discovery + s.prepareTagsAndDimensions(metric) + wg.Add(len(metric.MetricNames)) + for _, metricName := range metric.MetricNames { + + <-lmtr.C + go func(metricName string, metric *Metric) { + defer wg.Done() + acc.AddError(s.gatherMetric(acc, metricName, metric)) + }(metricName, metric) + } + wg.Wait() + } + + return nil +} + +func (s *AliyunCMS) Stop() { + if s.dt != nil { + s.dt.Stop() + } +} + +func (s *AliyunCMS) updateWindow(relativeTo time.Time) { + + //https://help.aliyun.com/document_detail/51936.html?spm=a2c4g.11186623.6.701.54025679zh6wiR + //The start and end times are executed in the mode of + //opening left and closing right, and startTime cannot be equal + //to or greater than endTime. + + windowEnd := relativeTo.Add(-s.Delay.Duration) + + if s.windowEnd.IsZero() { + // this is the first run, no window info, so just get a single period + s.windowStart = windowEnd.Add(-s.Period.Duration) + } else { + // subsequent window, start where last window left off + s.windowStart = s.windowEnd + } + + s.windowEnd = windowEnd +} + +// Gather given metric and emit error +func (s *AliyunCMS) gatherMetric(acc telegraf.Accumulator, metricName string, metric *Metric) error { + + req := cms.CreateDescribeMetricListRequest() + req.Period = strconv.FormatInt(int64(s.Period.Duration.Seconds()), 10) + req.MetricName = metricName + req.Length = "10000" + req.Namespace = s.Project + req.EndTime = strconv.FormatInt(s.windowEnd.Unix()*1000, 10) + req.StartTime = strconv.FormatInt(s.windowStart.Unix()*1000, 10) + req.Dimensions = metric.requestDimensionsStr + + for more := true; more; { + resp, err := s.client.DescribeMetricList(req) + if err != nil { + return errors.Errorf("failed to query metricName list: %v", err) + } else if resp.Code != "200" { + s.Log.Errorf("failed to query metricName list: %v", resp.Message) + break + } + + var datapoints []map[string]interface{} + if err = json.Unmarshal([]byte(resp.Datapoints), &datapoints); err != nil { + return errors.Errorf("failed to decode response datapoints: %v", err) + } + + if len(datapoints) == 0 { + s.Log.Debugf("No metrics returned from CMS, response msg: %s", resp.Message) + break + } + + NextDataPoint: + for _, datapoint := range datapoints { + fields := map[string]interface{}{} + datapointTime := int64(0) + tags := map[string]string{} + for key, value := range datapoint { + switch key { + case "instanceId", "BucketName": + tags[key] = value.(string) + if metric.discoveryTags != nil { //discovery can be not activated + + //Skipping data point if discovery data not exist + if _, ok := metric.discoveryTags[value.(string)]; !ok && + !metric.AllowDataPointWODiscoveryData { + s.Log.Warnf("Instance %q is not found in discovery, skipping monitoring datapoint...", value.(string)) + continue NextDataPoint + } + + for k, v := range metric.discoveryTags[value.(string)] { + tags[k] = v + } + } + case "userId": + tags[key] = value.(string) + case "timestamp": + datapointTime = int64(value.(float64)) / 1000 + default: + fields[formatField(metricName, key)] = value + } + } + //Log.logW("Datapoint time: %s, now: %s", time.Unix(datapointTime, 0).Format(time.RFC3339), time.Now().Format(time.RFC3339)) + acc.AddFields(s.measurement, fields, tags, time.Unix(datapointTime, 0)) + } + + req.NextToken = resp.NextToken + more = req.NextToken != "" + } + + return nil +} + +//Tag helper +func parseTag(tagSpec string, data interface{}) (string, string, error) { + + tagKey := tagSpec + queryPath := tagSpec + + //Split query path to tagKey and query path + if splitted := strings.Split(tagSpec, ":"); len(splitted) == 2 { + tagKey = splitted[0] + queryPath = splitted[1] + } + + tagRawValue, err := jmespath.Search(queryPath, data) + if err != nil { + return "", "", errors.Errorf("Can't query data from discovery data using query path %q: %v", + queryPath, err) + } + + if tagRawValue == nil { //Nothing found + return "", "", nil + } + + tagValue, ok := tagRawValue.(string) + if !ok { + return "", "", errors.Errorf("Tag value %v parsed by query %q is not a string value", + tagRawValue, queryPath) + } + + return tagKey, tagValue, nil +} + +func (s *AliyunCMS) prepareTagsAndDimensions(metric *Metric) { + var ( + newData bool + defaulTags = []string{"RegionId:RegionId"} + ) + + if s.dt == nil { //Discovery is not activated + return + } + + //Reading all data from buffered channel +L: + for { + select { + case s.discoveryData = <-s.dt.dataChan: + newData = true + continue + default: + break L + } + } + + if newData || //new data arrives, process it + len(metric.discoveryTags) == 0 { //or this is the first call + + metric.dtLock.Lock() + defer metric.dtLock.Unlock() + + if metric.discoveryTags == nil { + metric.discoveryTags = make(map[string]map[string]string, len(s.discoveryData)) + } + + metric.requestDimensions = nil //erasing + metric.requestDimensions = make([]map[string]string, 0, len(s.discoveryData)) + + //Preparing tags & dims... + for instanceId, elem := range s.discoveryData { + + //Start filing tags + //Remove old value if exist + delete(metric.discoveryTags, instanceId) + metric.discoveryTags[instanceId] = make(map[string]string, len(metric.TagsQueryPath)+len(defaulTags)) + + for _, tagQueryPath := range metric.TagsQueryPath { + + tagKey, tagValue, err := parseTag(tagQueryPath, elem) + if err != nil { + s.Log.Errorf("%v", err) + continue + } + if err == nil && tagValue == "" { //Nothing found + s.Log.Debugf("Data by query path %q: is not found, for instance %q", tagQueryPath, instanceId) + continue + } + + metric.discoveryTags[instanceId][tagKey] = tagValue + } + + //Adding default tags if not already there + for _, defaultTagQP := range defaulTags { + tagKey, tagValue, err := parseTag(defaultTagQP, elem) + + if err != nil { + s.Log.Errorf("%v", err) + continue + } + + if err == nil && tagValue == "" { //Nothing found + s.Log.Debugf("Data by query path %q: is not found, for instance %q", + defaultTagQP, instanceId) + continue + } + + metric.discoveryTags[instanceId][tagKey] = tagValue + } + + //Preparing dimensions (first adding dimensions that comes from discovery data) + metric.requestDimensions = append( + metric.requestDimensions, + map[string]string{s.dimensionKey: instanceId}) + + } + + //Get final dimension (need to get full lis of + //what was provided in config + what comes from discovery + if len(metric.dimensionsUdArr) != 0 { + metric.requestDimensions = append(metric.requestDimensions, metric.dimensionsUdArr...) + } + if len(metric.dimensionsUdObj) != 0 { + metric.requestDimensions = append(metric.requestDimensions, metric.dimensionsUdObj) + } + + //Unmarshalling to string + reqDim, err := json.Marshal(metric.requestDimensions) + if err != nil { + s.Log.Errorf("Can't marshal metric request dimensions %v :%v", + metric.requestDimensions, err) + metric.requestDimensionsStr = "" + } else { + metric.requestDimensionsStr = string(reqDim) + } + + } +} + +// Formatting helpers +func formatField(metricName string, statistic string) string { + if metricName == statistic { + statistic = "value" + } + return fmt.Sprintf("%s_%s", snakeCase(metricName), snakeCase(statistic)) +} + +func formatMeasurement(project string) string { + project = strings.Replace(project, "/", "_", -1) + project = snakeCase(project) + return fmt.Sprintf("aliyuncms_%s", project) +} + +func snakeCase(s string) string { + s = internal.SnakeCase(s) + s = strings.Replace(s, "__", "_", -1) + return s +} + +func init() { + inputs.Add("aliyuncms", func() telegraf.Input { + return &AliyunCMS{ + RateLimit: 200, + DiscoveryInterval: internal.Duration{Duration: time.Minute}, + dimensionKey: "instanceId", + } + }) +} diff --git a/plugins/inputs/aliyuncms/aliyuncms_test.go b/plugins/inputs/aliyuncms/aliyuncms_test.go new file mode 100644 index 000000000..37430bbdd --- /dev/null +++ b/plugins/inputs/aliyuncms/aliyuncms_test.go @@ -0,0 +1,410 @@ +package aliyuncms + +import ( + "bytes" + "io/ioutil" + "net/http" + "testing" + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials/providers" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses" + "github.com/aliyun/alibaba-cloud-sdk-go/services/cms" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" + "github.com/influxdata/telegraf/testutil" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +const inputTitle = "inputs.aliyuncms" + +type mockGatherAliyunCMSClient struct{} + +func (m *mockGatherAliyunCMSClient) DescribeMetricList(request *cms.DescribeMetricListRequest) (*cms.DescribeMetricListResponse, error) { + + resp := new(cms.DescribeMetricListResponse) + + //switch request.Metric { + switch request.MetricName { + case "InstanceActiveConnection": + resp.Code = "200" + resp.Period = "60" + resp.Datapoints = ` + [{ + "timestamp": 1490152860000, + "Maximum": 200, + "userId": "1234567898765432", + "Minimum": 100, + "instanceId": "i-abcdefgh123456", + "Average": 150, + "Value": 300 + }]` + case "ErrorCode": + resp.Code = "404" + resp.Message = "ErrorCode" + case "ErrorDatapoint": + resp.Code = "200" + resp.Period = "60" + resp.Datapoints = ` + [{ + "timestamp": 1490152860000, + "Maximum": 200, + "userId": "1234567898765432", + "Minimum": 100, + "instanceId": "i-abcdefgh123456", + "Average": 150, + }]` + case "EmptyDatapoint": + resp.Code = "200" + resp.Period = "60" + resp.Datapoints = `[]` + case "ErrorResp": + return nil, errors.New("error response") + } + return resp, nil +} + +type mockAliyunSDKCli struct { + resp *responses.CommonResponse +} + +func (m *mockAliyunSDKCli) ProcessCommonRequest(req *requests.CommonRequest) (response *responses.CommonResponse, err error) { + return m.resp, nil +} + +func getDiscoveryTool(project string, discoverRegions []string) (*discoveryTool, error) { + var ( + err error + credential auth.Credential + ) + + configuration := &providers.Configuration{ + AccessKeyID: "dummyKey", + AccessKeySecret: "dummySecret", + } + credentialProviders := []providers.Provider{ + providers.NewConfigurationCredentialProvider(configuration), + providers.NewEnvCredentialProvider(), + providers.NewInstanceMetadataProvider(), + } + credential, err = providers.NewChainProvider(credentialProviders).Retrieve() + if err != nil { + return nil, errors.Errorf("failed to retrieve credential: %v", err) + } + + dt, err := NewDiscoveryTool(discoverRegions, project, testutil.Logger{Name: inputTitle}, credential, 1, time.Minute*2) + + if err != nil { + return nil, errors.Errorf("Can't create discovery tool object: %v", err) + } + return dt, nil +} + +func getMockSdkCli(httpResp *http.Response) (mockAliyunSDKCli, error) { + resp := responses.NewCommonResponse() + if err := responses.Unmarshal(resp, httpResp, "JSON"); err != nil { + return mockAliyunSDKCli{}, errors.Errorf("Can't parse response: %v", err) + } + return mockAliyunSDKCli{resp: resp}, nil +} + +func TestPluginDefaults(t *testing.T) { + require.Equal(t, &AliyunCMS{RateLimit: 200, + DiscoveryInterval: internal.Duration{Duration: time.Minute}, + dimensionKey: "instanceId", + }, inputs.Inputs["aliyuncms"]()) +} + +func TestPluginInitialize(t *testing.T) { + var err error + + plugin := new(AliyunCMS) + plugin.DiscoveryRegions = []string{"cn-shanghai"} + plugin.dt, err = getDiscoveryTool("acs_slb_dashboard", plugin.DiscoveryRegions) + if err != nil { + t.Fatalf("Can't create discovery tool object: %v", err) + } + + plugin.Log = testutil.Logger{Name: inputTitle} + + httpResp := &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString( + `{ + "LoadBalancers": + { + "LoadBalancer": [ + {"LoadBalancerId":"bla"} + ] + }, + "TotalCount": 1, + "PageSize": 1, + "PageNumber": 1 + }`)), + } + mockCli, err := getMockSdkCli(httpResp) + if err != nil { + t.Fatalf("Can't create mock sdk cli: %v", err) + } + plugin.dt.cli = map[string]aliyunSdkClient{plugin.DiscoveryRegions[0]: &mockCli} + + tests := []struct { + name string + project string + accessKeyID string + accessKeySecret string + expectedErrorString string + }{ + { + name: "Empty project", + expectedErrorString: "project is not set", + }, + { + name: "Valid project", + project: "acs_slb_dashboard", + accessKeyID: "dummy", + accessKeySecret: "dummy", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + plugin.Project = tt.project + plugin.AccessKeyID = tt.accessKeyID + plugin.AccessKeySecret = tt.accessKeySecret + + if tt.expectedErrorString != "" { + require.EqualError(t, plugin.Init(), tt.expectedErrorString) + } else { + require.Equal(t, nil, plugin.Init()) + } + }) + } +} + +func TestUpdateWindow(t *testing.T) { + duration, _ := time.ParseDuration("1m") + internalDuration := internal.Duration{ + Duration: duration, + } + + plugin := &AliyunCMS{ + Project: "acs_slb_dashboard", + Period: internalDuration, + Delay: internalDuration, + Log: testutil.Logger{Name: inputTitle}, + } + + now := time.Now() + + require.True(t, plugin.windowEnd.IsZero()) + require.True(t, plugin.windowStart.IsZero()) + + plugin.updateWindow(now) + + newStartTime := plugin.windowEnd + + // initial window just has a single period + require.EqualValues(t, plugin.windowEnd, now.Add(-plugin.Delay.Duration)) + require.EqualValues(t, plugin.windowStart, now.Add(-plugin.Delay.Duration).Add(-plugin.Period.Duration)) + + now = time.Now() + plugin.updateWindow(now) + + // subsequent window uses previous end time as start time + require.EqualValues(t, plugin.windowEnd, now.Add(-plugin.Delay.Duration)) + require.EqualValues(t, plugin.windowStart, newStartTime) +} + +func TestGatherMetric(t *testing.T) { + + plugin := &AliyunCMS{ + Project: "acs_slb_dashboard", + client: new(mockGatherAliyunCMSClient), + measurement: formatMeasurement("acs_slb_dashboard"), + Log: testutil.Logger{Name: inputTitle}, + } + + metric := &Metric{ + MetricNames: []string{}, + Dimensions: `"instanceId": "i-abcdefgh123456"`, + } + + tests := []struct { + name string + metricName string + expectedErrorString string + }{ + { + name: "Datapoint with corrupted JSON", + metricName: "ErrorDatapoint", + expectedErrorString: `failed to decode response datapoints: invalid character '}' looking for beginning of object key string`, + }, + { + name: "General CMS response error", + metricName: "ErrorResp", + expectedErrorString: "failed to query metricName list: error response", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var acc telegraf.Accumulator + require.EqualError(t, plugin.gatherMetric(acc, tt.metricName, metric), tt.expectedErrorString) + + }) + } +} + +func TestGather(t *testing.T) { + + metric := &Metric{ + MetricNames: []string{}, + Dimensions: `{"instanceId": "i-abcdefgh123456"}`, + } + plugin := &AliyunCMS{ + AccessKeyID: "my_access_key_id", + AccessKeySecret: "my_access_key_secret", + Project: "acs_slb_dashboard", + Metrics: []*Metric{metric}, + RateLimit: 200, + measurement: formatMeasurement("acs_slb_dashboard"), + DiscoveryRegions: []string{"cn-shanghai"}, + client: new(mockGatherAliyunCMSClient), + Log: testutil.Logger{Name: inputTitle}, + } + + //test table: + tests := []struct { + name string + hasMeasurment bool + metricNames []string + expected []telegraf.Metric + }{ + { + name: "Empty data point", + metricNames: []string{"EmptyDatapoint"}, + expected: []telegraf.Metric{ + testutil.MustMetric( + "aliyuncms_acs_slb_dashboard", + nil, + nil, + time.Time{}), + }, + }, + { + name: "Data point with fields & tags", + hasMeasurment: true, + metricNames: []string{"InstanceActiveConnection"}, + expected: []telegraf.Metric{ + testutil.MustMetric( + "aliyuncms_acs_slb_dashboard", + map[string]string{ + "instanceId": "i-abcdefgh123456", + "userId": "1234567898765432", + }, + map[string]interface{}{ + "instance_active_connection_minimum": float64(100), + "instance_active_connection_maximum": float64(200), + "instance_active_connection_average": float64(150), + "instance_active_connection_value": float64(300), + }, + time.Unix(1490152860000, 0)), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var acc testutil.Accumulator + plugin.Metrics[0].MetricNames = tt.metricNames + require.Empty(t, acc.GatherError(plugin.Gather)) + require.Equal(t, acc.HasMeasurement("aliyuncms_acs_slb_dashboard"), tt.hasMeasurment) + if tt.hasMeasurment { + acc.AssertContainsTaggedFields(t, "aliyuncms_acs_slb_dashboard", tt.expected[0].Fields(), tt.expected[0].Tags()) + } + }) + } +} + +func TestGetDiscoveryDataAllRegions(t *testing.T) { + + //test table: + tests := []struct { + name string + project string + region string + httpResp *http.Response + discData map[string]interface{} + totalCount int + pageSize int + pageNumber int + expectedErrorString string + }{ + { + name: "No root key in discovery response", + project: "acs_slb_dashboard", + region: "cn-hongkong", + httpResp: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString(`{}`)), + }, + totalCount: 0, + pageSize: 0, + pageNumber: 0, + expectedErrorString: `Didn't find root key "LoadBalancers" in discovery response`, + }, + { + name: "1 object discovered", + project: "acs_slb_dashboard", + region: "cn-hongkong", + httpResp: &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString( + `{ + "LoadBalancers": + { + "LoadBalancer": [ + {"LoadBalancerId":"bla"} + ] + }, + "TotalCount": 1, + "PageSize": 1, + "PageNumber": 1 + }`)), + }, + discData: map[string]interface{}{"bla": map[string]interface{}{"LoadBalancerId": "bla"}}, + totalCount: 1, + pageSize: 1, + pageNumber: 1, + expectedErrorString: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dt, err := getDiscoveryTool(tt.project, []string{tt.region}) + if err != nil { + t.Fatalf("Can't create discovery tool object: %v", err) + } + + mockCli, err := getMockSdkCli(tt.httpResp) + if err != nil { + t.Fatalf("Can't create mock sdk cli: %v", err) + } + dt.cli = map[string]aliyunSdkClient{tt.region: &mockCli} + data, err := dt.getDiscoveryDataAllRegions(nil) + + require.Equal(t, tt.discData, data) + if err != nil { + require.EqualError(t, err, tt.expectedErrorString) + } + + }) + } + +} diff --git a/plugins/inputs/aliyuncms/discovery.go b/plugins/inputs/aliyuncms/discovery.go new file mode 100644 index 000000000..39e0044b6 --- /dev/null +++ b/plugins/inputs/aliyuncms/discovery.go @@ -0,0 +1,511 @@ +package aliyuncms + +import ( + "encoding/json" + "github.com/influxdata/telegraf" + "reflect" + "regexp" + "strconv" + "sync" + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses" + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/aliyun/alibaba-cloud-sdk-go/services/rds" + "github.com/aliyun/alibaba-cloud-sdk-go/services/slb" + "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" + "github.com/influxdata/telegraf/internal/limiter" + "github.com/pkg/errors" +) + +// https://www.alibabacloud.com/help/doc-detail/40654.htm?gclid=Cj0KCQjw4dr0BRCxARIsAKUNjWTAMfyVUn_Y3OevFBV3CMaazrhq0URHsgE7c0m0SeMQRKlhlsJGgIEaAviyEALw_wcB +var aliyunRegionList = []string{ + "cn-qingdao", + "cn-beijing", + "cn-zhangjiakou", + "cn-huhehaote", + "cn-hangzhou", + "cn-shanghai", + "cn-shenzhen", + "cn-heyuan", + "cn-chengdu", + "cn-hongkong", + "ap-southeast-1", + "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-5", + "ap-south-1", + "ap-northeast-1", + "us-west-1", + "us-east-1", + "eu-central-1", + "eu-west-1", + "me-east-1", +} + +type discoveryRequest interface { +} + +type aliyunSdkClient interface { + ProcessCommonRequest(req *requests.CommonRequest) (response *responses.CommonResponse, err error) +} + +type discoveryTool struct { + req map[string]discoveryRequest //Discovery request (specific per object type) + rateLimit int //Rate limit for API query, as it is limited by API backend + reqDefaultPageSize int //Default page size while querying data from API (how many objects per request) + cli map[string]aliyunSdkClient //API client, which perform discovery request + + respRootKey string //Root key in JSON response where to look for discovery data + respObjectIdKey string //Key in element of array under root key, that stores object ID + //for ,majority of cases it would be InstanceId, for OSS it is BucketName. This key is also used in dimension filtering// ) + wg sync.WaitGroup //WG for primary discovery goroutine + interval time.Duration //Discovery interval + done chan bool //Done channel to stop primary discovery goroutine + dataChan chan map[string]interface{} //Discovery data + lg telegraf.Logger //Telegraf logger (should be provided) +} + +//getRpcReqFromDiscoveryRequest - utility function to map between aliyun request primitives +//discoveryRequest represents different type of discovery requests +func getRpcReqFromDiscoveryRequest(req discoveryRequest) (*requests.RpcRequest, error) { + + if reflect.ValueOf(req).Type().Kind() != reflect.Ptr || + reflect.ValueOf(req).IsNil() { + return nil, errors.Errorf("Not expected type of the discovery request object: %q, %q", reflect.ValueOf(req).Type(), reflect.ValueOf(req).Kind()) + } + + ptrV := reflect.Indirect(reflect.ValueOf(req)) + + for i := 0; i < ptrV.NumField(); i++ { + + if ptrV.Field(i).Type().String() == "*requests.RpcRequest" { + if !ptrV.Field(i).CanInterface() { + return nil, errors.Errorf("Can't get interface of %v", ptrV.Field(i)) + } + + rpcReq, ok := ptrV.Field(i).Interface().(*requests.RpcRequest) + + if !ok { + return nil, errors.Errorf("Cant convert interface of %v to '*requests.RpcRequest' type", ptrV.Field(i).Interface()) + } + + return rpcReq, nil + } + } + return nil, errors.Errorf("Didn't find *requests.RpcRequest embedded struct in %q", ptrV.Type()) +} + +//NewDiscoveryTool function returns discovery tool object. +//The object is used to periodically get data about aliyun objects and send this +//data into channel. The intention is to enrich reported metrics with discovery data. +//Discovery is supported for a limited set of object types (defined by project) and can be extended in future. +//Discovery can be limited by region if not set, then all regions is queried. +//Request against API can inquire additional costs, consult with aliyun API documentation. +func NewDiscoveryTool(regions []string, project string, lg telegraf.Logger, credential auth.Credential, rateLimit int, discoveryInterval time.Duration) (*discoveryTool, error) { + var ( + dscReq = map[string]discoveryRequest{} + cli = map[string]aliyunSdkClient{} + parseRootKey = regexp.MustCompile(`Describe(.*)`) + responseRootKey string + responseObjectIdKey string + err error + noDiscoverySupportErr = errors.Errorf("no discovery support for project %q", project) + ) + + if len(regions) == 0 { + regions = aliyunRegionList + lg.Warnf("Discovery regions are not provided! Data will be queried across %d regions!", len(aliyunRegionList)) + } + + if rateLimit == 0 { //Can be a rounding case + rateLimit = 1 + } + + for _, region := range regions { + switch project { + case "acs_ecs_dashboard": + dscReq[region] = ecs.CreateDescribeInstancesRequest() + responseObjectIdKey = "InstanceId" + case "acs_rds_dashboard": + dscReq[region] = rds.CreateDescribeDBInstancesRequest() + responseObjectIdKey = "DBInstanceId" + case "acs_slb_dashboard": + dscReq[region] = slb.CreateDescribeLoadBalancersRequest() + responseObjectIdKey = "LoadBalancerId" + case "acs_memcache": + return nil, noDiscoverySupportErr + case "acs_ocs": + return nil, noDiscoverySupportErr + case "acs_oss": + //oss is really complicated + //it is on it's own format + return nil, noDiscoverySupportErr + + //As a possible solution we can + //mimic to request format supported by oss + + //req := DescribeLOSSRequest{ + // RpcRequest: &requests.RpcRequest{}, + //} + //req.InitWithApiInfo("oss", "2014-08-15", "DescribeDBInstances", "oss", "openAPI") + case "acs_vpc_eip": + dscReq[region] = vpc.CreateDescribeEipAddressesRequest() + responseObjectIdKey = "AllocationId" + case "acs_kvstore": + return nil, noDiscoverySupportErr + case "acs_mns_new": + return nil, noDiscoverySupportErr + case "acs_cdn": + //API replies are in its own format. + return nil, noDiscoverySupportErr + case "acs_polardb": + return nil, noDiscoverySupportErr + case "acs_gdb": + return nil, noDiscoverySupportErr + case "acs_ads": + return nil, noDiscoverySupportErr + case "acs_mongodb": + return nil, noDiscoverySupportErr + case "acs_express_connect": + return nil, noDiscoverySupportErr + case "acs_fc": + return nil, noDiscoverySupportErr + case "acs_nat_gateway": + return nil, noDiscoverySupportErr + case "acs_sls_dashboard": + return nil, noDiscoverySupportErr + case "acs_containerservice_dashboard": + return nil, noDiscoverySupportErr + case "acs_vpn": + return nil, noDiscoverySupportErr + case "acs_bandwidth_package": + return nil, noDiscoverySupportErr + case "acs_cen": + return nil, noDiscoverySupportErr + case "acs_ens": + return nil, noDiscoverySupportErr + case "acs_opensearch": + return nil, noDiscoverySupportErr + case "acs_scdn": + return nil, noDiscoverySupportErr + case "acs_drds": + return nil, noDiscoverySupportErr + case "acs_iot": + return nil, noDiscoverySupportErr + case "acs_directmail": + return nil, noDiscoverySupportErr + case "acs_elasticsearch": + return nil, noDiscoverySupportErr + case "acs_ess_dashboard": + return nil, noDiscoverySupportErr + case "acs_streamcompute": + return nil, noDiscoverySupportErr + case "acs_global_acceleration": + return nil, noDiscoverySupportErr + case "acs_hitsdb": + return nil, noDiscoverySupportErr + case "acs_kafka": + return nil, noDiscoverySupportErr + case "acs_openad": + return nil, noDiscoverySupportErr + case "acs_pcdn": + return nil, noDiscoverySupportErr + case "acs_dcdn": + return nil, noDiscoverySupportErr + case "acs_petadata": + return nil, noDiscoverySupportErr + case "acs_videolive": + return nil, noDiscoverySupportErr + case "acs_hybriddb": + return nil, noDiscoverySupportErr + case "acs_adb": + return nil, noDiscoverySupportErr + case "acs_mps": + return nil, noDiscoverySupportErr + case "acs_maxcompute_prepay": + return nil, noDiscoverySupportErr + case "acs_hdfs": + return nil, noDiscoverySupportErr + case "acs_ddh": + return nil, noDiscoverySupportErr + case "acs_hbr": + return nil, noDiscoverySupportErr + case "acs_hdr": + return nil, noDiscoverySupportErr + case "acs_cds": + return nil, noDiscoverySupportErr + default: + return nil, errors.Errorf("project %q is not recognized by discovery...", project) + } + + cli[region], err = sdk.NewClientWithOptions(region, sdk.NewConfig(), credential) + if err != nil { + return nil, err + } + } + + if len(dscReq) == 0 || len(cli) == 0 { + return nil, errors.Errorf("Can't build discovery request for project: %q,\nregions: %v", project, regions) + } + + //Getting response root key (if not set already). This is to be able to parse discovery responses + //As they differ per object type + //Discovery requests are of the same type per every region, so pick the first one + rpcReq, err := getRpcReqFromDiscoveryRequest(dscReq[regions[0]]) + //This means that the discovery request is not of proper type/kind + if err != nil { + return nil, errors.Errorf("Can't parse rpc request object from discovery request %v", dscReq[regions[0]]) + } + + /* + The action name is of the following format Describe, + For example: DescribeLoadBalancers -> for SLB project, or DescribeInstances for ECS project + We will use it to construct root key name in the discovery API response. + It follows the following logic: for 'DescribeLoadBalancers' action in discovery request we get the response + in json of the following structure: + { + ... + "LoadBalancers": { + "LoadBalancer": [ here comes objects, one per every instance] + } + } + As we can see, the root key is a part of action name, except first word (part) 'Describe' + */ + result := parseRootKey.FindStringSubmatch(rpcReq.GetActionName()) + if result == nil || len(result) != 2 { + return nil, errors.Errorf("Can't parse the discovery response root key from request action name %q", rpcReq.GetActionName()) + } + responseRootKey = result[1] + + return &discoveryTool{ + req: dscReq, + cli: cli, + respRootKey: responseRootKey, + respObjectIdKey: responseObjectIdKey, + rateLimit: rateLimit, + interval: discoveryInterval, + reqDefaultPageSize: 20, + dataChan: make(chan map[string]interface{}, 1), + lg: lg, + }, nil +} + +func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse) (discData []interface{}, totalCount int, pageSize int, pageNumber int, err error) { + var ( + fullOutput = map[string]interface{}{} + data []byte + foundDataItem bool + foundRootKey bool + ) + + data = resp.GetHttpContentBytes() + if data == nil { //No data + return nil, 0, 0, 0, errors.Errorf("No data in response to be parsed") + } + + err = json.Unmarshal(data, &fullOutput) + if err != nil { + return nil, 0, 0, 0, errors.Errorf("Can't parse JSON from discovery response: %v", err) + } + + for key, val := range fullOutput { + switch key { + case dt.respRootKey: + foundRootKey = true + rootKeyVal, ok := val.(map[string]interface{}) + if !ok { + return nil, 0, 0, 0, errors.Errorf("Content of root key %q, is not an object: %v", key, val) + } + + //It should contain the array with discovered data + for _, item := range rootKeyVal { + + if discData, foundDataItem = item.([]interface{}); foundDataItem { + break + } + } + if !foundDataItem { + return nil, 0, 0, 0, errors.Errorf("Didn't find array item in root key %q", key) + } + case "TotalCount": + totalCount = int(val.(float64)) + case "PageSize": + pageSize = int(val.(float64)) + case "PageNumber": + pageNumber = int(val.(float64)) + } + + } + if !foundRootKey { + return nil, 0, 0, 0, errors.Errorf("Didn't find root key %q in discovery response", dt.respRootKey) + } + + return +} + +func (dt *discoveryTool) getDiscoveryData(cli aliyunSdkClient, req *requests.CommonRequest, limiter chan bool) (map[string]interface{}, error) { + var ( + err error + resp *responses.CommonResponse + data []interface{} + discoveryData []interface{} + totalCount int + pageNumber int + ) + defer delete(req.QueryParams, "PageNumber") + + for { + if limiter != nil { + <-limiter //Rate limiting + } + + resp, err = cli.ProcessCommonRequest(req) + if err != nil { + return nil, err + } + + data, totalCount, _, pageNumber, err = dt.parseDiscoveryResponse(resp) + if err != nil { + return nil, err + } + discoveryData = append(discoveryData, data...) + + //Pagination + pageNumber++ + req.QueryParams["PageNumber"] = strconv.Itoa(pageNumber) + + if len(discoveryData) == totalCount { //All data received + //Map data to appropriate shape before return + preparedData := map[string]interface{}{} + + for _, raw := range discoveryData { + if elem, ok := raw.(map[string]interface{}); ok { + if objectId, ok := elem[dt.respObjectIdKey].(string); ok { + preparedData[objectId] = elem + } + } else { + return nil, errors.Errorf("Can't parse input data element, not a map[string]interface{} type") + } + + } + + return preparedData, nil + } + + } + +} + +func (dt *discoveryTool) getDiscoveryDataAllRegions(limiter chan bool) (map[string]interface{}, error) { + var ( + data map[string]interface{} + resultData = map[string]interface{}{} + ) + + for region, cli := range dt.cli { + //Building common request, as the code below is the same no matter + //which aliyun object type (project) is used + dscReq, ok := dt.req[region] + if !ok { + return nil, errors.Errorf("Error building common discovery request: not valid region %q", region) + } + + rpcReq, err := getRpcReqFromDiscoveryRequest(dscReq) + if err != nil { + return nil, err + } + + commonRequest := requests.NewCommonRequest() + commonRequest.Method = rpcReq.GetMethod() + commonRequest.Product = rpcReq.GetProduct() + commonRequest.Domain = rpcReq.GetDomain() + commonRequest.Version = rpcReq.GetVersion() + commonRequest.Scheme = rpcReq.GetScheme() + commonRequest.ApiName = rpcReq.GetActionName() + commonRequest.QueryParams = rpcReq.QueryParams + commonRequest.QueryParams["PageSize"] = strconv.Itoa(dt.reqDefaultPageSize) + commonRequest.TransToAcsRequest() + + //Get discovery data using common request + data, err = dt.getDiscoveryData(cli, commonRequest, limiter) + if err != nil { + return nil, err + } + + for k, v := range data { + resultData[k] = v + } + } + return resultData, nil +} + +func (dt *discoveryTool) Start() { + var ( + err error + data map[string]interface{} + lastData map[string]interface{} + ) + + //Initializing channel + dt.done = make(chan bool) + + dt.wg.Add(1) + go func() { + defer dt.wg.Done() + + ticker := time.NewTicker(dt.interval) + defer ticker.Stop() + + lmtr := limiter.NewRateLimiter(dt.rateLimit, time.Second) + defer lmtr.Stop() + + for { + select { + case <-dt.done: + return + case <-ticker.C: + + data, err = dt.getDiscoveryDataAllRegions(lmtr.C) + if err != nil { + dt.lg.Errorf("Can't get discovery data: %v", err) + continue + } + + if !reflect.DeepEqual(data, lastData) { + lastData = nil + lastData = map[string]interface{}{} + for k, v := range data { + lastData[k] = v + } + + //send discovery data in blocking mode + dt.dataChan <- data + } + + } + } + }() +} + +func (dt *discoveryTool) Stop() { + + close(dt.done) + + //Shutdown timer + timer := time.NewTimer(time.Second * 3) + defer timer.Stop() +L: + for { //Unblock go routine by reading from dt.dataChan + select { + case <-timer.C: + break L + case <-dt.dataChan: + } + } + + dt.wg.Wait() +} diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index d5eeead0a..9b22cd442 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -4,6 +4,7 @@ import ( //Blank imports for plugins to register themselves _ "github.com/influxdata/telegraf/plugins/inputs/activemq" _ "github.com/influxdata/telegraf/plugins/inputs/aerospike" + _ "github.com/influxdata/telegraf/plugins/inputs/aliyuncms" _ "github.com/influxdata/telegraf/plugins/inputs/amqp_consumer" _ "github.com/influxdata/telegraf/plugins/inputs/apache" _ "github.com/influxdata/telegraf/plugins/inputs/apcupsd"