From de01d37a36288fd1688797ec50a5f58a578dab76 Mon Sep 17 00:00:00 2001 From: Alexander Krantz Date: Tue, 3 Aug 2021 14:29:26 -0700 Subject: [PATCH] feat: Support AWS Web Identity Provider (#9411) --- config/aws/credentials.go | 26 ++++--- plugins/inputs/cloudwatch/README.md | 27 ++++--- plugins/inputs/cloudwatch/cloudwatch.go | 39 +++------- plugins/inputs/cloudwatch/cloudwatch_test.go | 9 ++- plugins/inputs/kinesis_consumer/README.md | 15 ++-- .../kinesis_consumer/kinesis_consumer.go | 38 +++------ plugins/outputs/cloudwatch/README.md | 16 ++-- plugins/outputs/cloudwatch/cloudwatch.go | 39 +++------- plugins/outputs/cloudwatch_logs/README.md | 28 ++++--- .../cloudwatch_logs/cloudwatch_logs.go | 40 +++------- .../cloudwatch_logs/cloudwatch_logs_test.go | 77 ++++++++++++------- plugins/outputs/kinesis/README.md | 16 ++-- plugins/outputs/kinesis/kinesis.go | 39 +++------- plugins/outputs/timestream/README.md | 17 ++-- plugins/outputs/timestream/timestream.go | 38 +++------ 15 files changed, 214 insertions(+), 250 deletions(-) diff --git a/config/aws/credentials.go b/config/aws/credentials.go index d697d96ac..d2c2b2848 100644 --- a/config/aws/credentials.go +++ b/config/aws/credentials.go @@ -9,14 +9,16 @@ import ( ) type CredentialConfig struct { - Region string - AccessKey string - SecretKey string - RoleARN string - Profile string - Filename string - Token string - EndpointURL string + Region string `toml:"region"` + AccessKey string `toml:"access_key"` + SecretKey string `toml:"secret_key"` + RoleARN string `toml:"role_arn"` + Profile string `toml:"profile"` + Filename string `toml:"shared_credential_file"` + Token string `toml:"token"` + EndpointURL string `toml:"endpoint_url"` + RoleSessionName string `toml:"role_session_name"` + WebIdentityTokenFile string `toml:"web_identity_token_file"` } func (c *CredentialConfig) Credentials() client.ConfigProvider { @@ -49,6 +51,12 @@ func (c *CredentialConfig) assumeCredentials() client.ConfigProvider { Region: aws.String(c.Region), Endpoint: &c.EndpointURL, } - config.Credentials = stscreds.NewCredentials(rootCredentials, c.RoleARN) + + if c.WebIdentityTokenFile != "" { + config.Credentials = stscreds.NewWebIdentityCredentials(rootCredentials, c.RoleARN, c.RoleSessionName, c.WebIdentityTokenFile) + } else { + config.Credentials = stscreds.NewCredentials(rootCredentials, c.RoleARN) + } + return session.New(config) } diff --git a/plugins/inputs/cloudwatch/README.md b/plugins/inputs/cloudwatch/README.md index a0c175e1e..e09acc518 100644 --- a/plugins/inputs/cloudwatch/README.md +++ b/plugins/inputs/cloudwatch/README.md @@ -23,18 +23,21 @@ API endpoint. In the following order the plugin will attempt to authenticate. ## Amazon Credentials ## Credentials are loaded in the following order - ## 1) Assumed credentials via STS if role_arn is specified - ## 2) explicit credentials from 'access_key' and 'secret_key' - ## 3) shared profile from 'profile' - ## 4) environment variables - ## 5) shared credentials file - ## 6) EC2 Instance Profile - # access_key = "" - # secret_key = "" - # token = "" - # role_arn = "" - # profile = "" - # shared_credential_file = "" + ## 1) Web identity provider credentials via STS if role_arn and web_identity_token_file are specified + ## 2) Assumed credentials via STS if role_arn is specified + ## 3) explicit credentials from 'access_key' and 'secret_key' + ## 4) shared profile from 'profile' + ## 5) environment variables + ## 6) shared credentials file + ## 7) EC2 Instance Profile + #access_key = "" + #secret_key = "" + #token = "" + #role_arn = "" + #web_identity_token_file = "" + #role_session_name = "" + #profile = "" + #shared_credential_file = "" ## Endpoint to make request against, the correct endpoint is automatically ## determined and this option should only be set if you wish to override the diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go index c4df8f9a7..fff2da1d3 100644 --- a/plugins/inputs/cloudwatch/cloudwatch.go +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -25,14 +25,6 @@ import ( // CloudWatch contains the configuration and cache for the cloudwatch plugin. type CloudWatch struct { - Region string `toml:"region"` - AccessKey string `toml:"access_key"` - SecretKey string `toml:"secret_key"` - RoleARN string `toml:"role_arn"` - Profile string `toml:"profile"` - CredentialPath string `toml:"shared_credential_file"` - Token string `toml:"token"` - EndpointURL string `toml:"endpoint_url"` StatisticExclude []string `toml:"statistic_exclude"` StatisticInclude []string `toml:"statistic_include"` Timeout config.Duration `toml:"timeout"` @@ -55,6 +47,8 @@ type CloudWatch struct { queryDimensions map[string]*map[string]string windowStart time.Time windowEnd time.Time + + internalaws.CredentialConfig } // Metric defines a simplified Cloudwatch metric. @@ -93,16 +87,19 @@ func (c *CloudWatch) SampleConfig() string { ## Amazon Credentials ## Credentials are loaded in the following order - ## 1) Assumed credentials via STS if role_arn is specified - ## 2) explicit credentials from 'access_key' and 'secret_key' - ## 3) shared profile from 'profile' - ## 4) environment variables - ## 5) shared credentials file - ## 6) EC2 Instance Profile + ## 1) Web identity provider credentials via STS if role_arn and web_identity_token_file are specified + ## 2) Assumed credentials via STS if role_arn is specified + ## 3) explicit credentials from 'access_key' and 'secret_key' + ## 4) shared profile from 'profile' + ## 5) environment variables + ## 6) shared credentials file + ## 7) EC2 Instance Profile # access_key = "" # secret_key = "" # token = "" # role_arn = "" + # web_identity_token_file = "" + # role_session_name = "" # profile = "" # shared_credential_file = "" @@ -258,18 +255,6 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { } func (c *CloudWatch) initializeCloudWatch() error { - credentialConfig := &internalaws.CredentialConfig{ - Region: c.Region, - AccessKey: c.AccessKey, - SecretKey: c.SecretKey, - RoleARN: c.RoleARN, - Profile: c.Profile, - Filename: c.CredentialPath, - Token: c.Token, - EndpointURL: c.EndpointURL, - } - configProvider := credentialConfig.Credentials() - proxy, err := c.HTTPProxy.Proxy() if err != nil { return err @@ -295,7 +280,7 @@ func (c *CloudWatch) initializeCloudWatch() error { } loglevel := aws.LogOff - c.client = cwClient.New(configProvider, cfg.WithLogLevel(loglevel)) + c.client = cwClient.New(c.CredentialConfig.Credentials(), cfg.WithLogLevel(loglevel)) // Initialize regex matchers for each Dimension value. for _, m := range c.Metrics { diff --git a/plugins/inputs/cloudwatch/cloudwatch_test.go b/plugins/inputs/cloudwatch/cloudwatch_test.go index ccd27ec22..860bf41d9 100644 --- a/plugins/inputs/cloudwatch/cloudwatch_test.go +++ b/plugins/inputs/cloudwatch/cloudwatch_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/influxdata/telegraf/config" + internalaws "github.com/influxdata/telegraf/config/aws" "github.com/influxdata/telegraf/filter" "github.com/influxdata/telegraf/plugins/common/proxy" "github.com/influxdata/telegraf/testutil" @@ -105,7 +106,9 @@ func TestGather(t *testing.T) { duration, _ := time.ParseDuration("1m") internalDuration := config.Duration(duration) c := &CloudWatch{ - Region: "us-east-1", + CredentialConfig: internalaws.CredentialConfig{ + Region: "us-east-1", + }, Namespace: "AWS/ELB", Delay: internalDuration, Period: internalDuration, @@ -189,7 +192,9 @@ func TestSelectMetrics(t *testing.T) { duration, _ := time.ParseDuration("1m") internalDuration := config.Duration(duration) c := &CloudWatch{ - Region: "us-east-1", + CredentialConfig: internalaws.CredentialConfig{ + Region: "us-east-1", + }, Namespace: "AWS/ELB", Delay: internalDuration, Period: internalDuration, diff --git a/plugins/inputs/kinesis_consumer/README.md b/plugins/inputs/kinesis_consumer/README.md index ad25940d5..ba1a7580f 100644 --- a/plugins/inputs/kinesis_consumer/README.md +++ b/plugins/inputs/kinesis_consumer/README.md @@ -13,16 +13,19 @@ and creates metrics using one of the supported [input data formats][]. ## Amazon Credentials ## Credentials are loaded in the following order - ## 1) Assumed credentials via STS if role_arn is specified - ## 2) explicit credentials from 'access_key' and 'secret_key' - ## 3) shared profile from 'profile' - ## 4) environment variables - ## 5) shared credentials file - ## 6) EC2 Instance Profile + ## 1) Web identity provider credentials via STS if role_arn and web_identity_token_file are specified + ## 2) Assumed credentials via STS if role_arn is specified + ## 3) explicit credentials from 'access_key' and 'secret_key' + ## 4) shared profile from 'profile' + ## 5) environment variables + ## 6) shared credentials file + ## 7) EC2 Instance Profile # access_key = "" # secret_key = "" # token = "" # role_arn = "" + # web_identity_token_file = "" + # role_session_name = "" # profile = "" # shared_credential_file = "" diff --git a/plugins/inputs/kinesis_consumer/kinesis_consumer.go b/plugins/inputs/kinesis_consumer/kinesis_consumer.go index 57b199831..7acd3202c 100644 --- a/plugins/inputs/kinesis_consumer/kinesis_consumer.go +++ b/plugins/inputs/kinesis_consumer/kinesis_consumer.go @@ -30,14 +30,6 @@ type ( } KinesisConsumer struct { - Region string `toml:"region"` - AccessKey string `toml:"access_key"` - SecretKey string `toml:"secret_key"` - RoleARN string `toml:"role_arn"` - Profile string `toml:"profile"` - Filename string `toml:"shared_credential_file"` - Token string `toml:"token"` - EndpointURL string `toml:"endpoint_url"` StreamName string `toml:"streamname"` ShardIteratorType string `toml:"shard_iterator_type"` DynamoDB *DynamoDB `toml:"checkpoint_dynamodb"` @@ -62,6 +54,8 @@ type ( processContentEncodingFunc processContent lastSeqNum *big.Int + + internalaws.CredentialConfig } checkpoint struct { @@ -85,16 +79,19 @@ var sampleConfig = ` ## Amazon Credentials ## Credentials are loaded in the following order - ## 1) Assumed credentials via STS if role_arn is specified - ## 2) explicit credentials from 'access_key' and 'secret_key' - ## 3) shared profile from 'profile' - ## 4) environment variables - ## 5) shared credentials file - ## 6) EC2 Instance Profile + ## 1) Web identity provider credentials via STS if role_arn and web_identity_token_file are specified + ## 2) Assumed credentials via STS if role_arn is specified + ## 3) explicit credentials from 'access_key' and 'secret_key' + ## 4) shared profile from 'profile' + ## 5) environment variables + ## 6) shared credentials file + ## 7) EC2 Instance Profile # access_key = "" # secret_key = "" # token = "" # role_arn = "" + # web_identity_token_file = "" + # role_session_name = "" # profile = "" # shared_credential_file = "" @@ -156,18 +153,7 @@ func (k *KinesisConsumer) SetParser(parser parsers.Parser) { } func (k *KinesisConsumer) connect(ac telegraf.Accumulator) error { - credentialConfig := &internalaws.CredentialConfig{ - Region: k.Region, - AccessKey: k.AccessKey, - SecretKey: k.SecretKey, - RoleARN: k.RoleARN, - Profile: k.Profile, - Filename: k.Filename, - Token: k.Token, - EndpointURL: k.EndpointURL, - } - configProvider := credentialConfig.Credentials() - client := kinesis.New(configProvider) + client := kinesis.New(k.CredentialConfig.Credentials()) k.checkpoint = &noopCheckpoint{} if k.DynamoDB != nil { diff --git a/plugins/outputs/cloudwatch/README.md b/plugins/outputs/cloudwatch/README.md index 418fe86ff..56436c3c5 100644 --- a/plugins/outputs/cloudwatch/README.md +++ b/plugins/outputs/cloudwatch/README.md @@ -6,12 +6,16 @@ This plugin will send metrics to Amazon CloudWatch. This plugin uses a credential chain for Authentication with the CloudWatch API endpoint. In the following order the plugin will attempt to authenticate. -1. Assumed credentials via STS if `role_arn` attribute is specified (source credentials are evaluated from subsequent rules) -2. Explicit credentials from `access_key`, `secret_key`, and `token` attributes -3. Shared profile from `profile` attribute -4. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables) -5. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file) -6. [EC2 Instance Profile](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) +1. Web identity provider credentials via STS if `role_arn` and `web_identity_token_file` are specified +2. Assumed credentials via STS if `role_arn` attribute is specified (source credentials are evaluated from subsequent rules) +3. Explicit credentials from `access_key`, `secret_key`, and `token` attributes +4. Shared profile from `profile` attribute +5. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables) +6. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file) +7. [EC2 Instance Profile](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) + +If you are using credentials from a web identity provider, you can specify the session name using `role_session_name`. If +left empty, the current timestamp will be used. The IAM user needs only the `cloudwatch:PutMetricData` permission. diff --git a/plugins/outputs/cloudwatch/cloudwatch.go b/plugins/outputs/cloudwatch/cloudwatch.go index 9844ab271..ddf362232 100644 --- a/plugins/outputs/cloudwatch/cloudwatch.go +++ b/plugins/outputs/cloudwatch/cloudwatch.go @@ -15,15 +15,6 @@ import ( ) type CloudWatch struct { - Region string `toml:"region"` - AccessKey string `toml:"access_key"` - SecretKey string `toml:"secret_key"` - RoleARN string `toml:"role_arn"` - Profile string `toml:"profile"` - Filename string `toml:"shared_credential_file"` - Token string `toml:"token"` - EndpointURL string `toml:"endpoint_url"` - Namespace string `toml:"namespace"` // CloudWatch Metrics Namespace HighResolutionMetrics bool `toml:"high_resolution_metrics"` svc *cloudwatch.CloudWatch @@ -31,6 +22,8 @@ type CloudWatch struct { WriteStatistics bool `toml:"write_statistics"` Log telegraf.Logger `toml:"-"` + + internalaws.CredentialConfig } type statisticType int @@ -159,16 +152,19 @@ var sampleConfig = ` ## Amazon Credentials ## Credentials are loaded in the following order - ## 1) Assumed credentials via STS if role_arn is specified - ## 2) explicit credentials from 'access_key' and 'secret_key' - ## 3) shared profile from 'profile' - ## 4) environment variables - ## 5) shared credentials file - ## 6) EC2 Instance Profile + ## 1) Web identity provider credentials via STS if role_arn and web_identity_token_file are specified + ## 2) Assumed credentials via STS if role_arn is specified + ## 3) explicit credentials from 'access_key' and 'secret_key' + ## 4) shared profile from 'profile' + ## 5) environment variables + ## 6) shared credentials file + ## 7) EC2 Instance Profile #access_key = "" #secret_key = "" #token = "" #role_arn = "" + #web_identity_token_file = "" + #role_session_name = "" #profile = "" #shared_credential_file = "" @@ -202,18 +198,7 @@ func (c *CloudWatch) Description() string { } func (c *CloudWatch) Connect() error { - credentialConfig := &internalaws.CredentialConfig{ - Region: c.Region, - AccessKey: c.AccessKey, - SecretKey: c.SecretKey, - RoleARN: c.RoleARN, - Profile: c.Profile, - Filename: c.Filename, - Token: c.Token, - EndpointURL: c.EndpointURL, - } - configProvider := credentialConfig.Credentials() - c.svc = cloudwatch.New(configProvider) + c.svc = cloudwatch.New(c.CredentialConfig.Credentials()) return nil } diff --git a/plugins/outputs/cloudwatch_logs/README.md b/plugins/outputs/cloudwatch_logs/README.md index 26dd3cfaf..ab745d877 100644 --- a/plugins/outputs/cloudwatch_logs/README.md +++ b/plugins/outputs/cloudwatch_logs/README.md @@ -6,12 +6,13 @@ This plugin will send logs to Amazon CloudWatch. This plugin uses a credential chain for Authentication with the CloudWatch Logs API endpoint. In the following order the plugin will attempt to authenticate. -1. Assumed credentials via STS if `role_arn` attribute is specified (source credentials are evaluated from subsequent rules) -2. Explicit credentials from `access_key`, `secret_key`, and `token` attributes -3. Shared profile from `profile` attribute -4. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables) -5. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file) -6. [EC2 Instance Profile](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) +1. Web identity provider credentials via STS if `role_arn` and `web_identity_token_file` are specified +2. Assumed credentials via STS if `role_arn` attribute is specified (source credentials are evaluated from subsequent rules) +3. Explicit credentials from `access_key`, `secret_key`, and `token` attributes +4. Shared profile from `profile` attribute +5. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables) +6. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file) +7. [EC2 Instance Profile](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) The IAM user needs the following permissions ( https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/permissions-reference-cwl.html): - `logs:DescribeLogGroups` - required for check if configured log group exist @@ -34,16 +35,19 @@ The IAM user needs the following permissions ( https://docs.aws.amazon.com/Amazo ## Amazon Credentials ## Credentials are loaded in the following order - ## 1) Assumed credentials via STS if role_arn is specified - ## 2) explicit credentials from 'access_key' and 'secret_key' - ## 3) shared profile from 'profile' - ## 4) environment variables - ## 5) shared credentials file - ## 6) EC2 Instance Profile + ## 1) Web identity provider credentials via STS if role_arn and web_identity_token_file are specified + ## 2) Assumed credentials via STS if role_arn is specified + ## 3) explicit credentials from 'access_key' and 'secret_key' + ## 4) shared profile from 'profile' + ## 5) environment variables + ## 6) shared credentials file + ## 7) EC2 Instance Profile #access_key = "" #secret_key = "" #token = "" #role_arn = "" + #web_identity_token_file = "" + #role_session_name = "" #profile = "" #shared_credential_file = "" diff --git a/plugins/outputs/cloudwatch_logs/cloudwatch_logs.go b/plugins/outputs/cloudwatch_logs/cloudwatch_logs.go index d1d96b0b3..79eb5d772 100644 --- a/plugins/outputs/cloudwatch_logs/cloudwatch_logs.go +++ b/plugins/outputs/cloudwatch_logs/cloudwatch_logs.go @@ -33,15 +33,6 @@ type cloudWatchLogs interface { // CloudWatchLogs plugin object definition type CloudWatchLogs struct { - Region string `toml:"region"` - AccessKey string `toml:"access_key"` - SecretKey string `toml:"secret_key"` - RoleARN string `toml:"role_arn"` - Profile string `toml:"profile"` - Filename string `toml:"shared_credential_file"` - Token string `toml:"token"` - EndpointURL string `toml:"endpoint_url"` - LogGroup string `toml:"log_group"` lg *cloudwatchlogs.LogGroup //log group data @@ -59,6 +50,8 @@ type CloudWatchLogs struct { svc cloudWatchLogs //cloudwatch logs service Log telegraf.Logger `toml:"-"` + + internalaws.CredentialConfig } const ( @@ -91,16 +84,19 @@ region = "us-east-1" ## Amazon Credentials ## Credentials are loaded in the following order -## 1) Assumed credentials via STS if role_arn is specified -## 2) explicit credentials from 'access_key' and 'secret_key' -## 3) shared profile from 'profile' -## 4) environment variables -## 5) shared credentials file -## 6) EC2 Instance Profile +## 1) Web identity provider credentials via STS if role_arn and web_identity_token_file are specified +## 2) Assumed credentials via STS if role_arn is specified +## 3) explicit credentials from 'access_key' and 'secret_key' +## 4) shared profile from 'profile' +## 5) environment variables +## 6) shared credentials file +## 7) EC2 Instance Profile #access_key = "" #secret_key = "" #token = "" #role_arn = "" +#web_identity_token_file = "" +#role_session_name = "" #profile = "" #shared_credential_file = "" @@ -191,19 +187,7 @@ func (c *CloudWatchLogs) Connect() error { var logGroupsOutput = &cloudwatchlogs.DescribeLogGroupsOutput{NextToken: &dummyToken} var err error - credentialConfig := &internalaws.CredentialConfig{ - Region: c.Region, - AccessKey: c.AccessKey, - SecretKey: c.SecretKey, - RoleARN: c.RoleARN, - Profile: c.Profile, - Filename: c.Filename, - Token: c.Token, - EndpointURL: c.EndpointURL, - } - configProvider := credentialConfig.Credentials() - - c.svc = cloudwatchlogs.New(configProvider) + c.svc = cloudwatchlogs.New(c.CredentialConfig.Credentials()) if c.svc == nil { return fmt.Errorf("can't create cloudwatch logs service endpoint") } diff --git a/plugins/outputs/cloudwatch_logs/cloudwatch_logs_test.go b/plugins/outputs/cloudwatch_logs/cloudwatch_logs_test.go index 66378969f..f2f9f0cac 100644 --- a/plugins/outputs/cloudwatch_logs/cloudwatch_logs_test.go +++ b/plugins/outputs/cloudwatch_logs/cloudwatch_logs_test.go @@ -10,6 +10,7 @@ import ( "github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/influxdata/telegraf" + internalaws "github.com/influxdata/telegraf/config/aws" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" ) @@ -82,9 +83,11 @@ func TestInit(t *testing.T) { name: "log group is not set", expectedErrorString: "log group is not set", plugin: &CloudWatchLogs{ - Region: "eu-central-1", - AccessKey: "dummy", - SecretKey: "dummy", + CredentialConfig: internalaws.CredentialConfig{ + Region: "eu-central-1", + AccessKey: "dummy", + SecretKey: "dummy", + }, LogGroup: "", LogStream: "tag:source", LDMetricName: "docker_log", @@ -98,9 +101,11 @@ func TestInit(t *testing.T) { name: "log stream is not set", expectedErrorString: "log stream is not set", plugin: &CloudWatchLogs{ - Region: "eu-central-1", - AccessKey: "dummy", - SecretKey: "dummy", + CredentialConfig: internalaws.CredentialConfig{ + Region: "eu-central-1", + AccessKey: "dummy", + SecretKey: "dummy", + }, LogGroup: "TestLogGroup", LogStream: "", LDMetricName: "docker_log", @@ -114,9 +119,11 @@ func TestInit(t *testing.T) { name: "log data metrics name is not set", expectedErrorString: "log data metrics name is not set", plugin: &CloudWatchLogs{ - Region: "eu-central-1", - AccessKey: "dummy", - SecretKey: "dummy", + CredentialConfig: internalaws.CredentialConfig{ + Region: "eu-central-1", + AccessKey: "dummy", + SecretKey: "dummy", + }, LogGroup: "TestLogGroup", LogStream: "tag:source", LDMetricName: "", @@ -130,9 +137,11 @@ func TestInit(t *testing.T) { name: "log data source is not set", expectedErrorString: "log data source is not set", plugin: &CloudWatchLogs{ - Region: "eu-central-1", - AccessKey: "dummy", - SecretKey: "dummy", + CredentialConfig: internalaws.CredentialConfig{ + Region: "eu-central-1", + AccessKey: "dummy", + SecretKey: "dummy", + }, LogGroup: "TestLogGroup", LogStream: "tag:source", LDMetricName: "docker_log", @@ -147,9 +156,11 @@ func TestInit(t *testing.T) { expectedErrorString: "log data source is not properly formatted, ':' is missed.\n" + "Should be 'tag:' or 'field:'", plugin: &CloudWatchLogs{ - Region: "eu-central-1", - AccessKey: "dummy", - SecretKey: "dummy", + CredentialConfig: internalaws.CredentialConfig{ + Region: "eu-central-1", + AccessKey: "dummy", + SecretKey: "dummy", + }, LogGroup: "TestLogGroup", LogStream: "tag:source", LDMetricName: "docker_log", @@ -164,9 +175,11 @@ func TestInit(t *testing.T) { expectedErrorString: "log data source is not properly formatted.\n" + "Should be 'tag:' or 'field:'", plugin: &CloudWatchLogs{ - Region: "eu-central-1", - AccessKey: "dummy", - SecretKey: "dummy", + CredentialConfig: internalaws.CredentialConfig{ + Region: "eu-central-1", + AccessKey: "dummy", + SecretKey: "dummy", + }, LogGroup: "TestLogGroup", LogStream: "tag:source", LDMetricName: "docker_log", @@ -179,9 +192,11 @@ func TestInit(t *testing.T) { { name: "valid config", plugin: &CloudWatchLogs{ - Region: "eu-central-1", - AccessKey: "dummy", - SecretKey: "dummy", + CredentialConfig: internalaws.CredentialConfig{ + Region: "eu-central-1", + AccessKey: "dummy", + SecretKey: "dummy", + }, LogGroup: "TestLogGroup", LogStream: "tag:source", LDMetricName: "docker_log", @@ -225,10 +240,12 @@ func TestConnect(t *testing.T) { defer ts.Close() plugin := &CloudWatchLogs{ - Region: "eu-central-1", - AccessKey: "dummy", - SecretKey: "dummy", - EndpointURL: ts.URL, + CredentialConfig: internalaws.CredentialConfig{ + Region: "eu-central-1", + AccessKey: "dummy", + SecretKey: "dummy", + EndpointURL: ts.URL, + }, LogGroup: "TestLogGroup", LogStream: "tag:source", LDMetricName: "docker_log", @@ -263,10 +280,12 @@ func TestWrite(t *testing.T) { defer ts.Close() plugin := &CloudWatchLogs{ - Region: "eu-central-1", - AccessKey: "dummy", - SecretKey: "dummy", - EndpointURL: ts.URL, + CredentialConfig: internalaws.CredentialConfig{ + Region: "eu-central-1", + AccessKey: "dummy", + SecretKey: "dummy", + EndpointURL: ts.URL, + }, LogGroup: "TestLogGroup", LogStream: "tag:source", LDMetricName: "docker_log", diff --git a/plugins/outputs/kinesis/README.md b/plugins/outputs/kinesis/README.md index 1931dacb9..2d909090b 100644 --- a/plugins/outputs/kinesis/README.md +++ b/plugins/outputs/kinesis/README.md @@ -13,12 +13,16 @@ maybe useful for users to review Amazons official documentation which is availab This plugin uses a credential chain for Authentication with the Kinesis API endpoint. In the following order the plugin will attempt to authenticate. -1. Assumed credentials via STS if `role_arn` attribute is specified (source credentials are evaluated from subsequent rules) -2. Explicit credentials from `access_key`, `secret_key`, and `token` attributes -3. Shared profile from `profile` attribute -4. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables) -5. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file) -6. [EC2 Instance Profile](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) +1. Web identity provider credentials via STS if `role_arn` and `web_identity_token_file` are specified +2. Assumed credentials via STS if `role_arn` attribute is specified (source credentials are evaluated from subsequent rules) +3. Explicit credentials from `access_key`, `secret_key`, and `token` attributes +4. Shared profile from `profile` attribute +5. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables) +6. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file) +7. [EC2 Instance Profile](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) + +If you are using credentials from a web identity provider, you can specify the session name using `role_session_name`. If +left empty, the current timestamp will be used. ## Config diff --git a/plugins/outputs/kinesis/kinesis.go b/plugins/outputs/kinesis/kinesis.go index 333214614..2e7578840 100644 --- a/plugins/outputs/kinesis/kinesis.go +++ b/plugins/outputs/kinesis/kinesis.go @@ -18,15 +18,6 @@ const maxRecordsPerRequest uint32 = 500 type ( KinesisOutput struct { - Region string `toml:"region"` - AccessKey string `toml:"access_key"` - SecretKey string `toml:"secret_key"` - RoleARN string `toml:"role_arn"` - Profile string `toml:"profile"` - Filename string `toml:"shared_credential_file"` - Token string `toml:"token"` - EndpointURL string `toml:"endpoint_url"` - StreamName string `toml:"streamname"` PartitionKey string `toml:"partitionkey"` RandomPartitionKey bool `toml:"use_random_partitionkey"` @@ -36,6 +27,8 @@ type ( Log telegraf.Logger `toml:"-"` serializer serializers.Serializer svc kinesisiface.KinesisAPI + + internalaws.CredentialConfig } Partition struct { @@ -51,16 +44,19 @@ var sampleConfig = ` ## Amazon Credentials ## Credentials are loaded in the following order - ## 1) Assumed credentials via STS if role_arn is specified - ## 2) explicit credentials from 'access_key' and 'secret_key' - ## 3) shared profile from 'profile' - ## 4) environment variables - ## 5) shared credentials file - ## 6) EC2 Instance Profile + ## 1) Web identity provider credentials via STS if role_arn and web_identity_token_file are specified + ## 2) Assumed credentials via STS if role_arn is specified + ## 3) explicit credentials from 'access_key' and 'secret_key' + ## 4) shared profile from 'profile' + ## 5) environment variables + ## 6) shared credentials file + ## 7) EC2 Instance Profile #access_key = "" #secret_key = "" #token = "" #role_arn = "" + #web_identity_token_file = "" + #role_session_name = "" #profile = "" #shared_credential_file = "" @@ -130,18 +126,7 @@ func (k *KinesisOutput) Connect() error { k.Log.Infof("Establishing a connection to Kinesis in %s", k.Region) } - credentialConfig := &internalaws.CredentialConfig{ - Region: k.Region, - AccessKey: k.AccessKey, - SecretKey: k.SecretKey, - RoleARN: k.RoleARN, - Profile: k.Profile, - Filename: k.Filename, - Token: k.Token, - EndpointURL: k.EndpointURL, - } - configProvider := credentialConfig.Credentials() - svc := kinesis.New(configProvider) + svc := kinesis.New(k.CredentialConfig.Credentials()) _, err := svc.DescribeStreamSummary(&kinesis.DescribeStreamSummaryInput{ StreamName: aws.String(k.StreamName), diff --git a/plugins/outputs/timestream/README.md b/plugins/outputs/timestream/README.md index 4c35df04e..dc063a068 100644 --- a/plugins/outputs/timestream/README.md +++ b/plugins/outputs/timestream/README.md @@ -11,17 +11,20 @@ The Timestream output plugin writes metrics to the [Amazon Timestream] service. region = "us-east-1" ## Amazon Credentials - ## Credentials are loaded in the following order: - ## 1) Assumed credentials via STS if role_arn is specified - ## 2) Explicit credentials from 'access_key' and 'secret_key' - ## 3) Shared profile from 'profile' - ## 4) Environment variables - ## 5) Shared credentials file - ## 6) EC2 Instance Profile + ## Credentials are loaded in the following order + ## 1) Web identity provider credentials via STS if role_arn and web_identity_token_file are specified + ## 2) Assumed credentials via STS if role_arn is specified + ## 3) explicit credentials from 'access_key' and 'secret_key' + ## 4) shared profile from 'profile' + ## 5) environment variables + ## 6) shared credentials file + ## 7) EC2 Instance Profile #access_key = "" #secret_key = "" #token = "" #role_arn = "" + #web_identity_token_file = "" + #role_session_name = "" #profile = "" #shared_credential_file = "" diff --git a/plugins/outputs/timestream/timestream.go b/plugins/outputs/timestream/timestream.go index 8af7c5665..2c77c408e 100644 --- a/plugins/outputs/timestream/timestream.go +++ b/plugins/outputs/timestream/timestream.go @@ -19,15 +19,6 @@ import ( type ( Timestream struct { - Region string `toml:"region"` - AccessKey string `toml:"access_key"` - SecretKey string `toml:"secret_key"` - RoleARN string `toml:"role_arn"` - Profile string `toml:"profile"` - Filename string `toml:"shared_credential_file"` - Token string `toml:"token"` - EndpointURL string `toml:"endpoint_url"` - MappingMode string `toml:"mapping_mode"` DescribeDatabaseOnStart bool `toml:"describe_database_on_start"` DatabaseName string `toml:"database_name"` @@ -42,6 +33,8 @@ type ( Log telegraf.Logger svc WriteClient + + internalaws.CredentialConfig } WriteClient interface { @@ -67,16 +60,19 @@ var sampleConfig = ` ## Amazon Credentials ## Credentials are loaded in the following order: - ## 1) Assumed credentials via STS if role_arn is specified - ## 2) Explicit credentials from 'access_key' and 'secret_key' - ## 3) Shared profile from 'profile' - ## 4) Environment variables - ## 5) Shared credentials file - ## 6) EC2 Instance Profile + ## 1) Web identity provider credentials via STS if role_arn and web_identity_token_file are specified + ## 2) Assumed credentials via STS if role_arn is specified + ## 3) explicit credentials from 'access_key' and 'secret_key' + ## 4) shared profile from 'profile' + ## 5) environment variables + ## 6) shared credentials file + ## 7) EC2 Instance Profile #access_key = "" #secret_key = "" #token = "" #role_arn = "" + #web_identity_token_file = "" + #role_session_name = "" #profile = "" #shared_credential_file = "" @@ -225,17 +221,7 @@ func (t *Timestream) Connect() error { t.Log.Infof("Constructing Timestream client for '%s' mode", t.MappingMode) - credentialConfig := &internalaws.CredentialConfig{ - Region: t.Region, - AccessKey: t.AccessKey, - SecretKey: t.SecretKey, - RoleARN: t.RoleARN, - Profile: t.Profile, - Filename: t.Filename, - Token: t.Token, - EndpointURL: t.EndpointURL, - } - svc := WriteFactory(credentialConfig) + svc := WriteFactory(&t.CredentialConfig) if t.DescribeDatabaseOnStart { t.Log.Infof("Describing database '%s' in region '%s'", t.DatabaseName, t.Region)