From 7c636b4b6b38e7da4446a48854b5fc55b8b78104 Mon Sep 17 00:00:00 2001 From: Neelay Upadhyaya Date: Wed, 24 May 2023 13:17:25 +0530 Subject: [PATCH] feat(config): Support shell like syntax for environment variable substitution (#13229) --- config/config.go | 105 ++++++++++++++------ config/config_test.go | 98 +++++++++++++++++- config/testdata/single_plugin_env_vars.toml | 16 ++- docs/CONFIGURATION.md | 11 +- docs/LICENSE_OF_DEPENDENCIES.md | 1 + go.mod | 3 +- go.sum | 9 +- 7 files changed, 203 insertions(+), 40 deletions(-) diff --git a/config/config.go b/config/config.go index 07094f9b7..9958df064 100644 --- a/config/config.go +++ b/config/config.go @@ -20,6 +20,8 @@ import ( "sync" "time" + "github.com/compose-spec/compose-go/template" + "github.com/compose-spec/compose-go/utils" "github.com/coreos/go-semver/semver" "github.com/influxdata/toml" "github.com/influxdata/toml/ast" @@ -39,13 +41,6 @@ import ( ) var ( - // envVarRe is a regex to find environment variables in the config file - envVarRe = regexp.MustCompile(`\${(\w+)}|\$(\w+)`) - - envVarEscaper = strings.NewReplacer( - `"`, `\"`, - `\`, `\\`, - ) httpLoadConfigRetryInterval = 10 * time.Second // fetchURLRe is a regex to determine whether the requested file should @@ -697,11 +692,6 @@ func trimBOM(f []byte) []byte { return bytes.TrimPrefix(f, []byte("\xef\xbb\xbf")) } -// escapeEnv escapes a value for inserting into a TOML string. -func escapeEnv(value string) string { - return envVarEscaper.Replace(value) -} - func LoadConfigFile(config string) ([]byte, error) { if fetchURLRe.MatchString(config) { u, err := url.Parse(config) @@ -782,30 +772,87 @@ func fetchConfig(u *url.URL) ([]byte, error) { // will find environment variables and replace them. func parseConfig(contents []byte) (*ast.Table, error) { contents = trimBOM(contents) + var err error + contents, err = removeComments(contents) + if err != nil { + return nil, err + } + outputBytes, err := substituteEnvironment(contents) + if err != nil { + return nil, err + } + return toml.Parse(outputBytes) +} - parameters := envVarRe.FindAllSubmatch(contents, -1) - for _, parameter := range parameters { - if len(parameter) != 3 { - continue +func removeComments(contents []byte) ([]byte, error) { + tomlReader := bytes.NewReader(contents) + + // Initialize variables for tracking state + var inQuote, inComment bool + var quoteChar, prevChar byte + + // Initialize buffer for modified TOML data + var output bytes.Buffer + + buf := make([]byte, 1) + // Iterate over each character in the file + for { + _, err := tomlReader.Read(buf) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err } + char := buf[0] - var envVar []byte - if parameter[1] != nil { - envVar = parameter[1] - } else if parameter[2] != nil { - envVar = parameter[2] + if inComment { + // If we're currently in a comment, check if this character ends the comment + if char == '\n' { + // End of line, comment is finished + inComment = false + _, _ = output.WriteRune('\n') + } + } else if inQuote { + // If we're currently in a quote, check if this character ends the quote + if char == quoteChar && prevChar != '\\' { + // End of quote, we're no longer in a quote + inQuote = false + } + output.WriteByte(char) } else { - continue - } - - envVal, ok := os.LookupEnv(strings.TrimPrefix(string(envVar), "$")) - if ok { - envVal = escapeEnv(envVal) - contents = bytes.Replace(contents, parameter[0], []byte(envVal), 1) + // Not in a comment or a quote + if char == '"' || char == '\'' { + // Start of quote + inQuote = true + quoteChar = char + output.WriteByte(char) + } else if char == '#' { + // Start of comment + inComment = true + } else { + // Not a comment or a quote, just output the character + output.WriteByte(char) + } + prevChar = char } } + return output.Bytes(), nil +} - return toml.Parse(contents) +func substituteEnvironment(contents []byte) ([]byte, error) { + envMap := utils.GetAsEqualsMap(os.Environ()) + retVal, err := template.Substitute(string(contents), func(k string) (string, bool) { + if v, ok := envMap[k]; ok { + return v, ok + } + return "", false + }) + var invalidTmplError *template.InvalidTemplateError + if err != nil && !errors.As(err, &invalidTmplError) { + return nil, err + } + return []byte(retVal), nil } func (c *Config) addAggregator(name string, table *ast.Table) error { diff --git a/config/config_test.go b/config/config_test.go index fff9c5911..a6c5de9fc 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -70,6 +70,8 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) { input := inputs.Inputs["memcached"]().(*MockupInputPlugin) input.Servers = []string{"192.168.1.1"} + input.Command = `Raw command which may or may not contain # in it +# is unique` filter := models.Filter{ NameDrop: []string{"metricname2"}, @@ -85,7 +87,7 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) { TagPassFilters: []models.TagFilter{ { Name: "goodtag", - Values: []string{"mytag"}, + Values: []string{"mytag", "tagwith#value", "TagWithMultilineSyntax"}, }, }, } @@ -105,6 +107,96 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) { require.Equal(t, inputConfig, c.Inputs[0].Config, "Testdata did not produce correct input metadata.") } +func Test_envSub(t *testing.T) { + tests := []struct { + name string + setEnv func(*testing.T) + contents string + expected string + wantErr bool + errSubstring string + }{ + { + name: "Legacy with ${} and without {}", + setEnv: func(t *testing.T) { + t.Setenv("TEST_ENV1", "VALUE1") + t.Setenv("TEST_ENV2", "VALUE2") + }, + contents: "A string with ${TEST_ENV1}, $TEST_ENV2 and $TEST_ENV1 as repeated", + expected: "A string with VALUE1, VALUE2 and VALUE1 as repeated", + }, + { + name: "Env not set", + contents: "Env variable ${NOT_SET} will be empty", + expected: "Env variable will be empty", // Two spaces present + }, + { + name: "Env not set, fallback to default", + contents: "Env variable ${THIS_IS_ABSENT:-Fallback}", + expected: "Env variable Fallback", + }, + { + name: "No fallback", + setEnv: func(t *testing.T) { + t.Setenv("MY_ENV1", "VALUE1") + }, + contents: "Env variable ${MY_ENV1:-Fallback}", + expected: "Env variable VALUE1", + }, + { + name: "Mix and match", + setEnv: func(t *testing.T) { + t.Setenv("MY_VAR", "VALUE") + t.Setenv("MY_VAR2", "VALUE2") + }, + contents: "Env var ${MY_VAR} is set, with $MY_VAR syntax and default on this ${MY_VAR1:-Substituted}, no default on this ${MY_VAR2:-NoDefault}", + expected: "Env var VALUE is set, with VALUE syntax and default on this Substituted, no default on this VALUE2", + }, + { + name: "Default has special chars", + contents: `Not recommended but supported ${MY_VAR:-Default with special chars Supported#$\"}`, + expected: `Not recommended but supported Default with special chars Supported#$\"`, // values are escaped + }, + { + name: "unset error", + contents: "Contains ${THIS_IS_NOT_SET?unset-error}", + wantErr: true, + errSubstring: "unset-error", + }, + { + name: "env empty error", + setEnv: func(t *testing.T) { + t.Setenv("ENV_EMPTY", "") + }, + contents: "Contains ${ENV_EMPTY:?empty-error}", + wantErr: true, + errSubstring: "empty-error", + }, + { + name: "Fallback as env variable", + setEnv: func(t *testing.T) { + t.Setenv("FALLBACK", "my-fallback") + }, + contents: "Should output ${NOT_SET:-${FALLBACK}}", + expected: "Should output my-fallback", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setEnv != nil { + tt.setEnv(t) + } + actual, err := substituteEnvironment([]byte(tt.contents)) + if tt.wantErr { + require.ErrorContains(t, err, tt.errSubstring) + return + } + require.EqualValues(t, tt.expected, string(actual)) + }) + } +} + func TestConfig_LoadSingleInput(t *testing.T) { c := NewConfig() require.NoError(t, c.LoadConfig("./testdata/single_plugin.toml")) @@ -399,13 +491,15 @@ func TestConfig_WrongFieldType(t *testing.T) { func TestConfig_InlineTables(t *testing.T) { // #4098 + t.Setenv("TOKEN", "test") + c := NewConfig() require.NoError(t, c.LoadConfig("./testdata/inline_table.toml")) require.Len(t, c.Outputs, 2) output, ok := c.Outputs[1].Output.(*MockupOuputPlugin) require.True(t, ok) - require.Equal(t, map[string]string{"Authorization": "Token $TOKEN", "Content-Type": "application/json"}, output.Headers) + require.Equal(t, map[string]string{"Authorization": "Token test", "Content-Type": "application/json"}, output.Headers) require.Equal(t, []string{"org_id"}, c.Outputs[0].Config.Filter.TagInclude) } diff --git a/config/testdata/single_plugin_env_vars.toml b/config/testdata/single_plugin_env_vars.toml index de7c47cf7..903bdab3e 100644 --- a/config/testdata/single_plugin_env_vars.toml +++ b/config/testdata/single_plugin_env_vars.toml @@ -14,13 +14,23 @@ # for numbers and booleans they should be plain (ie, ${INT_VAR}, ${BOOL_VAR}) [[inputs.memcached]] - servers = ["$MY_TEST_SERVER"] - namepass = ["metricname1", "ip_${MY_TEST_SERVER}_name"] + # this comment line will be ignored by the parser + servers = ["$MY_TEST_SERVER"] + namepass = ["metricname1", "ip_${MY_TEST_SERVER}_name"] # this comment will be ignored as well namedrop = ["metricname2"] fieldpass = ["some", "strings"] fielddrop = ["other", "stuff"] interval = "$TEST_INTERVAL" + ##### this input is provided to test multiline strings + command = """ +Raw command which may or may not contain # in it +# is unique""" # Multiline comment black starting with # [inputs.memcached.tagpass] - goodtag = ["mytag"] + goodtag = ["mytag", """tagwith#value""", + # comment in between array items + # should ignore "quotes" in comments + '''TagWithMultilineSyntax''', ## ignore this comment + ] # hastag [inputs.memcached.tagdrop] badtag = ["othertag"] + \ No newline at end of file diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index ed4fff6c1..7f93f6f6b 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -54,10 +54,19 @@ configuration files. ## Environment Variables Environment variables can be used anywhere in the config file, simply surround -them with `${}`. Replacement occurs before file parsing. For strings +them with `${}`. Replacement occurs before file parsing. For strings the variable must be within quotes, e.g., `"${STR_VAR}"`, for numbers and booleans they should be unquoted, e.g., `${INT_VAR}`, `${BOOL_VAR}`. +In addition to this, Telegraf also supports Shell parameter expansion for environment variables +which allows syntax such as: + +- `${VARIABLE:-default}` evaluates to default if VARIABLE is unset or empty in the environment. +- `${VARIABLE-default}` evaluates to default only if VARIABLE is unset in the environment. +Similarly, the following syntax allows you to specify mandatory variables: +- `${VARIABLE:?err}` exits with an error message containing err if VARIABLE is unset or empty in the environment. +- `${VARIABLE?err}` exits with an error message containing err if VARIABLE is unset in the environment. + When using the `.deb` or `.rpm` packages, you can define environment variables in the `/etc/default/telegraf` file. diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index dd13ced12..6fce9c446 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -89,6 +89,7 @@ following works: - github.com/cisco-ie/nx-telemetry-proto [Apache License 2.0](https://github.com/cisco-ie/nx-telemetry-proto/blob/master/LICENSE) - github.com/clarify/clarify-go [Apache License 2.0](https://github.com/clarify/clarify-go/blob/master/LICENSE) - github.com/cloudevents/sdk-go [Apache License 2.0](https://github.com/cloudevents/sdk-go/blob/main/LICENSE) +- github.com/compose-spec/compose-go [Apache License 2.0](https://github.com/compose-spec/compose-go/blob/master/LICENSE) - github.com/containerd/containerd [Apache License 2.0](https://github.com/containerd/containerd/blob/master/LICENSE) - github.com/coocood/freecache [MIT License](https://github.com/coocood/freecache/blob/master/LICENSE) - github.com/coreos/go-semver [Apache License 2.0](https://github.com/coreos/go-semver/blob/main/LICENSE) diff --git a/go.mod b/go.mod index 511b25a91..d416faf3f 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/caio/go-tdigest v3.1.0+incompatible github.com/cisco-ie/nx-telemetry-proto v0.0.0-20230117155933-f64c045c77df github.com/clarify/clarify-go v0.2.4 + github.com/compose-spec/compose-go v1.13.4 github.com/coocood/freecache v1.2.3 github.com/coreos/go-semver v0.3.1 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f @@ -325,7 +326,7 @@ require ( github.com/hashicorp/packer-plugin-sdk v0.3.1 // indirect github.com/hashicorp/serf v0.10.1 // indirect github.com/huandu/xstrings v1.3.2 // indirect - github.com/imdario/mergo v0.3.13 // indirect + github.com/imdario/mergo v0.3.15 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.2 // indirect diff --git a/go.sum b/go.sum index bfae871df..a395b6e46 100644 --- a/go.sum +++ b/go.sum @@ -417,6 +417,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/compose-spec/compose-go v1.13.4 h1:O6xAsPqaY1s9KXteiO7wRCDTJLahv1XP/z/eUO9EfbI= +github.com/compose-spec/compose-go v1.13.4/go.mod h1:rsiZ8uaOHJYJemDBzTe9UBpaq5ZFVEOO4TxM2G3SJxk= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns= github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= @@ -864,8 +866,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= +github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/go-syslog/v3 v3.0.0 h1:jichmjSZlYK0VMmlz+k4WeOQd7z745YLsvGMqwtYt4I= github.com/influxdata/go-syslog/v3 v3.0.0/go.mod h1:tulsOp+CecTAYC27u9miMgq21GqXRW6VdKbOG+QSP4Q= @@ -2275,10 +2277,9 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=