feat(config): Support shell like syntax for environment variable substitution (#13229)
This commit is contained in:
Neelay Upadhyaya
GitHub
parent
2010926e25
commit
7c636b4b6b
105
config/config.go
105
config/config.go
|
|
@ -20,6 +20,8 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/compose-spec/compose-go/template"
|
||||
"github.com/compose-spec/compose-go/utils"
|
||||
"github.com/coreos/go-semver/semver"
|
||||
"github.com/influxdata/toml"
|
||||
"github.com/influxdata/toml/ast"
|
||||
|
|
@ -39,13 +41,6 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
// envVarRe is a regex to find environment variables in the config file
|
||||
envVarRe = regexp.MustCompile(`\${(\w+)}|\$(\w+)`)
|
||||
|
||||
envVarEscaper = strings.NewReplacer(
|
||||
`"`, `\"`,
|
||||
`\`, `\\`,
|
||||
)
|
||||
httpLoadConfigRetryInterval = 10 * time.Second
|
||||
|
||||
// fetchURLRe is a regex to determine whether the requested file should
|
||||
|
|
@ -697,11 +692,6 @@ func trimBOM(f []byte) []byte {
|
|||
return bytes.TrimPrefix(f, []byte("\xef\xbb\xbf"))
|
||||
}
|
||||
|
||||
// escapeEnv escapes a value for inserting into a TOML string.
|
||||
func escapeEnv(value string) string {
|
||||
return envVarEscaper.Replace(value)
|
||||
}
|
||||
|
||||
func LoadConfigFile(config string) ([]byte, error) {
|
||||
if fetchURLRe.MatchString(config) {
|
||||
u, err := url.Parse(config)
|
||||
|
|
@ -782,30 +772,87 @@ func fetchConfig(u *url.URL) ([]byte, error) {
|
|||
// will find environment variables and replace them.
|
||||
func parseConfig(contents []byte) (*ast.Table, error) {
|
||||
contents = trimBOM(contents)
|
||||
var err error
|
||||
contents, err = removeComments(contents)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outputBytes, err := substituteEnvironment(contents)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return toml.Parse(outputBytes)
|
||||
}
|
||||
|
||||
parameters := envVarRe.FindAllSubmatch(contents, -1)
|
||||
for _, parameter := range parameters {
|
||||
if len(parameter) != 3 {
|
||||
continue
|
||||
func removeComments(contents []byte) ([]byte, error) {
|
||||
tomlReader := bytes.NewReader(contents)
|
||||
|
||||
// Initialize variables for tracking state
|
||||
var inQuote, inComment bool
|
||||
var quoteChar, prevChar byte
|
||||
|
||||
// Initialize buffer for modified TOML data
|
||||
var output bytes.Buffer
|
||||
|
||||
buf := make([]byte, 1)
|
||||
// Iterate over each character in the file
|
||||
for {
|
||||
_, err := tomlReader.Read(buf)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
char := buf[0]
|
||||
|
||||
var envVar []byte
|
||||
if parameter[1] != nil {
|
||||
envVar = parameter[1]
|
||||
} else if parameter[2] != nil {
|
||||
envVar = parameter[2]
|
||||
if inComment {
|
||||
// If we're currently in a comment, check if this character ends the comment
|
||||
if char == '\n' {
|
||||
// End of line, comment is finished
|
||||
inComment = false
|
||||
_, _ = output.WriteRune('\n')
|
||||
}
|
||||
} else if inQuote {
|
||||
// If we're currently in a quote, check if this character ends the quote
|
||||
if char == quoteChar && prevChar != '\\' {
|
||||
// End of quote, we're no longer in a quote
|
||||
inQuote = false
|
||||
}
|
||||
output.WriteByte(char)
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
|
||||
envVal, ok := os.LookupEnv(strings.TrimPrefix(string(envVar), "$"))
|
||||
if ok {
|
||||
envVal = escapeEnv(envVal)
|
||||
contents = bytes.Replace(contents, parameter[0], []byte(envVal), 1)
|
||||
// Not in a comment or a quote
|
||||
if char == '"' || char == '\'' {
|
||||
// Start of quote
|
||||
inQuote = true
|
||||
quoteChar = char
|
||||
output.WriteByte(char)
|
||||
} else if char == '#' {
|
||||
// Start of comment
|
||||
inComment = true
|
||||
} else {
|
||||
// Not a comment or a quote, just output the character
|
||||
output.WriteByte(char)
|
||||
}
|
||||
prevChar = char
|
||||
}
|
||||
}
|
||||
return output.Bytes(), nil
|
||||
}
|
||||
|
||||
return toml.Parse(contents)
|
||||
func substituteEnvironment(contents []byte) ([]byte, error) {
|
||||
envMap := utils.GetAsEqualsMap(os.Environ())
|
||||
retVal, err := template.Substitute(string(contents), func(k string) (string, bool) {
|
||||
if v, ok := envMap[k]; ok {
|
||||
return v, ok
|
||||
}
|
||||
return "", false
|
||||
})
|
||||
var invalidTmplError *template.InvalidTemplateError
|
||||
if err != nil && !errors.As(err, &invalidTmplError) {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(retVal), nil
|
||||
}
|
||||
|
||||
func (c *Config) addAggregator(name string, table *ast.Table) error {
|
||||
|
|
|
|||
|
|
@ -70,6 +70,8 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
|
|||
|
||||
input := inputs.Inputs["memcached"]().(*MockupInputPlugin)
|
||||
input.Servers = []string{"192.168.1.1"}
|
||||
input.Command = `Raw command which may or may not contain # in it
|
||||
# is unique`
|
||||
|
||||
filter := models.Filter{
|
||||
NameDrop: []string{"metricname2"},
|
||||
|
|
@ -85,7 +87,7 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
|
|||
TagPassFilters: []models.TagFilter{
|
||||
{
|
||||
Name: "goodtag",
|
||||
Values: []string{"mytag"},
|
||||
Values: []string{"mytag", "tagwith#value", "TagWithMultilineSyntax"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
@ -105,6 +107,96 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
|
|||
require.Equal(t, inputConfig, c.Inputs[0].Config, "Testdata did not produce correct input metadata.")
|
||||
}
|
||||
|
||||
func Test_envSub(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setEnv func(*testing.T)
|
||||
contents string
|
||||
expected string
|
||||
wantErr bool
|
||||
errSubstring string
|
||||
}{
|
||||
{
|
||||
name: "Legacy with ${} and without {}",
|
||||
setEnv: func(t *testing.T) {
|
||||
t.Setenv("TEST_ENV1", "VALUE1")
|
||||
t.Setenv("TEST_ENV2", "VALUE2")
|
||||
},
|
||||
contents: "A string with ${TEST_ENV1}, $TEST_ENV2 and $TEST_ENV1 as repeated",
|
||||
expected: "A string with VALUE1, VALUE2 and VALUE1 as repeated",
|
||||
},
|
||||
{
|
||||
name: "Env not set",
|
||||
contents: "Env variable ${NOT_SET} will be empty",
|
||||
expected: "Env variable will be empty", // Two spaces present
|
||||
},
|
||||
{
|
||||
name: "Env not set, fallback to default",
|
||||
contents: "Env variable ${THIS_IS_ABSENT:-Fallback}",
|
||||
expected: "Env variable Fallback",
|
||||
},
|
||||
{
|
||||
name: "No fallback",
|
||||
setEnv: func(t *testing.T) {
|
||||
t.Setenv("MY_ENV1", "VALUE1")
|
||||
},
|
||||
contents: "Env variable ${MY_ENV1:-Fallback}",
|
||||
expected: "Env variable VALUE1",
|
||||
},
|
||||
{
|
||||
name: "Mix and match",
|
||||
setEnv: func(t *testing.T) {
|
||||
t.Setenv("MY_VAR", "VALUE")
|
||||
t.Setenv("MY_VAR2", "VALUE2")
|
||||
},
|
||||
contents: "Env var ${MY_VAR} is set, with $MY_VAR syntax and default on this ${MY_VAR1:-Substituted}, no default on this ${MY_VAR2:-NoDefault}",
|
||||
expected: "Env var VALUE is set, with VALUE syntax and default on this Substituted, no default on this VALUE2",
|
||||
},
|
||||
{
|
||||
name: "Default has special chars",
|
||||
contents: `Not recommended but supported ${MY_VAR:-Default with special chars Supported#$\"}`,
|
||||
expected: `Not recommended but supported Default with special chars Supported#$\"`, // values are escaped
|
||||
},
|
||||
{
|
||||
name: "unset error",
|
||||
contents: "Contains ${THIS_IS_NOT_SET?unset-error}",
|
||||
wantErr: true,
|
||||
errSubstring: "unset-error",
|
||||
},
|
||||
{
|
||||
name: "env empty error",
|
||||
setEnv: func(t *testing.T) {
|
||||
t.Setenv("ENV_EMPTY", "")
|
||||
},
|
||||
contents: "Contains ${ENV_EMPTY:?empty-error}",
|
||||
wantErr: true,
|
||||
errSubstring: "empty-error",
|
||||
},
|
||||
{
|
||||
name: "Fallback as env variable",
|
||||
setEnv: func(t *testing.T) {
|
||||
t.Setenv("FALLBACK", "my-fallback")
|
||||
},
|
||||
contents: "Should output ${NOT_SET:-${FALLBACK}}",
|
||||
expected: "Should output my-fallback",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.setEnv != nil {
|
||||
tt.setEnv(t)
|
||||
}
|
||||
actual, err := substituteEnvironment([]byte(tt.contents))
|
||||
if tt.wantErr {
|
||||
require.ErrorContains(t, err, tt.errSubstring)
|
||||
return
|
||||
}
|
||||
require.EqualValues(t, tt.expected, string(actual))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_LoadSingleInput(t *testing.T) {
|
||||
c := NewConfig()
|
||||
require.NoError(t, c.LoadConfig("./testdata/single_plugin.toml"))
|
||||
|
|
@ -399,13 +491,15 @@ func TestConfig_WrongFieldType(t *testing.T) {
|
|||
|
||||
func TestConfig_InlineTables(t *testing.T) {
|
||||
// #4098
|
||||
t.Setenv("TOKEN", "test")
|
||||
|
||||
c := NewConfig()
|
||||
require.NoError(t, c.LoadConfig("./testdata/inline_table.toml"))
|
||||
require.Len(t, c.Outputs, 2)
|
||||
|
||||
output, ok := c.Outputs[1].Output.(*MockupOuputPlugin)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, map[string]string{"Authorization": "Token $TOKEN", "Content-Type": "application/json"}, output.Headers)
|
||||
require.Equal(t, map[string]string{"Authorization": "Token test", "Content-Type": "application/json"}, output.Headers)
|
||||
require.Equal(t, []string{"org_id"}, c.Outputs[0].Config.Filter.TagInclude)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,13 +14,23 @@
|
|||
# for numbers and booleans they should be plain (ie, ${INT_VAR}, ${BOOL_VAR})
|
||||
|
||||
[[inputs.memcached]]
|
||||
servers = ["$MY_TEST_SERVER"]
|
||||
namepass = ["metricname1", "ip_${MY_TEST_SERVER}_name"]
|
||||
# this comment line will be ignored by the parser
|
||||
servers = ["$MY_TEST_SERVER"]
|
||||
namepass = ["metricname1", "ip_${MY_TEST_SERVER}_name"] # this comment will be ignored as well
|
||||
namedrop = ["metricname2"]
|
||||
fieldpass = ["some", "strings"]
|
||||
fielddrop = ["other", "stuff"]
|
||||
interval = "$TEST_INTERVAL"
|
||||
##### this input is provided to test multiline strings
|
||||
command = """
|
||||
Raw command which may or may not contain # in it
|
||||
# is unique""" # Multiline comment black starting with #
|
||||
[inputs.memcached.tagpass]
|
||||
goodtag = ["mytag"]
|
||||
goodtag = ["mytag", """tagwith#value""",
|
||||
# comment in between array items
|
||||
# should ignore "quotes" in comments
|
||||
'''TagWithMultilineSyntax''', ## ignore this comment
|
||||
] # hastag
|
||||
[inputs.memcached.tagdrop]
|
||||
badtag = ["othertag"]
|
||||
|
||||
|
|
@ -54,10 +54,19 @@ configuration files.
|
|||
## Environment Variables
|
||||
|
||||
Environment variables can be used anywhere in the config file, simply surround
|
||||
them with `${}`. Replacement occurs before file parsing. For strings
|
||||
them with `${}`. Replacement occurs before file parsing. For strings
|
||||
the variable must be within quotes, e.g., `"${STR_VAR}"`, for numbers and booleans
|
||||
they should be unquoted, e.g., `${INT_VAR}`, `${BOOL_VAR}`.
|
||||
|
||||
In addition to this, Telegraf also supports Shell parameter expansion for environment variables
|
||||
which allows syntax such as:
|
||||
|
||||
- `${VARIABLE:-default}` evaluates to default if VARIABLE is unset or empty in the environment.
|
||||
- `${VARIABLE-default}` evaluates to default only if VARIABLE is unset in the environment.
|
||||
Similarly, the following syntax allows you to specify mandatory variables:
|
||||
- `${VARIABLE:?err}` exits with an error message containing err if VARIABLE is unset or empty in the environment.
|
||||
- `${VARIABLE?err}` exits with an error message containing err if VARIABLE is unset in the environment.
|
||||
|
||||
When using the `.deb` or `.rpm` packages, you can define environment variables
|
||||
in the `/etc/default/telegraf` file.
|
||||
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ following works:
|
|||
- github.com/cisco-ie/nx-telemetry-proto [Apache License 2.0](https://github.com/cisco-ie/nx-telemetry-proto/blob/master/LICENSE)
|
||||
- github.com/clarify/clarify-go [Apache License 2.0](https://github.com/clarify/clarify-go/blob/master/LICENSE)
|
||||
- github.com/cloudevents/sdk-go [Apache License 2.0](https://github.com/cloudevents/sdk-go/blob/main/LICENSE)
|
||||
- github.com/compose-spec/compose-go [Apache License 2.0](https://github.com/compose-spec/compose-go/blob/master/LICENSE)
|
||||
- github.com/containerd/containerd [Apache License 2.0](https://github.com/containerd/containerd/blob/master/LICENSE)
|
||||
- github.com/coocood/freecache [MIT License](https://github.com/coocood/freecache/blob/master/LICENSE)
|
||||
- github.com/coreos/go-semver [Apache License 2.0](https://github.com/coreos/go-semver/blob/main/LICENSE)
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -54,6 +54,7 @@ require (
|
|||
github.com/caio/go-tdigest v3.1.0+incompatible
|
||||
github.com/cisco-ie/nx-telemetry-proto v0.0.0-20230117155933-f64c045c77df
|
||||
github.com/clarify/clarify-go v0.2.4
|
||||
github.com/compose-spec/compose-go v1.13.4
|
||||
github.com/coocood/freecache v1.2.3
|
||||
github.com/coreos/go-semver v0.3.1
|
||||
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
|
||||
|
|
@ -325,7 +326,7 @@ require (
|
|||
github.com/hashicorp/packer-plugin-sdk v0.3.1 // indirect
|
||||
github.com/hashicorp/serf v0.10.1 // indirect
|
||||
github.com/huandu/xstrings v1.3.2 // indirect
|
||||
github.com/imdario/mergo v0.3.13 // indirect
|
||||
github.com/imdario/mergo v0.3.15 // indirect
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgproto3/v2 v2.3.2 // indirect
|
||||
|
|
|
|||
9
go.sum
9
go.sum
|
|
@ -417,6 +417,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
|
|||
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
|
||||
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
|
||||
github.com/compose-spec/compose-go v1.13.4 h1:O6xAsPqaY1s9KXteiO7wRCDTJLahv1XP/z/eUO9EfbI=
|
||||
github.com/compose-spec/compose-go v1.13.4/go.mod h1:rsiZ8uaOHJYJemDBzTe9UBpaq5ZFVEOO4TxM2G3SJxk=
|
||||
github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
|
||||
github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns=
|
||||
github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw=
|
||||
|
|
@ -864,8 +866,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
|
|||
github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
|
||||
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
|
||||
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
|
||||
github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
|
||||
github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/influxdata/go-syslog/v3 v3.0.0 h1:jichmjSZlYK0VMmlz+k4WeOQd7z745YLsvGMqwtYt4I=
|
||||
github.com/influxdata/go-syslog/v3 v3.0.0/go.mod h1:tulsOp+CecTAYC27u9miMgq21GqXRW6VdKbOG+QSP4Q=
|
||||
|
|
@ -2275,10 +2277,9 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
|
|||
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
|
||||
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
|
|
|||
Loading…
Reference in New Issue