fix(config): Restore old environment var behavior with option (#13457)

This commit is contained in:
Sven Rebhan 2023-06-21 15:58:21 +02:00 committed by GitHub
parent 24b3d83f49
commit 47e10e1679
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 278 additions and 93 deletions

View File

@ -219,17 +219,18 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi
filters := processFilterFlags(cCtx)
g := GlobalFlags{
config: cCtx.StringSlice("config"),
configDir: cCtx.StringSlice("config-directory"),
testWait: cCtx.Int("test-wait"),
watchConfig: cCtx.String("watch-config"),
pidFile: cCtx.String("pidfile"),
plugindDir: cCtx.String("plugin-directory"),
password: cCtx.String("password"),
test: cCtx.Bool("test"),
debug: cCtx.Bool("debug"),
once: cCtx.Bool("once"),
quiet: cCtx.Bool("quiet"),
config: cCtx.StringSlice("config"),
configDir: cCtx.StringSlice("config-directory"),
testWait: cCtx.Int("test-wait"),
watchConfig: cCtx.String("watch-config"),
pidFile: cCtx.String("pidfile"),
plugindDir: cCtx.String("plugin-directory"),
password: cCtx.String("password"),
oldEnvBehavior: cCtx.Bool("old-env-behavior"),
test: cCtx.Bool("test"),
debug: cCtx.Bool("debug"),
once: cCtx.Bool("once"),
quiet: cCtx.Bool("quiet"),
}
w := WindowFlags{
@ -294,6 +295,10 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi
},
//
// Bool flags
&cli.BoolFlag{
Name: "old-env-behavior",
Usage: "switch back to pre v1.27 environment replacement behavior",
},
&cli.BoolFlag{
Name: "once",
Usage: "run one gather and exit",

View File

@ -32,17 +32,18 @@ import (
var stop chan struct{}
type GlobalFlags struct {
config []string
configDir []string
testWait int
watchConfig string
pidFile string
plugindDir string
password string
test bool
debug bool
once bool
quiet bool
config []string
configDir []string
testWait int
watchConfig string
pidFile string
plugindDir string
password string
oldEnvBehavior bool
test bool
debug bool
once bool
quiet bool
}
type WindowFlags struct {
@ -87,6 +88,9 @@ func (t *Telegraf) Init(pprofErr <-chan error, f Filters, g GlobalFlags, w Windo
if g.password != "" {
config.Password = config.NewSecret([]byte(g.password))
}
// Set environment replacement behavior
config.OldEnvVarReplacement = g.oldEnvBehavior
}
func (t *Telegraf) ListSecretStores() ([]string, error) {

View File

@ -40,12 +40,6 @@ import (
"github.com/influxdata/telegraf/plugins/serializers"
)
// envVarPattern is a regex to determine environment variables in the
// config file for substitution. Those should start with a dollar signs.
// Expression modified from
// https://github.com/compose-spec/compose-go/blob/v1.14.0/template/template.go
const envVarPattern = `\\(?P<escaped>\$)|\$(?i:(?P<named>[_a-z][_a-z0-9]*)|\${(?:(?P<braced>[_a-z][_a-z0-9]*(?::?[-+?](.*))?)}|(?P<invalid>)))`
var (
httpLoadConfigRetryInterval = 10 * time.Second
@ -53,8 +47,12 @@ var (
// be fetched from a remote or read from the filesystem.
fetchURLRe = regexp.MustCompile(`^\w+://`)
// envVarRe is the compiled regex of envVarPattern
envVarRe = regexp.MustCompile(envVarPattern)
// oldVarRe is a regex to reproduce pre v1.27.0 environment variable
// replacement behavior
oldVarRe = regexp.MustCompile(`\$(?i:(?P<named>[_a-z][_a-z0-9]*)|{(?:(?P<braced>[_a-z][_a-z0-9]*(?::?[-+?](.*))?)}|(?P<invalid>)))`)
// OldEnvVarReplacement is a switch to allow going back to pre v1.27.0
// environment variable replacement behavior
OldEnvVarReplacement = false
// Password specified via command-line
Password Secret
@ -794,7 +792,7 @@ func parseConfig(contents []byte) (*ast.Table, error) {
if err != nil {
return nil, err
}
outputBytes, err := substituteEnvironment(contents)
outputBytes, err := substituteEnvironment(contents, OldEnvVarReplacement)
if err != nil {
return nil, err
}
@ -857,19 +855,36 @@ func removeComments(contents []byte) ([]byte, error) {
return output.Bytes(), nil
}
func substituteEnvironment(contents []byte) ([]byte, error) {
func substituteEnvironment(contents []byte, oldReplacementBehavior bool) ([]byte, error) {
options := []template.Option{
template.WithReplacementFunction(func(s string, m template.Mapping, cfg *template.Config) (string, error) {
result, err := template.DefaultReplacementFunc(s, m, cfg)
if err == nil && result == "" {
// Keep undeclared environment-variable patterns to reproduce
// pre-v1.27 behavior
return s, nil
}
if err != nil && strings.HasPrefix(err.Error(), "Invalid template:") {
// Keep invalid template patterns to ignore regexp substitutions
// like ${1}
return s, nil
}
return result, err
}),
template.WithoutLogging,
}
if oldReplacementBehavior {
options = append(options, template.WithPattern(oldVarRe))
}
envMap := utils.GetAsEqualsMap(os.Environ())
retVal, err := template.SubstituteWith(string(contents), func(k string) (string, bool) {
retVal, err := template.SubstituteWithOptions(string(contents), func(k string) (string, bool) {
if v, ok := envMap[k]; ok {
return v, ok
}
return "", false
}, envVarRe)
var invalidTmplError *template.InvalidTemplateError
if err != nil && !errors.As(err, &invalidTmplError) {
return nil, err
}
return []byte(retVal), nil
}, options...)
return []byte(retVal), err
}
func (c *Config) addAggregator(name string, table *ast.Table) error {

View File

@ -71,7 +71,7 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
input := inputs.Inputs["memcached"]().(*MockupInputPlugin)
input.Servers = []string{"192.168.1.1"}
input.Command = `Raw command which may or may not contain # or ${var} in it
input.Command = `Raw command which may or may not contain # in it
# is unique`
filter := models.Filter{

View File

@ -25,17 +25,17 @@ func TestEnvironmentSubstitution(t *testing.T) {
t.Setenv("TEST_ENV1", "VALUE1")
t.Setenv("TEST_ENV2", "VALUE2")
},
contents: "A string with $${TEST_ENV1}, $TEST_ENV2 and $TEST_ENV1 as repeated",
contents: "A string with ${TEST_ENV1}, $TEST_ENV2 and $TEST_ENV1 as repeated",
expected: "A string with VALUE1, VALUE2 and VALUE1 as repeated",
},
{
name: "Env not set",
contents: "Env variable $${NOT_SET} will be empty",
expected: "Env variable will be empty", // Two spaces present
contents: "Env variable ${NOT_SET} will be empty",
expected: "Env variable ${NOT_SET} will be empty",
},
{
name: "Env not set, fallback to default",
contents: "Env variable $${THIS_IS_ABSENT:-Fallback}",
contents: "Env variable ${THIS_IS_ABSENT:-Fallback}",
expected: "Env variable Fallback",
},
{
@ -43,7 +43,7 @@ func TestEnvironmentSubstitution(t *testing.T) {
setEnv: func(t *testing.T) {
t.Setenv("MY_ENV1", "VALUE1")
},
contents: "Env variable $${MY_ENV1:-Fallback}",
contents: "Env variable ${MY_ENV1:-Fallback}",
expected: "Env variable VALUE1",
},
{
@ -52,17 +52,17 @@ func TestEnvironmentSubstitution(t *testing.T) {
t.Setenv("MY_VAR", "VALUE")
t.Setenv("MY_VAR2", "VALUE2")
},
contents: "Env var $${MY_VAR} is set, with $MY_VAR syntax and default on this $${MY_VAR1:-Substituted}, no default on this $${MY_VAR2:-NoDefault}",
contents: "Env var ${MY_VAR} is set, with $MY_VAR syntax and default on this ${MY_VAR1:-Substituted}, no default on this ${MY_VAR2:-NoDefault}",
expected: "Env var VALUE is set, with VALUE syntax and default on this Substituted, no default on this VALUE2",
},
{
name: "Default has special chars",
contents: `Not recommended but supported $${MY_VAR:-Default with special chars Supported#$\"}`,
contents: `Not recommended but supported ${MY_VAR:-Default with special chars Supported#$\"}`,
expected: `Not recommended but supported Default with special chars Supported#$\"`, // values are escaped
},
{
name: "unset error",
contents: "Contains $${THIS_IS_NOT_SET?unset-error}",
contents: "Contains ${THIS_IS_NOT_SET?unset-error}",
wantErr: true,
errSubstring: "unset-error",
},
@ -71,7 +71,7 @@ func TestEnvironmentSubstitution(t *testing.T) {
setEnv: func(t *testing.T) {
t.Setenv("ENV_EMPTY", "")
},
contents: "Contains $${ENV_EMPTY:?empty-error}",
contents: "Contains ${ENV_EMPTY:?empty-error}",
wantErr: true,
errSubstring: "empty-error",
},
@ -80,33 +80,9 @@ func TestEnvironmentSubstitution(t *testing.T) {
setEnv: func(t *testing.T) {
t.Setenv("FALLBACK", "my-fallback")
},
contents: "Should output $${NOT_SET:-${FALLBACK}}",
contents: "Should output ${NOT_SET:-${FALLBACK}}",
expected: "Should output my-fallback",
},
{
name: "leave alone single dollar expressions #13432",
setEnv: func(t *testing.T) {
t.Setenv("MYVAR", "my-variable")
},
contents: "Should output ${MYVAR}",
expected: "Should output ${MYVAR}",
},
{
name: "leave alone escaped expressions (backslash)",
setEnv: func(t *testing.T) {
t.Setenv("MYVAR", "my-variable")
},
contents: `Should output \$MYVAR`,
expected: "Should output $MYVAR",
},
{
name: "double dollar no brackets",
setEnv: func(t *testing.T) {
t.Setenv("MYVAR", "my-variable")
},
contents: `Should output $$MYVAR`,
expected: "Should output $my-variable",
},
}
for _, tt := range tests {
@ -114,7 +90,7 @@ func TestEnvironmentSubstitution(t *testing.T) {
if tt.setEnv != nil {
tt.setEnv(t)
}
actual, err := substituteEnvironment([]byte(tt.contents))
actual, err := substituteEnvironment([]byte(tt.contents), false)
if tt.wantErr {
require.ErrorContains(t, err, tt.errSubstring)
return
@ -124,6 +100,194 @@ func TestEnvironmentSubstitution(t *testing.T) {
}
}
func TestEnvironmentSubstitutionOldBehavior(t *testing.T) {
tests := []struct {
name string
contents string
expected string
}{
{
name: "not defined no brackets",
contents: `my-da$tabase`,
expected: `my-da$tabase`,
},
{
name: "not defined brackets",
contents: `my-da${ta}base`,
expected: `my-da${ta}base`,
},
{
name: "not defined no brackets double dollar",
contents: `my-da$$tabase`,
expected: `my-da$$tabase`,
},
{
name: "not defined no brackets backslash",
contents: `my-da\$tabase`,
expected: `my-da\$tabase`,
},
{
name: "not defined brackets backslash",
contents: `my-da\${ta}base`,
expected: `my-da\${ta}base`,
},
{
name: "no brackets and suffix",
contents: `my-da$VARbase`,
expected: `my-da$VARbase`,
},
{
name: "no brackets",
contents: `my-da$VAR`,
expected: `my-dafoobar`,
},
{
name: "brackets",
contents: `my-da${VAR}base`,
expected: `my-dafoobarbase`,
},
{
name: "no brackets double dollar",
contents: `my-da$$VAR`,
expected: `my-da$foobar`,
},
{
name: "brackets double dollar",
contents: `my-da$${VAR}`,
expected: `my-da$foobar`,
},
{
name: "no brackets backslash",
contents: `my-da\$VAR`,
expected: `my-da\foobar`,
},
{
name: "brackets backslash",
contents: `my-da\${VAR}base`,
expected: `my-da\foobarbase`,
},
{
name: "fallback",
contents: `my-da${ta:-omg}base`,
expected: `my-daomgbase`,
},
{
name: "fallback env",
contents: `my-da${ta:-${FALLBACK}}base`,
expected: `my-dadefaultbase`,
},
{
name: "regex substitution",
contents: `${1}`,
expected: `${1}`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv("VAR", "foobar")
t.Setenv("FALLBACK", "default")
actual, err := substituteEnvironment([]byte(tt.contents), true)
require.NoError(t, err)
require.EqualValues(t, tt.expected, string(actual))
})
}
}
func TestEnvironmentSubstitutionNewBehavior(t *testing.T) {
tests := []struct {
name string
contents string
expected string
}{
{
name: "not defined no brackets",
contents: `my-da$tabase`,
expected: `my-da$tabase`,
},
{
name: "not defined brackets",
contents: `my-da${ta}base`,
expected: `my-da${ta}base`,
},
{
name: "not defined no brackets double dollar",
contents: `my-da$$tabase`,
expected: `my-da$tabase`,
},
{
name: "not defined no brackets backslash",
contents: `my-da\$tabase`,
expected: `my-da\$tabase`,
},
{
name: "not defined brackets backslash",
contents: `my-da\${ta}base`,
expected: `my-da\${ta}base`,
},
{
name: "no brackets and suffix",
contents: `my-da$VARbase`,
expected: `my-da$VARbase`,
},
{
name: "no brackets",
contents: `my-da$VAR`,
expected: `my-dafoobar`,
},
{
name: "brackets",
contents: `my-da${VAR}base`,
expected: `my-dafoobarbase`,
},
{
name: "no brackets double dollar",
contents: `my-da$$VAR`,
expected: `my-da$VAR`,
},
{
name: "brackets double dollar",
contents: `my-da$${VAR}`,
expected: `my-da${VAR}`,
},
{
name: "no brackets backslash",
contents: `my-da\$VAR`,
expected: `my-da\foobar`,
},
{
name: "brackets backslash",
contents: `my-da\${VAR}base`,
expected: `my-da\foobarbase`,
},
{
name: "fallback",
contents: `my-da${ta:-omg}base`,
expected: `my-daomgbase`,
},
{
name: "fallback env",
contents: `my-da${ta:-${FALLBACK}}base`,
expected: `my-dadefaultbase`,
},
{
name: "regex substitution",
contents: `${1}`,
expected: `${1}`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv("VAR", "foobar")
t.Setenv("FALLBACK", "default")
actual, err := substituteEnvironment([]byte(tt.contents), false)
require.NoError(t, err)
require.EqualValues(t, tt.expected, string(actual))
})
}
}
func TestURLRetries3Fails(t *testing.T) {
httpLoadConfigRetryInterval = 0 * time.Second
responseCounter := 0

View File

@ -10,20 +10,20 @@
# file would generate.
#
# Environment variables can be used anywhere in this config file, simply surround
# them with $${}. For strings the variable must be within quotes (ie, "$${STR_VAR}"),
# for numbers and booleans they should be plain (ie, $${INT_VAR}, $${BOOL_VAR})
# them with ${}. For strings the variable must be within quotes (ie, "${STR_VAR}"),
# for numbers and booleans they should be plain (ie, ${INT_VAR}, ${BOOL_VAR})
[[inputs.memcached]]
# this comment line will be ignored by the parser
servers = ["$MY_TEST_SERVER"]
namepass = ["metricname1", "ip_$${MY_TEST_SERVER}_name"] # this comment will be ignored as well
namepass = ["metricname1", "ip_${MY_TEST_SERVER}_name"] # this comment will be ignored as well
namedrop = ["metricname2"]
fieldpass = ["some", "strings"]
fielddrop = ["other", "stuff"]
interval = "$TEST_INTERVAL"
##### this input is provided to test multiline strings
command = """
Raw command which may or may not contain # or ${var} in it
Raw command which may or may not contain # in it
# is unique""" # Multiline comment black starting with #
[inputs.memcached.tagpass]
goodtag = ["mytag", """tagwith#value""",

13
go.mod
View File

@ -55,7 +55,7 @@ require (
github.com/caio/go-tdigest v3.1.0+incompatible
github.com/cisco-ie/nx-telemetry-proto v0.0.0-20230117155933-f64c045c77df
github.com/clarify/clarify-go v0.2.4
github.com/compose-spec/compose-go v1.13.4
github.com/compose-spec/compose-go v1.15.0
github.com/coocood/freecache v1.2.3
github.com/coreos/go-semver v0.3.1
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
@ -188,7 +188,7 @@ require (
golang.org/x/mod v0.10.0
golang.org/x/net v0.10.0
golang.org/x/oauth2 v0.8.0
golang.org/x/sync v0.2.0
golang.org/x/sync v0.3.0
golang.org/x/sys v0.9.0
golang.org/x/term v0.9.0
golang.org/x/text v0.9.0
@ -209,11 +209,6 @@ require (
modernc.org/sqlite v1.23.1
)
require (
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
)
require (
cloud.google.com/go v0.110.1 // indirect
cloud.google.com/go/compute v1.19.1 // indirect
@ -335,7 +330,7 @@ require (
github.com/hashicorp/packer-plugin-sdk v0.3.1 // indirect
github.com/hashicorp/serf v0.10.1 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.15 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.2 // indirect
@ -455,6 +450,8 @@ require (
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
golang.zx2c4.com/wireguard v0.0.0-20211209221555-9c9e7e272434 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
gopkg.in/errgo.v1 v1.0.1 // indirect
gopkg.in/fatih/pool.v2 v2.0.0 // indirect
gopkg.in/fsnotify.v1 v1.4.7 // indirect

12
go.sum
View File

@ -409,8 +409,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/compose-spec/compose-go v1.13.4 h1:O6xAsPqaY1s9KXteiO7wRCDTJLahv1XP/z/eUO9EfbI=
github.com/compose-spec/compose-go v1.13.4/go.mod h1:rsiZ8uaOHJYJemDBzTe9UBpaq5ZFVEOO4TxM2G3SJxk=
github.com/compose-spec/compose-go v1.15.0 h1:rv3TTgbS3U4Y8sRTngrcxDmpbz+fq26wTqHculSCi6s=
github.com/compose-spec/compose-go v1.15.0/go.mod h1:3yngGBGfls6FHGQsg4B1z6gz8ej9SOvmAJtxCwgbcnc=
github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns=
github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw=
@ -839,8 +839,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/go-syslog/v3 v3.0.0 h1:jichmjSZlYK0VMmlz+k4WeOQd7z745YLsvGMqwtYt4I=
github.com/influxdata/go-syslog/v3 v3.0.0/go.mod h1:tulsOp+CecTAYC27u9miMgq21GqXRW6VdKbOG+QSP4Q=
@ -1798,8 +1798,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=