feat(agent): Introduce CLI option to set config URL retry attempts (#15377)

cmd/telegraf/main.go

@@ -221,19 +221,20 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi
 		filters := processFilterFlags(cCtx)
 
 		g := GlobalFlags{
-			config:         cCtx.StringSlice("config"),
+			config:                 cCtx.StringSlice("config"),
-			configDir:      cCtx.StringSlice("config-directory"),
+			configDir:              cCtx.StringSlice("config-directory"),
-			testWait:       cCtx.Int("test-wait"),
+			testWait:               cCtx.Int("test-wait"),
-			watchConfig:    cCtx.String("watch-config"),
+			configURLRetryAttempts: cCtx.Int("config-url-retry-attempts"),
-			pidFile:        cCtx.String("pidfile"),
+			watchConfig:            cCtx.String("watch-config"),
-			plugindDir:     cCtx.String("plugin-directory"),
+			pidFile:                cCtx.String("pidfile"),
-			password:       cCtx.String("password"),
+			plugindDir:             cCtx.String("plugin-directory"),
-			oldEnvBehavior: cCtx.Bool("old-env-behavior"),
+			password:               cCtx.String("password"),
-			test:           cCtx.Bool("test"),
+			oldEnvBehavior:         cCtx.Bool("old-env-behavior"),
-			debug:          cCtx.Bool("debug"),
+			test:                   cCtx.Bool("test"),
-			once:           cCtx.Bool("once"),
+			debug:                  cCtx.Bool("debug"),
-			quiet:          cCtx.Bool("quiet"),
+			once:                   cCtx.Bool("once"),
-			unprotected:    cCtx.Bool("unprotected"),
+			quiet:                  cCtx.Bool("quiet"),
+			unprotected:            cCtx.Bool("unprotected"),
 		}
 
 		w := WindowFlags{
@@ -275,6 +276,11 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi
 					Name:  "test-wait",
 					Usage: "wait up to this many seconds for service inputs to complete in test mode",
 				},
+				&cli.IntFlag{
+					Name: "config-url-retry-attempts",
+					Usage: "Number of attempts to obtain a remote configuration via a URL during startup. " +
+						"Set to -1 for unlimited attempts. (default: 3)",
+				},
 				//
 				// String flags
 				&cli.StringFlag{

cmd/telegraf/telegraf.go

@@ -32,19 +32,20 @@ import (
 var stop chan struct{}
 
 type GlobalFlags struct {
-	config         []string
+	config                 []string
-	configDir      []string
+	configDir              []string
-	testWait       int
+	testWait               int
-	watchConfig    string
+	configURLRetryAttempts int
-	pidFile        string
+	watchConfig            string
-	plugindDir     string
+	pidFile                string
-	password       string
+	plugindDir             string
-	oldEnvBehavior bool
+	password               string
-	test           bool
+	oldEnvBehavior         bool
-	debug          bool
+	test                   bool
-	once           bool
+	debug                  bool
-	quiet          bool
+	once                   bool
-	unprotected    bool
+	quiet                  bool
+	unprotected            bool
 }
 
 type WindowFlags struct {
@@ -248,6 +249,7 @@ func (t *Telegraf) loadConfiguration() (*config.Config, error) {
 		configFiles = append(configFiles, defaultFiles...)
 	}
 
+	c.Agent.ConfigURLRetryAttempts = t.configURLRetryAttempts
 	t.configFiles = configFiles
 	if err := c.LoadAll(configFiles...); err != nil {
 		return c, err

config/config.go

@@ -271,6 +271,10 @@ type AgentConfig struct {
 	// By default, processors are run a second time after aggregators. Changing
 	// this setting to true will skip the second run of processors.
 	SkipProcessorsAfterAggregators bool `toml:"skip_processors_after_aggregators"`
+
+	// Number of attempts to obtain a remote configuration via a URL during
+	// startup. Set to -1 for unlimited attempts.
+	ConfigURLRetryAttempts int `toml:"config-url-retry-attempts"`
 }
 
 // InputNames returns a list of strings of the configured inputs.
@@ -449,7 +453,7 @@ func (c *Config) LoadConfig(path string) error {
 		log.Printf("I! Loading config: %s", path)
 	}
 
-	data, _, err := LoadConfigFile(path)
+	data, _, err := LoadConfigFileWithRetries(path, c.Agent.ConfigURLRetryAttempts)
 	if err != nil {
 		return fmt.Errorf("error loading config file %s: %w", path, err)
 	}
@@ -718,6 +722,10 @@ func trimBOM(f []byte) []byte {
 // together with a flag denoting if the file is from a remote location such
 // as a web server.
 func LoadConfigFile(config string) ([]byte, bool, error) {
+	return LoadConfigFileWithRetries(config, 0)
+}
+
+func LoadConfigFileWithRetries(config string, urlRetryAttempts int) ([]byte, bool, error) {
 	if fetchURLRe.MatchString(config) {
 		u, err := url.Parse(config)
 		if err != nil {
@@ -726,7 +734,7 @@ func LoadConfigFile(config string) ([]byte, bool, error) {
 
 		switch u.Scheme {
 		case "https", "http":
-			data, err := fetchConfig(u)
+			data, err := fetchConfig(u, urlRetryAttempts)
 			return data, true, err
 		default:
 			return nil, true, fmt.Errorf("scheme %q not supported", u.Scheme)
@@ -747,7 +755,7 @@ func LoadConfigFile(config string) ([]byte, bool, error) {
 	return buffer, false, nil
 }
 
-func fetchConfig(u *url.URL) ([]byte, error) {
+func fetchConfig(u *url.URL, urlRetryAttempts int) ([]byte, error) {
 	req, err := http.NewRequest("GET", u.String(), nil)
 	if err != nil {
 		return nil, err
@@ -759,38 +767,53 @@ func fetchConfig(u *url.URL) ([]byte, error) {
 	req.Header.Add("Accept", "application/toml")
 	req.Header.Set("User-Agent", internal.ProductToken())
 
-	retries := 3
+	var totalAttempts int
-	for i := 0; i <= retries; i++ {
+	if urlRetryAttempts == -1 {
-		body, err, retry := func() ([]byte, error, bool) {
+		totalAttempts = -1
-			resp, err := http.DefaultClient.Do(req)
+		log.Printf("Using unlimited number of attempts to fetch HTTP config")
-			if err != nil {
+	} else if urlRetryAttempts == 0 {
-				return nil, fmt.Errorf("retry %d of %d failed connecting to HTTP config server: %w", i, retries, err), false
+		totalAttempts = 3
-			}
+		log.Printf("Using default number of attempts to fetch HTTP config: %d", totalAttempts)
-			defer resp.Body.Close()
+	} else if urlRetryAttempts > 0 {
-			if resp.StatusCode != http.StatusOK {
+		totalAttempts = urlRetryAttempts
-				if i < retries {
+	} else {
-					log.Printf("Error getting HTTP config.  Retry %d of %d in %s.  Status=%d", i, retries, httpLoadConfigRetryInterval, resp.StatusCode)
+		return nil, fmt.Errorf("invalid number of attempts: %d", urlRetryAttempts)
-					return nil, nil, true
+	}
-				}
-				return nil, fmt.Errorf("retry %d of %d failed to retrieve remote config: %s", i, retries, resp.Status), false
-			}
-			body, err := io.ReadAll(resp.Body)
-			return body, err, false
-		}()
 
-		if err != nil {
+	attempt := 0
+	for {
+		body, err := requestURLConfig(req)
+		if err == nil {
+			return body, nil
+		}
+
+		log.Printf("Error getting HTTP config (attempt %d of %d): %s", attempt, totalAttempts, err)
+		if urlRetryAttempts != -1 && attempt >= totalAttempts {
 			return nil, err
 		}
 
-		if retry {
+		time.Sleep(httpLoadConfigRetryInterval)
-			time.Sleep(httpLoadConfigRetryInterval)
+		attempt++
-			continue
+	}
-		}
+}
 
-		return body, err
+func requestURLConfig(req *http.Request) ([]byte, error) {
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to HTTP config server: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to fetch HTTP config: %s", resp.Status)
 	}
 
-	return nil, nil
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	return body, nil
 }
 
 // parseConfig loads a TOML configuration from a provided path and

config/internal_test.go

@@ -383,7 +383,7 @@ func TestURLRetries3Fails(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	expected := fmt.Sprintf("error loading config file %s: retry 3 of 3 failed to retrieve remote config: 404 Not Found", ts.URL)
+	expected := fmt.Sprintf("error loading config file %s: failed to fetch HTTP config: 404 Not Found", ts.URL)
 
 	c := NewConfig()
 	err := c.LoadConfig(ts.URL)