Add default retry for load config via url (#8803)

config/config.go

@@ -49,6 +49,7 @@ var (
 		`"`, `\"`,
 		`\`, `\\`,
 	)
+	httpLoadConfigRetryInterval = 10 * time.Second
 )
 
 // Config specifies the URL/user/password for the database that telegraf
@@ -921,17 +922,27 @@ func fetchConfig(u *url.URL) ([]byte, error) {
 	}
 	req.Header.Add("Accept", "application/toml")
 	req.Header.Set("User-Agent", internal.ProductToken())
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
+
+	retries := 3
+	for i := 0; i <= retries; i++ {
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("Retry %d of %d failed connecting to HTTP config server %s", i, retries, err)
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			if i < retries {
+				log.Printf("Error getting HTTP config.  Retry %d of %d in %s.  Status=%d", i, retries, httpLoadConfigRetryInterval, resp.StatusCode)
+				time.Sleep(httpLoadConfigRetryInterval)
+				continue
+			}
+			return nil, fmt.Errorf("Retry %d of %d failed to retrieve remote config: %s", i, retries, resp.Status)
+		}
+		defer resp.Body.Close()
+		return ioutil.ReadAll(resp.Body)
 	}
 
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("failed to retrieve remote config: %s", resp.Status)
-	}
-
-	defer resp.Body.Close()
-	return ioutil.ReadAll(resp.Body)
+	return nil, nil
 }
 
 // parseConfig loads a TOML configuration from a provided path and

config/config_test.go

@@ -1,6 +1,8 @@
 package config
 
 import (
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"strings"
 	"testing"
@@ -278,3 +280,37 @@ func TestConfig_AzureMonitorNamespacePrefix(t *testing.T) {
 	assert.Equal(t, "", azureMonitor.NamespacePrefix)
 	assert.Equal(t, true, ok)
 }
+
+func TestConfig_URLRetries3Fails(t *testing.T) {
+	httpLoadConfigRetryInterval = 0 * time.Second
+	responseCounter := 0
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+		responseCounter++
+	}))
+	defer ts.Close()
+
+	c := NewConfig()
+	err := c.LoadConfig(ts.URL)
+	require.Error(t, err)
+	require.Equal(t, 4, responseCounter)
+}
+
+func TestConfig_URLRetries3FailsThenPasses(t *testing.T) {
+	httpLoadConfigRetryInterval = 0 * time.Second
+	responseCounter := 0
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if responseCounter <= 2 {
+			w.WriteHeader(http.StatusNotFound)
+		} else {
+			w.WriteHeader(http.StatusOK)
+		}
+		responseCounter++
+	}))
+	defer ts.Close()
+
+	c := NewConfig()
+	err := c.LoadConfig(ts.URL)
+	require.NoError(t, err)
+	require.Equal(t, 4, responseCounter)
+}