feat(outputs.http): Support configuration of `MaxIdleConns` and `MaxIdleConnsPerHost` (#10954)

plugins/common/http/config.go

@@ -16,8 +16,10 @@ import (
 
 // Common HTTP client struct.
 type HTTPClientConfig struct {
-	Timeout         config.Duration `toml:"timeout"`
-	IdleConnTimeout config.Duration `toml:"idle_conn_timeout"`
+	Timeout             config.Duration `toml:"timeout"`
+	IdleConnTimeout     config.Duration `toml:"idle_conn_timeout"`
+	MaxIdleConns        int             `toml:"max_idle_conn"`
+	MaxIdleConnsPerHost int             `toml:"max_idle_conn_per_host"`
 
 	proxy.HTTPProxy
 	tls.ClientConfig
@@ -37,9 +39,11 @@ func (h *HTTPClientConfig) CreateClient(ctx context.Context, log telegraf.Logger
 	}
 
 	transport := &http.Transport{
-		TLSClientConfig: tlsCfg,
-		Proxy:           prox,
-		IdleConnTimeout: time.Duration(h.IdleConnTimeout),
+		TLSClientConfig:     tlsCfg,
+		Proxy:               prox,
+		IdleConnTimeout:     time.Duration(h.IdleConnTimeout),
+		MaxIdleConns:        h.MaxIdleConns,
+		MaxIdleConnsPerHost: h.MaxIdleConnsPerHost,
 	}
 
 	timeout := h.Timeout

plugins/outputs/http/README.md

@@ -65,6 +65,15 @@ format by default.
   #   # Should be set manually to "application/json" for json data_format
   #   Content-Type = "text/plain; charset=utf-8"
 
+  ## MaxIdleConns controls the maximum number of idle (keep-alive)
+  ## connections across all hosts. Zero means no limit.
+  # max_idle_conn = 0
+
+  ## MaxIdleConnsPerHost, if non-zero, controls the maximum idle
+  ## (keep-alive) connections to keep per-host. If zero,
+  ## DefaultMaxIdleConnsPerHost is used(2).
+  # max_idle_conn_per_host = 2
+
   ## Idle (keep-alive) connection timeout.
   ## Maximum amount of time before idle connection is closed.
   ## Zero means no limit.

plugins/outputs/http/http_test.go

@@ -10,6 +10,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/influxdata/telegraf/config"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/influxdata/telegraf"
@@ -125,6 +127,74 @@ func TestMethod(t *testing.T) {
 	}
 }
 
+func TestHTTPClientConfig(t *testing.T) {
+	ts := httptest.NewServer(http.NotFoundHandler())
+	defer ts.Close()
+
+	u, err := url.Parse(fmt.Sprintf("http://%s", ts.Listener.Addr().String()))
+	require.NoError(t, err)
+
+	tests := []struct {
+		name                        string
+		plugin                      *HTTP
+		connectError                bool
+		expectedMaxIdleConns        int
+		expectedMaxIdleConnsPerHost int
+	}{
+		{
+			name: "With default client Config",
+			plugin: &HTTP{
+				URL:    u.String(),
+				Method: defaultMethod,
+				HTTPClientConfig: httpconfig.HTTPClientConfig{
+					IdleConnTimeout: config.Duration(5 * time.Second),
+				},
+			},
+			expectedMaxIdleConns:        0,
+			expectedMaxIdleConnsPerHost: 0,
+		},
+		{
+			name: "With MaxIdleConns client Config",
+			plugin: &HTTP{
+				URL:    u.String(),
+				Method: defaultMethod,
+				HTTPClientConfig: httpconfig.HTTPClientConfig{
+					MaxIdleConns:        100,
+					MaxIdleConnsPerHost: 100,
+					IdleConnTimeout:     config.Duration(5 * time.Second),
+				},
+			},
+			expectedMaxIdleConns:        100,
+			expectedMaxIdleConnsPerHost: 100,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+			})
+
+			serializer := influx.NewSerializer()
+			tt.plugin.SetSerializer(serializer)
+			err = tt.plugin.Connect()
+			if tt.connectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			tr := tt.plugin.client.Transport.(*http.Transport)
+			maxIdleConns, maxIdleConnsPerHost := tr.MaxIdleConns, tr.MaxIdleConnsPerHost
+			require.Equal(t, tt.expectedMaxIdleConns, maxIdleConns)
+			require.Equal(t, tt.expectedMaxIdleConnsPerHost, maxIdleConnsPerHost)
+
+			err = tt.plugin.Write([]telegraf.Metric{getMetric()})
+			require.NoError(t, err)
+		})
+	}
+}
+
 func TestStatusCode(t *testing.T) {
 	ts := httptest.NewServer(http.NotFoundHandler())
 	defer ts.Close()