From dc9eb0230168f93fd35e0463b544c0feb33ef364 Mon Sep 17 00:00:00 2001 From: Pascal Zimmermann Date: Mon, 30 Jan 2023 15:52:54 +0100 Subject: [PATCH] fix(inputs.prometheus): Set the timeout for slow running API endpoints correctly (#12559) --- plugins/inputs/prometheus/prometheus.go | 1 + plugins/inputs/prometheus/prometheus_test.go | 59 ++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/plugins/inputs/prometheus/prometheus.go b/plugins/inputs/prometheus/prometheus.go index 6c8cd34be..edf35bcae 100644 --- a/plugins/inputs/prometheus/prometheus.go +++ b/plugins/inputs/prometheus/prometheus.go @@ -163,6 +163,7 @@ func (p *Prometheus) Init() error { } ctx := context.Background() + p.HTTPClientConfig.Timeout = p.ResponseTimeout client, err := p.HTTPClientConfig.CreateClient(ctx, p.Log) if err != nil { return err diff --git a/plugins/inputs/prometheus/prometheus_test.go b/plugins/inputs/prometheus/prometheus_test.go index d89f94ec0..caa75ba38 100644 --- a/plugins/inputs/prometheus/prometheus_test.go +++ b/plugins/inputs/prometheus/prometheus_test.go @@ -1,7 +1,9 @@ package prometheus import ( + "errors" "fmt" + "github.com/influxdata/telegraf/config" "math" "net/http" "net/http/httptest" @@ -137,6 +139,63 @@ func TestPrometheusGeneratesMetricsAlthoughFirstDNSFailsIntegration(t *testing.T require.True(t, acc.HasTimestamp("test_metric", time.Unix(1490802350, 0))) } +func TestPrometheusGeneratesMetricsSlowEndpoint(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(4 * time.Second) + _, err := fmt.Fprintln(w, sampleTextFormat) + require.NoError(t, err) + })) + defer ts.Close() + + p := &Prometheus{ + Log: testutil.Logger{}, + URLs: []string{ts.URL}, + URLTag: "url", + ResponseTimeout: config.Duration(time.Second * 5), + } + err := p.Init() + require.NoError(t, err) + + var acc testutil.Accumulator + + err = acc.GatherError(p.Gather) + require.NoError(t, err) + + require.True(t, acc.HasFloatField("go_gc_duration_seconds", "count")) + require.True(t, acc.HasFloatField("go_goroutines", "gauge")) + require.True(t, acc.HasFloatField("test_metric", "value")) + require.True(t, acc.HasTimestamp("test_metric", time.Unix(1490802350, 0))) + require.False(t, acc.HasTag("test_metric", "address")) + require.True(t, acc.TagValue("test_metric", "url") == ts.URL+"/metrics") +} + +func TestPrometheusGeneratesMetricsSlowEndpointHitTheTimeout(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(6 * time.Second) + _, err := fmt.Fprintln(w, sampleTextFormat) + require.NoError(t, err) + })) + defer ts.Close() + + p := &Prometheus{ + Log: testutil.Logger{}, + URLs: []string{ts.URL}, + URLTag: "url", + ResponseTimeout: config.Duration(time.Second * 5), + } + err := p.Init() + require.NoError(t, err) + + var acc testutil.Accumulator + + err = acc.GatherError(p.Gather) + errMessage := fmt.Sprintf("error making HTTP request to %s/metrics: Get \"%s/metrics\": "+ + "context deadline exceeded (Client.Timeout exceeded while awaiting headers)", ts.URL, ts.URL) + errExpected := errors.New(errMessage) + require.Equal(t, errExpected, err) + require.Error(t, err) +} + func TestPrometheusGeneratesSummaryMetricsV2(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, err := fmt.Fprintln(w, sampleSummaryTextFormat)