Add ability to collect response body as field with http_response (#7596)

plugins/inputs/http_response/README.md

@@ -31,6 +31,15 @@ This input plugin checks HTTP/HTTPS connections.
   # {'fake':'data'}
   # '''
 
+  ## Optional name of the field that will contain the body of the response. 
+  ## By default it is set to an empty String indicating that the body's content won't be added 
+  # response_body_field = ''
+
+  ## Maximum allowed HTTP response body size in bytes.
+  ## 0 means to use the default of 32MiB.
+  ## If the response body size exceeds this limit a "body_read_error" will be raised
+  # response_body_max_size = "32MiB"
+
   ## Optional substring or regex match in body of the response (case sensitive)
   # response_string_match = "\"service_status\": \"up\""
   # response_string_match = "ok"
@@ -47,7 +56,7 @@ This input plugin checks HTTP/HTTPS connections.
   # [inputs.http_response.headers]
   #   Host = "github.com"
 
-  ## Optional setting to map reponse http headers into tags
+  ## Optional setting to map response http headers into tags
   ## If the http header is not present on the request, no corresponding tag will be added
   ## If multiple instances of the http header are present, only the first value will be used
   # http_header_tags = {"HTTP_HEADER" = "TAG_NAME"}
@@ -82,7 +91,7 @@ This tag is used to expose network and plugin errors. HTTP errors are considered
 --------------------------|-------------------------|-----------|
 |success                  | 0                       |The HTTP request completed, even if the HTTP code represents an error|
 |response_string_mismatch | 1                       |The option `response_string_match` was used, and the body of the response didn't match the regex. HTTP errors with content in their body (like 4xx, 5xx) will trigger this error|
-|body_read_error          | 2                       |The option `response_string_match` was used, but the plugin wasn't able to read the body of the response. Responses with empty bodies (like 3xx, HEAD, etc) will trigger this error|
+|body_read_error          | 2                       |The option `response_string_match` was used, but the plugin wasn't able to read the body of the response. Responses with empty bodies (like 3xx, HEAD, etc) will trigger this error. Or the option `response_body_field` was used and the content of the response body was not a valid uft-8. Or the size of the body of the response exceeded the `response_body_max_size` |
 |connection_failed        | 3                       |Catch all for any network error not specifically handled by the plugin|
 |timeout                  | 4                       |The plugin timed out while awaiting the HTTP connection to complete|
 |dns_error                | 5                       |There was a DNS error while attempting to connect to the host|

plugins/inputs/http_response/http_response.go

@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
@@ -19,6 +20,12 @@ import (
 	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
+const (
+	// defaultResponseBodyMaxSize is the default maximum response body size, in bytes.
+	// if the response body is over this size, we will raise a body_read_error.
+	defaultResponseBodyMaxSize = 32 * 1024 * 1024
+)
+
 // HTTPResponse struct
 type HTTPResponse struct {
 	Address         string   // deprecated in 1.12
@@ -31,7 +38,9 @@ type HTTPResponse struct {
 	Headers         map[string]string
 	FollowRedirects bool
 	// Absolute path to file with Bearer token
-	BearerToken         string `toml:"bearer_token"`
+	BearerToken         string        `toml:"bearer_token"`
+	ResponseBodyField   string        `toml:"response_body_field"`
+	ResponseBodyMaxSize internal.Size `toml:"response_body_max_size"`
 	ResponseStringMatch string
 	Interface           string
 	// HTTP Basic Auth Credentials
@@ -83,6 +92,15 @@ var sampleConfig = `
   # {'fake':'data'}
   # '''
 
+  ## Optional name of the field that will contain the body of the response. 
+  ## By default it is set to an empty String indicating that the body's content won't be added 
+  # response_body_field = ''
+
+  ## Maximum allowed HTTP response body size in bytes.
+  ## 0 means to use the default of 32MiB.
+  ## If the response body size exceeds this limit a "body_read_error" will be raised
+  # response_body_max_size = "32MiB"
+
   ## Optional substring or regex match in body of the response
   # response_string_match = "\"service_status\": \"up\""
   # response_string_match = "ok"
@@ -99,7 +117,7 @@ var sampleConfig = `
   # [inputs.http_response.headers]
   #   Host = "github.com"
 
-  ## Optional setting to map reponse http headers into tags
+  ## Optional setting to map response http headers into tags
   ## If the http header is not present on the request, no corresponding tag will be added
   ## If multiple instances of the http header are present, only the first value will be used
   # http_header_tags = {"HTTP_HEADER" = "TAG_NAME"}
@@ -310,17 +328,28 @@ func (h *HTTPResponse) httpGather(u string) (map[string]interface{}, map[string]
 	tags["status_code"] = strconv.Itoa(resp.StatusCode)
 	fields["http_response_code"] = resp.StatusCode
 
-	bodyBytes, err := ioutil.ReadAll(resp.Body)
+	if h.ResponseBodyMaxSize.Size == 0 {
-	if err != nil {
+		h.ResponseBodyMaxSize.Size = defaultResponseBodyMaxSize
-		h.Log.Debugf("Failed to read body of HTTP Response : %s", err.Error())
+	}
-		setResult("body_read_error", fields, tags)
+	bodyBytes, err := ioutil.ReadAll(io.LimitReader(resp.Body, h.ResponseBodyMaxSize.Size+1))
-		fields["content_length"] = len(bodyBytes)
+	// Check first if the response body size exceeds the limit.
-		if h.ResponseStringMatch != "" {
+	if err == nil && int64(len(bodyBytes)) > h.ResponseBodyMaxSize.Size {
-			fields["response_string_match"] = 0
+		h.setBodyReadError("The body of the HTTP Response is too large", bodyBytes, fields, tags)
-		}
+		return fields, tags, nil
+	} else if err != nil {
+		h.setBodyReadError(fmt.Sprintf("Failed to read body of HTTP Response : %s", err.Error()), bodyBytes, fields, tags)
 		return fields, tags, nil
 	}
 
+	// Add the body of the response if expected
+	if len(h.ResponseBodyField) > 0 {
+		// Check that the content of response contains only valid utf-8 characters.
+		if !utf8.Valid(bodyBytes) {
+			h.setBodyReadError("The body of the HTTP Response is not a valid utf-8 string", bodyBytes, fields, tags)
+			return fields, tags, nil
+		}
+		fields[h.ResponseBodyField] = string(bodyBytes)
+	}
 	fields["content_length"] = len(bodyBytes)
 
 	// Check the response for a regex match.
@@ -339,6 +368,16 @@ func (h *HTTPResponse) httpGather(u string) (map[string]interface{}, map[string]
 	return fields, tags, nil
 }
 
+// Set result in case of a body read error
+func (h *HTTPResponse) setBodyReadError(error_msg string, bodyBytes []byte, fields map[string]interface{}, tags map[string]string) {
+	h.Log.Debugf(error_msg)
+	setResult("body_read_error", fields, tags)
+	fields["content_length"] = len(bodyBytes)
+	if h.ResponseStringMatch != "" {
+		fields["response_string_match"] = 0
+	}
+}
+
 // Gather gets all metric fields and tags and returns any errors it encounters
 func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error {
 	// Compile the body regex if it exist

plugins/inputs/http_response/http_response_test.go

@@ -90,6 +90,9 @@ func setUpTestMux() http.Handler {
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		fmt.Fprintf(w, "hit the good page!")
 	})
+	mux.HandleFunc("/invalidUTF8", func(w http.ResponseWriter, req *http.Request) {
+		w.Write([]byte{0xff, 0xfe, 0xfd})
+	})
 	mux.HandleFunc("/noheader", func(w http.ResponseWriter, req *http.Request) {
 		fmt.Fprintf(w, "hit the good page!")
 	})
@@ -223,6 +226,109 @@ func TestFields(t *testing.T) {
 	checkOutput(t, &acc, expectedFields, expectedTags, absentFields, nil)
 }
 
+func TestResponseBodyField(t *testing.T) {
+	mux := setUpTestMux()
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+
+	h := &HTTPResponse{
+		Log:             testutil.Logger{},
+		Address:         ts.URL + "/good",
+		Body:            "{ 'test': 'data'}",
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		ResponseBodyField: "my_body_field",
+		FollowRedirects:   true,
+	}
+
+	var acc testutil.Accumulator
+	err := h.Gather(&acc)
+	require.NoError(t, err)
+
+	expectedFields := map[string]interface{}{
+		"http_response_code": http.StatusOK,
+		"result_type":        "success",
+		"result_code":        0,
+		"response_time":      nil,
+		"content_length":     nil,
+		"my_body_field":      "hit the good page!",
+	}
+	expectedTags := map[string]interface{}{
+		"server":      nil,
+		"method":      "GET",
+		"status_code": "200",
+		"result":      "success",
+	}
+	absentFields := []string{"response_string_match"}
+	checkOutput(t, &acc, expectedFields, expectedTags, absentFields, nil)
+
+	// Invalid UTF-8 String
+	h = &HTTPResponse{
+		Log:             testutil.Logger{},
+		Address:         ts.URL + "/invalidUTF8",
+		Body:            "{ 'test': 'data'}",
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		ResponseBodyField: "my_body_field",
+		FollowRedirects:   true,
+	}
+
+	acc = testutil.Accumulator{}
+	err = h.Gather(&acc)
+	require.NoError(t, err)
+
+	expectedFields = map[string]interface{}{
+		"result_type": "body_read_error",
+		"result_code": 2,
+	}
+	expectedTags = map[string]interface{}{
+		"server": nil,
+		"method": "GET",
+		"result": "body_read_error",
+	}
+	checkOutput(t, &acc, expectedFields, expectedTags, nil, nil)
+}
+
+func TestResponseBodyMaxSize(t *testing.T) {
+	mux := setUpTestMux()
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+
+	h := &HTTPResponse{
+		Log:             testutil.Logger{},
+		Address:         ts.URL + "/good",
+		Body:            "{ 'test': 'data'}",
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		ResponseBodyMaxSize: internal.Size{Size: 5},
+		FollowRedirects:     true,
+	}
+
+	var acc testutil.Accumulator
+	err := h.Gather(&acc)
+	require.NoError(t, err)
+
+	expectedFields := map[string]interface{}{
+		"result_type": "body_read_error",
+		"result_code": 2,
+	}
+	expectedTags := map[string]interface{}{
+		"server": nil,
+		"method": "GET",
+		"result": "body_read_error",
+	}
+	checkOutput(t, &acc, expectedFields, expectedTags, nil, nil)
+}
+
 func TestHTTPHeaderTags(t *testing.T) {
 	mux := setUpTestMux()
 	ts := httptest.NewServer(mux)