diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md index f1d1ab2d5..26fee33f0 100644 --- a/plugins/inputs/http_response/README.md +++ b/plugins/inputs/http_response/README.md @@ -31,6 +31,15 @@ This input plugin checks HTTP/HTTPS connections. # {'fake':'data'} # ''' + ## Optional name of the field that will contain the body of the response. + ## By default it is set to an empty String indicating that the body's content won't be added + # response_body_field = '' + + ## Maximum allowed HTTP response body size in bytes. + ## 0 means to use the default of 32MiB. + ## If the response body size exceeds this limit a "body_read_error" will be raised + # response_body_max_size = "32MiB" + ## Optional substring or regex match in body of the response (case sensitive) # response_string_match = "\"service_status\": \"up\"" # response_string_match = "ok" @@ -47,7 +56,7 @@ This input plugin checks HTTP/HTTPS connections. # [inputs.http_response.headers] # Host = "github.com" - ## Optional setting to map reponse http headers into tags + ## Optional setting to map response http headers into tags ## If the http header is not present on the request, no corresponding tag will be added ## If multiple instances of the http header are present, only the first value will be used # http_header_tags = {"HTTP_HEADER" = "TAG_NAME"} @@ -82,7 +91,7 @@ This tag is used to expose network and plugin errors. HTTP errors are considered --------------------------|-------------------------|-----------| |success | 0 |The HTTP request completed, even if the HTTP code represents an error| |response_string_mismatch | 1 |The option `response_string_match` was used, and the body of the response didn't match the regex. HTTP errors with content in their body (like 4xx, 5xx) will trigger this error| -|body_read_error | 2 |The option `response_string_match` was used, but the plugin wasn't able to read the body of the response. Responses with empty bodies (like 3xx, HEAD, etc) will trigger this error| +|body_read_error | 2 |The option `response_string_match` was used, but the plugin wasn't able to read the body of the response. Responses with empty bodies (like 3xx, HEAD, etc) will trigger this error. Or the option `response_body_field` was used and the content of the response body was not a valid uft-8. Or the size of the body of the response exceeded the `response_body_max_size` | |connection_failed | 3 |Catch all for any network error not specifically handled by the plugin| |timeout | 4 |The plugin timed out while awaiting the HTTP connection to complete| |dns_error | 5 |There was a DNS error while attempting to connect to the host| diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index bc9452efc..12c02d1c1 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "time" + "unicode/utf8" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" @@ -19,6 +20,12 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" ) +const ( + // defaultResponseBodyMaxSize is the default maximum response body size, in bytes. + // if the response body is over this size, we will raise a body_read_error. + defaultResponseBodyMaxSize = 32 * 1024 * 1024 +) + // HTTPResponse struct type HTTPResponse struct { Address string // deprecated in 1.12 @@ -31,7 +38,9 @@ type HTTPResponse struct { Headers map[string]string FollowRedirects bool // Absolute path to file with Bearer token - BearerToken string `toml:"bearer_token"` + BearerToken string `toml:"bearer_token"` + ResponseBodyField string `toml:"response_body_field"` + ResponseBodyMaxSize internal.Size `toml:"response_body_max_size"` ResponseStringMatch string Interface string // HTTP Basic Auth Credentials @@ -83,6 +92,15 @@ var sampleConfig = ` # {'fake':'data'} # ''' + ## Optional name of the field that will contain the body of the response. + ## By default it is set to an empty String indicating that the body's content won't be added + # response_body_field = '' + + ## Maximum allowed HTTP response body size in bytes. + ## 0 means to use the default of 32MiB. + ## If the response body size exceeds this limit a "body_read_error" will be raised + # response_body_max_size = "32MiB" + ## Optional substring or regex match in body of the response # response_string_match = "\"service_status\": \"up\"" # response_string_match = "ok" @@ -99,7 +117,7 @@ var sampleConfig = ` # [inputs.http_response.headers] # Host = "github.com" - ## Optional setting to map reponse http headers into tags + ## Optional setting to map response http headers into tags ## If the http header is not present on the request, no corresponding tag will be added ## If multiple instances of the http header are present, only the first value will be used # http_header_tags = {"HTTP_HEADER" = "TAG_NAME"} @@ -310,17 +328,28 @@ func (h *HTTPResponse) httpGather(u string) (map[string]interface{}, map[string] tags["status_code"] = strconv.Itoa(resp.StatusCode) fields["http_response_code"] = resp.StatusCode - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - h.Log.Debugf("Failed to read body of HTTP Response : %s", err.Error()) - setResult("body_read_error", fields, tags) - fields["content_length"] = len(bodyBytes) - if h.ResponseStringMatch != "" { - fields["response_string_match"] = 0 - } + if h.ResponseBodyMaxSize.Size == 0 { + h.ResponseBodyMaxSize.Size = defaultResponseBodyMaxSize + } + bodyBytes, err := ioutil.ReadAll(io.LimitReader(resp.Body, h.ResponseBodyMaxSize.Size+1)) + // Check first if the response body size exceeds the limit. + if err == nil && int64(len(bodyBytes)) > h.ResponseBodyMaxSize.Size { + h.setBodyReadError("The body of the HTTP Response is too large", bodyBytes, fields, tags) + return fields, tags, nil + } else if err != nil { + h.setBodyReadError(fmt.Sprintf("Failed to read body of HTTP Response : %s", err.Error()), bodyBytes, fields, tags) return fields, tags, nil } + // Add the body of the response if expected + if len(h.ResponseBodyField) > 0 { + // Check that the content of response contains only valid utf-8 characters. + if !utf8.Valid(bodyBytes) { + h.setBodyReadError("The body of the HTTP Response is not a valid utf-8 string", bodyBytes, fields, tags) + return fields, tags, nil + } + fields[h.ResponseBodyField] = string(bodyBytes) + } fields["content_length"] = len(bodyBytes) // Check the response for a regex match. @@ -339,6 +368,16 @@ func (h *HTTPResponse) httpGather(u string) (map[string]interface{}, map[string] return fields, tags, nil } +// Set result in case of a body read error +func (h *HTTPResponse) setBodyReadError(error_msg string, bodyBytes []byte, fields map[string]interface{}, tags map[string]string) { + h.Log.Debugf(error_msg) + setResult("body_read_error", fields, tags) + fields["content_length"] = len(bodyBytes) + if h.ResponseStringMatch != "" { + fields["response_string_match"] = 0 + } +} + // Gather gets all metric fields and tags and returns any errors it encounters func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error { // Compile the body regex if it exist diff --git a/plugins/inputs/http_response/http_response_test.go b/plugins/inputs/http_response/http_response_test.go index 9986ddefc..5a256e6e5 100644 --- a/plugins/inputs/http_response/http_response_test.go +++ b/plugins/inputs/http_response/http_response_test.go @@ -90,6 +90,9 @@ func setUpTestMux() http.Handler { w.Header().Set("Content-Type", "application/json; charset=utf-8") fmt.Fprintf(w, "hit the good page!") }) + mux.HandleFunc("/invalidUTF8", func(w http.ResponseWriter, req *http.Request) { + w.Write([]byte{0xff, 0xfe, 0xfd}) + }) mux.HandleFunc("/noheader", func(w http.ResponseWriter, req *http.Request) { fmt.Fprintf(w, "hit the good page!") }) @@ -223,6 +226,109 @@ func TestFields(t *testing.T) { checkOutput(t, &acc, expectedFields, expectedTags, absentFields, nil) } +func TestResponseBodyField(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Log: testutil.Logger{}, + Address: ts.URL + "/good", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: internal.Duration{Duration: time.Second * 20}, + Headers: map[string]string{ + "Content-Type": "application/json", + }, + ResponseBodyField: "my_body_field", + FollowRedirects: true, + } + + var acc testutil.Accumulator + err := h.Gather(&acc) + require.NoError(t, err) + + expectedFields := map[string]interface{}{ + "http_response_code": http.StatusOK, + "result_type": "success", + "result_code": 0, + "response_time": nil, + "content_length": nil, + "my_body_field": "hit the good page!", + } + expectedTags := map[string]interface{}{ + "server": nil, + "method": "GET", + "status_code": "200", + "result": "success", + } + absentFields := []string{"response_string_match"} + checkOutput(t, &acc, expectedFields, expectedTags, absentFields, nil) + + // Invalid UTF-8 String + h = &HTTPResponse{ + Log: testutil.Logger{}, + Address: ts.URL + "/invalidUTF8", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: internal.Duration{Duration: time.Second * 20}, + Headers: map[string]string{ + "Content-Type": "application/json", + }, + ResponseBodyField: "my_body_field", + FollowRedirects: true, + } + + acc = testutil.Accumulator{} + err = h.Gather(&acc) + require.NoError(t, err) + + expectedFields = map[string]interface{}{ + "result_type": "body_read_error", + "result_code": 2, + } + expectedTags = map[string]interface{}{ + "server": nil, + "method": "GET", + "result": "body_read_error", + } + checkOutput(t, &acc, expectedFields, expectedTags, nil, nil) +} + +func TestResponseBodyMaxSize(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Log: testutil.Logger{}, + Address: ts.URL + "/good", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: internal.Duration{Duration: time.Second * 20}, + Headers: map[string]string{ + "Content-Type": "application/json", + }, + ResponseBodyMaxSize: internal.Size{Size: 5}, + FollowRedirects: true, + } + + var acc testutil.Accumulator + err := h.Gather(&acc) + require.NoError(t, err) + + expectedFields := map[string]interface{}{ + "result_type": "body_read_error", + "result_code": 2, + } + expectedTags := map[string]interface{}{ + "server": nil, + "method": "GET", + "result": "body_read_error", + } + checkOutput(t, &acc, expectedFields, expectedTags, nil, nil) +} + func TestHTTPHeaderTags(t *testing.T) { mux := setUpTestMux() ts := httptest.NewServer(mux)