CL-Softwares
/
telegraf
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add ability to collect response body as field with http_response (#7596)

This commit is contained in:
Nicolas Filotto 2020-06-09 00:29:34 +02:00 committed by GitHub
parent 52f1410bfa
commit bf0f674458
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 166 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
plugins/inputs/http_response/README.md
View File

@ -31,6 +31,15 @@ This input plugin checks HTTP/HTTPS connections.
# {'fake':'data'}
# '''
## Optional name of the field that will contain the body of the response.
## By default it is set to an empty String indicating that the body's content won't be added
# response_body_field = ''
## Maximum allowed HTTP response body size in bytes.
## 0 means to use the default of 32MiB.
## If the response body size exceeds this limit a "body_read_error" will be raised
# response_body_max_size = "32MiB"
## Optional substring or regex match in body of the response (case sensitive)
# response_string_match = "\"service_status\": \"up\""
# response_string_match = "ok"
@ -47,7 +56,7 @@ This input plugin checks HTTP/HTTPS connections.
# [inputs.http_response.headers]
# Host = "github.com"
## Optional setting to map reponse http headers into tags
## Optional setting to map response http headers into tags
## If the http header is not present on the request, no corresponding tag will be added
## If multiple instances of the http header are present, only the first value will be used
# http_header_tags = {"HTTP_HEADER" = "TAG_NAME"}
@ -82,7 +91,7 @@ This tag is used to expose network and plugin errors. HTTP errors are considered
--------------------------|-------------------------|-----------|
|success | 0 |The HTTP request completed, even if the HTTP code represents an error|
|response_string_mismatch | 1 |The option `response_string_match` was used, and the body of the response didn't match the regex. HTTP errors with content in their body (like 4xx, 5xx) will trigger this error|
|body_read_error | 2 |The option `response_string_match` was used, but the plugin wasn't able to read the body of the response. Responses with empty bodies (like 3xx, HEAD, etc) will trigger this error|
|body_read_error | 2 |The option `response_string_match` was used, but the plugin wasn't able to read the body of the response. Responses with empty bodies (like 3xx, HEAD, etc) will trigger this error. Or the option `response_body_field` was used and the content of the response body was not a valid uft-8. Or the size of the body of the response exceeded the `response_body_max_size` |
|connection_failed | 3 |Catch all for any network error not specifically handled by the plugin|
|timeout | 4 |The plugin timed out while awaiting the HTTP connection to complete|
|dns_error | 5 |There was a DNS error while attempting to connect to the host|

59
plugins/inputs/http_response/http_response.go
View File

@ -12,6 +12,7 @@ import (
"strconv"
"strings"
"time"
"unicode/utf8"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
@ -19,6 +20,12 @@ import (
"github.com/influxdata/telegraf/plugins/inputs"
)
const (
// defaultResponseBodyMaxSize is the default maximum response body size, in bytes.
// if the response body is over this size, we will raise a body_read_error.
defaultResponseBodyMaxSize = 32 * 1024 * 1024
)
// HTTPResponse struct
type HTTPResponse struct {
Address string // deprecated in 1.12
@ -31,7 +38,9 @@ type HTTPResponse struct {
Headers map[string]string
FollowRedirects bool
// Absolute path to file with Bearer token
BearerToken string `toml:"bearer_token"`
BearerToken string `toml:"bearer_token"`
ResponseBodyField string `toml:"response_body_field"`
ResponseBodyMaxSize internal.Size `toml:"response_body_max_size"`
ResponseStringMatch string
Interface string
// HTTP Basic Auth Credentials
@ -83,6 +92,15 @@ var sampleConfig = `
# {'fake':'data'}
# '''
## Optional name of the field that will contain the body of the response.
## By default it is set to an empty String indicating that the body's content won't be added
# response_body_field = ''
## Maximum allowed HTTP response body size in bytes.
## 0 means to use the default of 32MiB.
## If the response body size exceeds this limit a "body_read_error" will be raised
# response_body_max_size = "32MiB"
## Optional substring or regex match in body of the response
# response_string_match = "\"service_status\": \"up\""
# response_string_match = "ok"
@ -99,7 +117,7 @@ var sampleConfig = `
# [inputs.http_response.headers]
# Host = "github.com"
## Optional setting to map reponse http headers into tags
## Optional setting to map response http headers into tags
## If the http header is not present on the request, no corresponding tag will be added
## If multiple instances of the http header are present, only the first value will be used
# http_header_tags = {"HTTP_HEADER" = "TAG_NAME"}
@ -310,17 +328,28 @@ func (h *HTTPResponse) httpGather(u string) (map[string]interface{}, map[string]
tags["status_code"] = strconv.Itoa(resp.StatusCode)
fields["http_response_code"] = resp.StatusCode
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
h.Log.Debugf("Failed to read body of HTTP Response : %s", err.Error())
setResult("body_read_error", fields, tags)
fields["content_length"] = len(bodyBytes)
if h.ResponseStringMatch != "" {
fields["response_string_match"] = 0
}
if h.ResponseBodyMaxSize.Size == 0 {
h.ResponseBodyMaxSize.Size = defaultResponseBodyMaxSize
}
bodyBytes, err := ioutil.ReadAll(io.LimitReader(resp.Body, h.ResponseBodyMaxSize.Size+1))
// Check first if the response body size exceeds the limit.
if err == nil && int64(len(bodyBytes)) > h.ResponseBodyMaxSize.Size {
h.setBodyReadError("The body of the HTTP Response is too large", bodyBytes, fields, tags)
return fields, tags, nil
} else if err != nil {
h.setBodyReadError(fmt.Sprintf("Failed to read body of HTTP Response : %s", err.Error()), bodyBytes, fields, tags)
return fields, tags, nil
}
// Add the body of the response if expected
if len(h.ResponseBodyField) > 0 {
// Check that the content of response contains only valid utf-8 characters.
if !utf8.Valid(bodyBytes) {
h.setBodyReadError("The body of the HTTP Response is not a valid utf-8 string", bodyBytes, fields, tags)
return fields, tags, nil
}
fields[h.ResponseBodyField] = string(bodyBytes)
}
fields["content_length"] = len(bodyBytes)
// Check the response for a regex match.
@ -339,6 +368,16 @@ func (h *HTTPResponse) httpGather(u string) (map[string]interface{}, map[string]
return fields, tags, nil
}
// Set result in case of a body read error
func (h *HTTPResponse) setBodyReadError(error_msg string, bodyBytes []byte, fields map[string]interface{}, tags map[string]string) {
h.Log.Debugf(error_msg)
setResult("body_read_error", fields, tags)
fields["content_length"] = len(bodyBytes)
if h.ResponseStringMatch != "" {
fields["response_string_match"] = 0
}
}
// Gather gets all metric fields and tags and returns any errors it encounters
func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error {
// Compile the body regex if it exist

106
plugins/inputs/http_response/http_response_test.go
View File

@ -90,6 +90,9 @@ func setUpTestMux() http.Handler {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
fmt.Fprintf(w, "hit the good page!")
})
mux.HandleFunc("/invalidUTF8", func(w http.ResponseWriter, req *http.Request) {
w.Write([]byte{0xff, 0xfe, 0xfd})
})
mux.HandleFunc("/noheader", func(w http.ResponseWriter, req *http.Request) {
fmt.Fprintf(w, "hit the good page!")
})
@ -223,6 +226,109 @@ func TestFields(t *testing.T) {
checkOutput(t, &acc, expectedFields, expectedTags, absentFields, nil)
}
func TestResponseBodyField(t *testing.T) {
mux := setUpTestMux()
ts := httptest.NewServer(mux)
defer ts.Close()
h := &HTTPResponse{
Log: testutil.Logger{},
Address: ts.URL + "/good",
Body: "{ 'test': 'data'}",
Method: "GET",
ResponseTimeout: internal.Duration{Duration: time.Second * 20},
Headers: map[string]string{
"Content-Type": "application/json",
},
ResponseBodyField: "my_body_field",
FollowRedirects: true,
}
var acc testutil.Accumulator
err := h.Gather(&acc)
require.NoError(t, err)
expectedFields := map[string]interface{}{
"http_response_code": http.StatusOK,
"result_type": "success",
"result_code": 0,
"response_time": nil,
"content_length": nil,
"my_body_field": "hit the good page!",
}
expectedTags := map[string]interface{}{
"server": nil,
"method": "GET",
"status_code": "200",
"result": "success",
}
absentFields := []string{"response_string_match"}
checkOutput(t, &acc, expectedFields, expectedTags, absentFields, nil)
// Invalid UTF-8 String
h = &HTTPResponse{
Log: testutil.Logger{},
Address: ts.URL + "/invalidUTF8",
Body: "{ 'test': 'data'}",
Method: "GET",
ResponseTimeout: internal.Duration{Duration: time.Second * 20},
Headers: map[string]string{
"Content-Type": "application/json",
},
ResponseBodyField: "my_body_field",
FollowRedirects: true,
}
acc = testutil.Accumulator{}
err = h.Gather(&acc)
require.NoError(t, err)
expectedFields = map[string]interface{}{
"result_type": "body_read_error",
"result_code": 2,
}
expectedTags = map[string]interface{}{
"server": nil,
"method": "GET",
"result": "body_read_error",
}
checkOutput(t, &acc, expectedFields, expectedTags, nil, nil)
}
func TestResponseBodyMaxSize(t *testing.T) {
mux := setUpTestMux()
ts := httptest.NewServer(mux)
defer ts.Close()
h := &HTTPResponse{
Log: testutil.Logger{},
Address: ts.URL + "/good",
Body: "{ 'test': 'data'}",
Method: "GET",
ResponseTimeout: internal.Duration{Duration: time.Second * 20},
Headers: map[string]string{
"Content-Type": "application/json",
},
ResponseBodyMaxSize: internal.Size{Size: 5},
FollowRedirects: true,
}
var acc testutil.Accumulator
err := h.Gather(&acc)
require.NoError(t, err)
expectedFields := map[string]interface{}{
"result_type": "body_read_error",
"result_code": 2,
}
expectedTags := map[string]interface{}{
"server": nil,
"method": "GET",
"result": "body_read_error",
}
checkOutput(t, &acc, expectedFields, expectedTags, nil, nil)
}
func TestHTTPHeaderTags(t *testing.T) {
mux := setUpTestMux()
ts := httptest.NewServer(mux)