Http plugin add cookie auth (#9395)

plugins/common/cookie/cookie.go

@@ -0,0 +1,95 @@
+package cookie
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/cookiejar"
+	"strings"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+)
+
+type CookieAuthConfig struct {
+	URL    string `toml:"cookie_auth_url"`
+	Method string `toml:"cookie_auth_method"`
+
+	// HTTP Basic Auth Credentials
+	Username string `toml:"cookie_auth_username"`
+	Password string `toml:"cookie_auth_password"`
+
+	Body    string          `toml:"cookie_auth_body"`
+	Renewal config.Duration `toml:"cookie_auth_renewal"`
+
+	client *http.Client
+}
+
+func (c *CookieAuthConfig) Start(client *http.Client, log telegraf.Logger) (err error) {
+	c.client = client
+
+	if c.Method == "" {
+		c.Method = http.MethodPost
+	}
+
+	// add cookie jar to HTTP client
+	if c.client.Jar, err = cookiejar.New(nil); err != nil {
+		return err
+	}
+
+	if err = c.auth(); err != nil {
+		return err
+	}
+
+	// continual auth renewal if set
+	if c.Renewal > 0 {
+		ticker := time.NewTicker(time.Duration(c.Renewal))
+		go func() {
+			for range ticker.C {
+				if err := c.auth(); err != nil && log != nil {
+					log.Errorf("renewal failed for %q: %v", c.URL, err)
+				}
+			}
+		}()
+	}
+
+	return nil
+}
+
+func (c *CookieAuthConfig) auth() error {
+	var body io.ReadCloser
+	if c.Body != "" {
+		body = ioutil.NopCloser(strings.NewReader(c.Body))
+		defer body.Close()
+	}
+
+	req, err := http.NewRequest(c.Method, c.URL, body)
+	if err != nil {
+		return err
+	}
+
+	if c.Username != "" {
+		req.SetBasicAuth(c.Username, c.Password)
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if _, err = io.Copy(ioutil.Discard, resp.Body); err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("cookie auth renewal received status code: %v (%v)",
+			resp.StatusCode,
+			http.StatusText(resp.StatusCode),
+		)
+	}
+
+	return nil
+}

plugins/common/cookie/cookie_test.go

@@ -0,0 +1,269 @@
+package cookie_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/plugins/common/cookie"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	reqUser   = "testUser"
+	reqPasswd = "testPassword"
+	reqBody   = "a body"
+
+	authEndpointNoCreds                   = "/auth"
+	authEndpointWithBasicAuth             = "/authWithCreds"
+	authEndpointWithBasicAuthOnlyUsername = "/authWithCredsUser"
+	authEndpointWithBody                  = "/authWithBody"
+)
+
+var fakeCookie = &http.Cookie{
+	Name:  "test-cookie",
+	Value: "this is an auth cookie",
+}
+
+type fakeServer struct {
+	*httptest.Server
+	*int32
+}
+
+func newFakeServer(t *testing.T) fakeServer {
+	var c int32
+	return fakeServer{
+		Server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			authed := func() {
+				atomic.AddInt32(&c, 1)        // increment auth counter
+				http.SetCookie(w, fakeCookie) // set fake cookie
+			}
+			switch r.URL.Path {
+			case authEndpointNoCreds:
+				authed()
+			case authEndpointWithBody:
+				body, err := ioutil.ReadAll(r.Body)
+				require.NoError(t, err)
+				if !cmp.Equal([]byte(reqBody), body) {
+					w.WriteHeader(http.StatusUnauthorized)
+					return
+				}
+				authed()
+			case authEndpointWithBasicAuth:
+				u, p, ok := r.BasicAuth()
+				if !ok || u != reqUser || p != reqPasswd {
+					w.WriteHeader(http.StatusUnauthorized)
+					return
+				}
+				authed()
+			case authEndpointWithBasicAuthOnlyUsername:
+				u, p, ok := r.BasicAuth()
+				if !ok || u != reqUser || p != "" {
+					w.WriteHeader(http.StatusUnauthorized)
+					return
+				}
+				authed()
+			default:
+				// ensure cookie exists on request
+				if _, err := r.Cookie(fakeCookie.Name); err != nil {
+					w.WriteHeader(http.StatusForbidden)
+					return
+				}
+				_, _ = w.Write([]byte("good test response"))
+			}
+		})),
+		int32: &c,
+	}
+}
+
+func (s fakeServer) checkResp(t *testing.T, expCode int) {
+	t.Helper()
+	resp, err := s.Client().Get(s.URL + "/endpoint")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	require.Equal(t, expCode, resp.StatusCode)
+
+	if expCode == http.StatusOK {
+		require.Len(t, resp.Request.Cookies(), 1)
+		require.Equal(t, "test-cookie", resp.Request.Cookies()[0].Name)
+	}
+}
+
+func (s fakeServer) checkAuthCount(t *testing.T, atLeast int32) {
+	t.Helper()
+	require.GreaterOrEqual(t, atomic.LoadInt32(s.int32), atLeast)
+}
+
+func TestAuthConfig_Start(t *testing.T) {
+	const (
+		renewal      = 50 * time.Millisecond
+		renewalCheck = 5 * renewal
+	)
+	type fields struct {
+		Method   string
+		Username string
+		Password string
+		Body     string
+	}
+	type args struct {
+		renewal  time.Duration
+		endpoint string
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		wantErr error
+		assert  func(t *testing.T, c *cookie.CookieAuthConfig, srv fakeServer)
+	}{
+		{
+			name: "zero renewal does not renew",
+			args: args{
+				renewal:  0,
+				endpoint: authEndpointNoCreds,
+			},
+			assert: func(t *testing.T, c *cookie.CookieAuthConfig, srv fakeServer) {
+				// should have Cookie Authed once
+				srv.checkAuthCount(t, 1)
+				srv.checkResp(t, http.StatusOK)
+				time.Sleep(renewalCheck)
+				// should have never Cookie Authed again
+				srv.checkAuthCount(t, 1)
+				srv.checkResp(t, http.StatusOK)
+			},
+		},
+		{
+			name: "success no creds, no body, default method",
+			args: args{
+				renewal:  renewal,
+				endpoint: authEndpointNoCreds,
+			},
+			assert: func(t *testing.T, c *cookie.CookieAuthConfig, srv fakeServer) {
+				// should have Cookie Authed once
+				srv.checkAuthCount(t, 1)
+				// default method set
+				require.Equal(t, http.MethodPost, c.Method)
+				srv.checkResp(t, http.StatusOK)
+				time.Sleep(renewalCheck)
+				// should have Cookie Authed at least twice more
+				srv.checkAuthCount(t, 3)
+				srv.checkResp(t, http.StatusOK)
+			},
+		},
+		{
+			name: "success with creds, no body",
+			fields: fields{
+				Method:   http.MethodPost,
+				Username: reqUser,
+				Password: reqPasswd,
+			},
+			args: args{
+				renewal:  renewal,
+				endpoint: authEndpointWithBasicAuth,
+			},
+			assert: func(t *testing.T, c *cookie.CookieAuthConfig, srv fakeServer) {
+				// should have Cookie Authed once
+				srv.checkAuthCount(t, 1)
+				srv.checkResp(t, http.StatusOK)
+				time.Sleep(renewalCheck)
+				// should have Cookie Authed at least twice more
+				srv.checkAuthCount(t, 3)
+				srv.checkResp(t, http.StatusOK)
+			},
+		},
+		{
+			name: "failure with bad creds",
+			fields: fields{
+				Method:   http.MethodPost,
+				Username: reqUser,
+				Password: "a bad password",
+			},
+			args: args{
+				renewal:  renewal,
+				endpoint: authEndpointWithBasicAuth,
+			},
+			wantErr: fmt.Errorf("cookie auth renewal received status code: 401 (Unauthorized)"),
+			assert: func(t *testing.T, c *cookie.CookieAuthConfig, srv fakeServer) {
+				// should have never Cookie Authed
+				srv.checkAuthCount(t, 0)
+				srv.checkResp(t, http.StatusForbidden)
+				time.Sleep(renewalCheck)
+				// should have still never Cookie Authed
+				srv.checkAuthCount(t, 0)
+				srv.checkResp(t, http.StatusForbidden)
+			},
+		},
+		{
+			name: "success with no creds, with good body",
+			fields: fields{
+				Method: http.MethodPost,
+				Body:   reqBody,
+			},
+			args: args{
+				renewal:  renewal,
+				endpoint: authEndpointWithBody,
+			},
+			assert: func(t *testing.T, c *cookie.CookieAuthConfig, srv fakeServer) {
+				// should have Cookie Authed once
+				srv.checkAuthCount(t, 1)
+				srv.checkResp(t, http.StatusOK)
+				time.Sleep(renewalCheck)
+				// should have Cookie Authed at least twice more
+				srv.checkAuthCount(t, 3)
+				srv.checkResp(t, http.StatusOK)
+			},
+		},
+		{
+			name: "failure with bad body",
+			fields: fields{
+				Method: http.MethodPost,
+				Body:   "a bad body",
+			},
+			args: args{
+				renewal:  renewal,
+				endpoint: authEndpointWithBody,
+			},
+			wantErr: fmt.Errorf("cookie auth renewal received status code: 401 (Unauthorized)"),
+			assert: func(t *testing.T, c *cookie.CookieAuthConfig, srv fakeServer) {
+				// should have never Cookie Authed
+				srv.checkAuthCount(t, 0)
+				srv.checkResp(t, http.StatusForbidden)
+				time.Sleep(renewalCheck)
+				// should have still never Cookie Authed
+				srv.checkAuthCount(t, 0)
+				srv.checkResp(t, http.StatusForbidden)
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			srv := newFakeServer(t)
+			c := &cookie.CookieAuthConfig{
+				URL:      srv.URL + tt.args.endpoint,
+				Method:   tt.fields.Method,
+				Username: tt.fields.Username,
+				Password: tt.fields.Password,
+				Body:     tt.fields.Body,
+				Renewal:  config.Duration(tt.args.renewal),
+			}
+
+			if err := c.Start(srv.Client(), testutil.Logger{Name: "cookie_auth"}); tt.wantErr != nil {
+				require.EqualError(t, err, tt.wantErr.Error())
+			} else {
+				require.NoError(t, err)
+			}
+
+			if tt.assert != nil {
+				tt.assert(t, c, srv)
+			}
+		})
+	}
+}

plugins/common/http/config.go

@@ -5,7 +5,9 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/plugins/common/cookie"
 	oauthConfig "github.com/influxdata/telegraf/plugins/common/oauth"
 	"github.com/influxdata/telegraf/plugins/common/proxy"
 	"github.com/influxdata/telegraf/plugins/common/tls"
@@ -19,9 +21,10 @@ type HTTPClientConfig struct {
 	proxy.HTTPProxy
 	tls.ClientConfig
 	oauthConfig.OAuth2Config
+	cookie.CookieAuthConfig
 }
 
-func (h *HTTPClientConfig) CreateClient(ctx context.Context) (*http.Client, error) {
+func (h *HTTPClientConfig) CreateClient(ctx context.Context, log telegraf.Logger) (*http.Client, error) {
 	tlsCfg, err := h.ClientConfig.TLSConfig()
 	if err != nil {
 		return nil, err
@@ -50,5 +53,11 @@ func (h *HTTPClientConfig) CreateClient(ctx context.Context) (*http.Client, erro
 
 	client = h.OAuth2Config.CreateOauth2Client(ctx, client)
 
+	if h.CookieAuthConfig.URL != "" {
+		if err := h.CookieAuthConfig.Start(client, log); err != nil {
+			return nil, err
+		}
+	}
+
 	return client, nil
 }

plugins/inputs/http/README.md

@@ -50,6 +50,15 @@ The HTTP input plugin collects metrics from one or more HTTP(S) endpoints.  The
   ## Use TLS but skip chain & host verification
   # insecure_skip_verify = false
 
+  ## Optional Cookie authentication
+  # cookie_auth_url = "https://localhost/authMe"
+  # cookie_auth_method = "POST"
+  # cookie_auth_username = "username"
+  # cookie_auth_password = "pa$$word"
+  # cookie_auth_body = '{"username": "user", "password": "pa$$word", "authenticate": "me"}'
+  ## cookie_auth_renewal not set or set to "0" will auth once and never renew the cookie
+  # cookie_auth_renewal = "5m"
+
   ## Amount of time allowed to complete the HTTP request
   # timeout = "5s"
 
@@ -73,3 +82,7 @@ The default values below are added if the input format does not specify a value:
 - http
   - tags:
     - url
+
+### Optional Cookie Authentication Settings:
+
+The optional Cookie Authentication Settings will retrieve a cookie from the given authorization endpoint, and use it in subsequent API requests.  This is useful for services that do not provide OAuth or Basic Auth authentication, e.g. the [Tesla Powerwall API](https://www.tesla.com/support/energy/powerwall/own/monitoring-from-home-network), which uses a Cookie Auth Body to retrieve an authorization cookie.  The Cookie Auth Renewal interval will renew the authorization by retrieving a new cookie at the given interval.

plugins/inputs/http/http.go

@@ -35,6 +35,7 @@ type HTTP struct {
 
 	client *http.Client
 	httpconfig.HTTPClientConfig
+	Log telegraf.Logger `toml:"-"`
 
 	// The parser will automatically be set by Telegraf core code because
 	// this plugin implements the ParserInput interface (i.e. the SetParser method)
@@ -84,6 +85,15 @@ var sampleConfig = `
   ## Use TLS but skip chain & host verification
   # insecure_skip_verify = false
 
+  ## Optional Cookie authentication
+  # cookie_auth_url = "https://localhost/authMe"
+  # cookie_auth_method = "POST"
+  # cookie_auth_username = "username"
+  # cookie_auth_password = "pa$$word"
+  # cookie_auth_body = '{"username": "user", "password": "pa$$word", "authenticate": "me"}'
+  ## cookie_auth_renewal not set or set to "0" will auth once and never renew the cookie
+  # cookie_auth_renewal = "5m"
+
   ## Amount of time allowed to complete the HTTP request
   # timeout = "5s"
 
@@ -109,7 +119,7 @@ func (*HTTP) Description() string {
 
 func (h *HTTP) Init() error {
 	ctx := context.Background()
-	client, err := h.HTTPClientConfig.CreateClient(ctx)
+	client, err := h.HTTPClientConfig.CreateClient(ctx, h.Log)
 	if err != nil {
 		return err
 	}

plugins/outputs/http/README.md

@@ -34,6 +34,15 @@ data formats. For data_formats that support batching, metrics are sent in batch
   ## Use TLS but skip chain & host verification
   # insecure_skip_verify = false
 
+  ## Optional Cookie authentication
+  # cookie_auth_url = "https://localhost/authMe"
+  # cookie_auth_method = "POST"
+  # cookie_auth_username = "username"
+  # cookie_auth_password = "pa$$word"
+  # cookie_auth_body = '{"username": "user", "password": "pa$$word", "authenticate": "me"}'
+  ## cookie_auth_renewal not set or set to "0" will auth once and never renew the cookie
+  # cookie_auth_renewal = "5m"
+
   ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
@@ -54,3 +63,7 @@ data formats. For data_formats that support batching, metrics are sent in batch
   ## Zero means no limit.
   # idle_conn_timeout = 0
 ```
+
+### Optional Cookie Authentication Settings:
+
+The optional Cookie Authentication Settings will retrieve a cookie from the given authorization endpoint, and use it in subsequent API requests.  This is useful for services that do not provide OAuth or Basic Auth authentication, e.g. the [Tesla Powerwall API](https://www.tesla.com/support/energy/powerwall/own/monitoring-from-home-network), which uses a Cookie Auth Body to retrieve an authorization cookie.  The Cookie Auth Renewal interval will renew the authorization by retrieving a new cookie at the given interval.

plugins/outputs/http/http.go

@@ -48,6 +48,15 @@ var sampleConfig = `
   ## Use TLS but skip chain & host verification
   # insecure_skip_verify = false
 
+  ## Optional Cookie authentication
+  # cookie_auth_url = "https://localhost/authMe"
+  # cookie_auth_method = "POST"
+  # cookie_auth_username = "username"
+  # cookie_auth_password = "pa$$word"
+  # cookie_auth_body = '{"username": "user", "password": "pa$$word", "authenticate": "me"}'
+  ## cookie_auth_renewal not set or set to "0" will auth once and never renew the cookie
+  # cookie_auth_renewal = "5m"
+
   ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
@@ -83,6 +92,7 @@ type HTTP struct {
 	Headers         map[string]string `toml:"headers"`
 	ContentEncoding string            `toml:"content_encoding"`
 	httpconfig.HTTPClientConfig
+	Log telegraf.Logger `toml:"-"`
 
 	client     *http.Client
 	serializer serializers.Serializer
@@ -102,7 +112,7 @@ func (h *HTTP) Connect() error {
 	}
 
 	ctx := context.Background()
-	client, err := h.HTTPClientConfig.CreateClient(ctx)
+	client, err := h.HTTPClientConfig.CreateClient(ctx, h.Log)
 	if err != nil {
 		return err
 	}