Support exclamation mark to create non-matching list in tail plugin (#8613)

* Replace exclamation mark with caret

* Update README and use table driven tests

* Use ReplaceAll instead

* Use doublestar package instead to glob filepath

* Add license

* Fix order of dependencies

* Doc improvement, maybe better then str replace?

* Forgot to remove nil from test

* Use regex instead of library

* Revert unnecessary change

* Go back to using library
replace string twice to handle edge case

docs/LICENSE_OF_DEPENDENCIES.md

@@ -36,6 +36,7 @@ following works:
 - github.com/aws/smithy-go [Apache License 2.0](https://github.com/aws/smithy-go/blob/main/LICENSE)
 - github.com/benbjohnson/clock [MIT License](https://github.com/benbjohnson/clock/blob/master/LICENSE)
 - github.com/beorn7/perks [MIT License](https://github.com/beorn7/perks/blob/master/LICENSE)
+- github.com/bmatcuk/doublestar [MIT License](https://github.com/bmatcuk/doublestar/blob/master/LICENSE)
 - github.com/caio/go-tdigest [MIT License](https://github.com/caio/go-tdigest/blob/master/LICENSE)
 - github.com/cenkalti/backoff [MIT License](https://github.com/cenkalti/backoff/blob/master/LICENSE)
 - github.com/cespare/xxhash [MIT License](https://github.com/cespare/xxhash/blob/master/LICENSE.txt)

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/aws/smithy-go v1.0.0
 	github.com/benbjohnson/clock v1.0.3
 	github.com/bitly/go-hostpool v0.1.0 // indirect
+	github.com/bmatcuk/doublestar/v3 v3.0.0
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869
 	github.com/caio/go-tdigest v2.3.0+incompatible // indirect
 	github.com/cenkalti/backoff v2.0.0+incompatible // indirect

go.sum

@@ -142,6 +142,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=
 github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
+github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkNo5WyHI=
+github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/caio/go-tdigest v2.3.0+incompatible h1:zP6nR0nTSUzlSqqr7F/LhslPlSZX/fZeGmgmwj2cxxY=

internal/globpath/globpath.go

@@ -5,8 +5,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/bmatcuk/doublestar/v3"
 	"github.com/gobwas/glob"
-	"github.com/karrick/godirwalk"
 )
 
 type GlobPath struct {
@@ -45,42 +45,13 @@ func Compile(path string) (*GlobPath, error) {
 // If it's a static path, returns path.
 // All returned path will have the host platform separator.
 func (g *GlobPath) Match() []string {
-	if !g.hasMeta {
+	// This string replacement is for backwards compatibility support
-		return []string{g.path}
+	// The original implemention allowed **.txt but the double star package requires **/**.txt
-	}
+	g.path = strings.ReplaceAll(g.path, "**/**", "**")
-	if !g.HasSuperMeta {
+	g.path = strings.ReplaceAll(g.path, "**", "**/**")
-		files, _ := filepath.Glob(g.path)
-		return files
-	}
-	roots, err := filepath.Glob(g.rootGlob)
-	if err != nil {
-		return []string{}
-	}
-	out := []string{}
-	walkfn := func(path string, _ *godirwalk.Dirent) error {
-		if g.g.Match(path) {
-			out = append(out, path)
-		}
-		return nil
 
-	}
+	files, _ := doublestar.Glob(g.path)
-	for _, root := range roots {
+	return files
-		fileinfo, err := os.Stat(root)
-		if err != nil {
-			continue
-		}
-		if !fileinfo.IsDir() {
-			if g.MatchString(root) {
-				out = append(out, root)
-			}
-			continue
-		}
-		godirwalk.Walk(root, &godirwalk.Options{
-			Callback: walkfn,
-			Unsorted: true,
-		})
-	}
-	return out
 }
 
 // MatchString tests the path string against the glob.  The path should contain

internal/globpath/globpath_test.go

@@ -19,32 +19,41 @@ var (
 )
 
 func TestCompileAndMatch(t *testing.T) {
-	// test super asterisk
-	g1, err := Compile(filepath.Join(testdataDir, "**"))
-	require.NoError(t, err)
-	// test single asterisk
-	g2, err := Compile(filepath.Join(testdataDir, "*.log"))
-	require.NoError(t, err)
-	// test no meta characters (file exists)
-	g3, err := Compile(filepath.Join(testdataDir, "log1.log"))
-	require.NoError(t, err)
-	// test file that doesn't exist
-	g4, err := Compile(filepath.Join(testdataDir, "i_dont_exist.log"))
-	require.NoError(t, err)
-	// test super asterisk that doesn't exist
-	g5, err := Compile(filepath.Join(testdataDir, "dir_doesnt_exist", "**"))
-	require.NoError(t, err)
 
-	matches := g1.Match()
+	type test struct {
-	require.Len(t, matches, 6)
+		path    string
-	matches = g2.Match()
+		matches int
-	require.Len(t, matches, 2)
+	}
-	matches = g3.Match()
+
-	require.Len(t, matches, 1)
+	tests := []test{
-	matches = g4.Match()
+		//test super asterisk
-	require.Len(t, matches, 1)
+		{path: filepath.Join(testdataDir, "**"), matches: 7},
-	matches = g5.Match()
+		// test single asterisk
-	require.Len(t, matches, 0)
+		{path: filepath.Join(testdataDir, "*.log"), matches: 3},
+		// test no meta characters (file exists)
+		{path: filepath.Join(testdataDir, "log1.log"), matches: 1},
+		// test file that doesn't exist
+		{path: filepath.Join(testdataDir, "i_dont_exist.log"), matches: 0},
+		// test super asterisk that doesn't exist
+		{path: filepath.Join(testdataDir, "dir_doesnt_exist", "**"), matches: 0},
+		// test exclamation mark creates non-matching list with a range
+		{path: filepath.Join(testdataDir, "log[!1-2]*"), matches: 1},
+		// test caret creates non-matching list
+		{path: filepath.Join(testdataDir, "log[^1-2]*"), matches: 1},
+		// test exclamation mark creates non-matching list without a range
+		{path: filepath.Join(testdataDir, "log[!2]*"), matches: 2},
+		// test exclamation mark creates non-matching list without a range
+		{path: filepath.Join(testdataDir, "log\\[!*"), matches: 1},
+		// test exclamation mark creates non-matching list without a range
+		{path: filepath.Join(testdataDir, "log\\[^*"), matches: 0},
+	}
+
+	for _, tc := range tests {
+		g, err := Compile(tc.path)
+		require.Nil(t, err)
+		matches := g.Match()
+		require.Len(t, matches, tc.matches)
+	}
 }
 
 func TestRootGlob(t *testing.T) {
@@ -82,7 +91,7 @@ func TestMatch_ErrPermission(t *testing.T) {
 		input    string
 		expected []string
 	}{
-		{"/root/foo", []string{"/root/foo"}},
+		{"/root/foo", []string(nil)},
 		{"/root/f*", []string(nil)},
 	}
 

plugins/inputs/phpfpm/phpfpm.go

@@ -299,7 +299,7 @@ func globUnixSocket(url string) ([]string, error) {
 	}
 	paths := glob.Match()
 	if len(paths) == 0 {
-		return nil, fmt.Errorf("socket doesn't exist %q: %v", pattern, err)
+		return nil, fmt.Errorf("socket doesn't exist %q", pattern)
 	}
 
 	addresses := make([]string, 0, len(paths))

plugins/inputs/phpfpm/phpfpm_test.go

@@ -327,7 +327,7 @@ func TestPhpFpmGeneratesMetrics_Throw_Error_When_Socket_Path_Is_Invalid(t *testi
 
 	err = acc.GatherError(r.Gather)
 	require.Error(t, err)
-	assert.Equal(t, `dial unix /tmp/invalid.sock: connect: no such file or directory`, err.Error())
+	assert.Equal(t, `socket doesn't exist "/tmp/invalid.sock"`, err.Error())
 
 }
 

plugins/inputs/tail/README.md

@@ -29,7 +29,8 @@ The plugin expects messages in one of the
   ##   "/var/log/**.log"  -> recursively find all .log files in /var/log
   ##   "/var/log/*/*.log" -> find all .log files with a parent dir in /var/log
   ##   "/var/log/apache.log" -> just tail the apache log file
-  ##
+  ##   "/var/log/log[!1-2]*  -> tail files without 1-2
+  ##   "/var/log/log[^1-2]*  -> identical behavior as above
   ## See https://github.com/gobwas/glob for more examples
   ##
   files = ["/var/mymetrics.out"]

plugins/inputs/tail/tail.go

@@ -81,7 +81,8 @@ const sampleConfig = `
   ##   "/var/log/**.log"  -> recursively find all .log files in /var/log
   ##   "/var/log/*/*.log" -> find all .log files with a parent dir in /var/log
   ##   "/var/log/apache.log" -> just tail the apache log file
-  ##
+  ##   "/var/log/log[!1-2]*  -> tail files without 1-2
+  ##   "/var/log/log[^1-2]*  -> identical behavior as above
   ## See https://github.com/gobwas/glob for more examples
   ##
   files = ["/var/mymetrics.out"]