From d3a25e4dc1865f1e229d1629c68fcd5dc24d52a5 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Sat, 23 Apr 2016 11:42:28 -0600 Subject: [PATCH] globpath refactor into pkg separate from filestat --- internal/globpath/globpath.go | 93 ++++++++++++++++++++++++ internal/globpath/globpath_test.go | 49 +++++++++++++ internal/globpath/testdata/log1.log | 0 internal/globpath/testdata/log2.log | 0 internal/globpath/testdata/test.conf | 5 ++ plugins/inputs/filestat/filestat.go | 76 ++----------------- plugins/inputs/filestat/filestat_test.go | 17 ----- 7 files changed, 153 insertions(+), 87 deletions(-) create mode 100644 internal/globpath/globpath.go create mode 100644 internal/globpath/globpath_test.go create mode 100644 internal/globpath/testdata/log1.log create mode 100644 internal/globpath/testdata/log2.log create mode 100644 internal/globpath/testdata/test.conf diff --git a/internal/globpath/globpath.go b/internal/globpath/globpath.go new file mode 100644 index 000000000..729754063 --- /dev/null +++ b/internal/globpath/globpath.go @@ -0,0 +1,93 @@ +package globpath + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/gobwas/glob" +) + +var sepStr = fmt.Sprintf("%v", string(os.PathSeparator)) + +type GlobPath struct { + path string + hasMeta bool + g glob.Glob + root string +} + +func Compile(path string) (*GlobPath, error) { + out := GlobPath{ + hasMeta: hasMeta(path), + path: path, + } + + // if there are no glob meta characters in the path, don't bother compiling + // a glob object or finding the root directory. (see short-circuit in Match) + if !out.hasMeta { + return &out, nil + } + + var err error + if out.g, err = glob.Compile(path, os.PathSeparator); err != nil { + return nil, err + } + // Get the root directory for this filepath + out.root = findRootDir(path) + return &out, nil +} + +func (g *GlobPath) Match() []string { + if !g.hasMeta { + return []string{g.path} + } + return walkFilePath(g.root, g.g) +} + +// walk the filepath from the given root and return a list of files that match +// the given glob. +func walkFilePath(root string, g glob.Glob) []string { + matchedFiles := []string{} + walkfn := func(path string, _ os.FileInfo, _ error) error { + if g.Match(path) { + matchedFiles = append(matchedFiles, path) + } + return nil + } + filepath.Walk(root, walkfn) + return matchedFiles +} + +// find the root dir of the given path (could include globs). +// ie: +// /var/log/telegraf.conf -> /var/log +// /home/** -> /home +// /home/*/** -> /home +// /lib/share/*/*/**.txt -> /lib/share +func findRootDir(path string) string { + pathItems := strings.Split(path, sepStr) + out := sepStr + for i, item := range pathItems { + if i == len(pathItems)-1 { + break + } + if item == "" { + continue + } + if hasMeta(item) { + break + } + out += item + sepStr + } + if out != "/" { + out = strings.TrimSuffix(out, "/") + } + return out +} + +// hasMeta reports whether path contains any magic glob characters. +func hasMeta(path string) bool { + return strings.IndexAny(path, "*?[") >= 0 +} diff --git a/internal/globpath/globpath_test.go b/internal/globpath/globpath_test.go new file mode 100644 index 000000000..9c3fc16e0 --- /dev/null +++ b/internal/globpath/globpath_test.go @@ -0,0 +1,49 @@ +package globpath + +import ( + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCompileAndMatch(t *testing.T) { + dir := getTestdataDir() + g1, err := Compile(dir + "/**") + require.NoError(t, err) + g2, err := Compile(dir + "/*.log") + require.NoError(t, err) + g3, err := Compile(dir + "/log1.log") + require.NoError(t, err) + + matches := g1.Match() + assert.Len(t, matches, 3) + matches = g2.Match() + assert.Len(t, matches, 2) + matches = g3.Match() + assert.Len(t, matches, 1) +} + +func TestFindRootDir(t *testing.T) { + tests := []struct { + input string + output string + }{ + {"/var/log/telegraf.conf", "/var/log"}, + {"/home/**", "/home"}, + {"/home/*/**", "/home"}, + {"/lib/share/*/*/**.txt", "/lib/share"}, + } + + for _, test := range tests { + actual := findRootDir(test.input) + assert.Equal(t, test.output, actual) + } +} + +func getTestdataDir() string { + _, filename, _, _ := runtime.Caller(1) + return strings.Replace(filename, "globpath_test.go", "testdata", 1) +} diff --git a/internal/globpath/testdata/log1.log b/internal/globpath/testdata/log1.log new file mode 100644 index 000000000..e69de29bb diff --git a/internal/globpath/testdata/log2.log b/internal/globpath/testdata/log2.log new file mode 100644 index 000000000..e69de29bb diff --git a/internal/globpath/testdata/test.conf b/internal/globpath/testdata/test.conf new file mode 100644 index 000000000..a06111991 --- /dev/null +++ b/internal/globpath/testdata/test.conf @@ -0,0 +1,5 @@ +# this is a fake testing config file +# for testing the filestat plugin + +option1 = "foo" +option2 = "bar" diff --git a/plugins/inputs/filestat/filestat.go b/plugins/inputs/filestat/filestat.go index e280cbb1b..6e61a0a5f 100644 --- a/plugins/inputs/filestat/filestat.go +++ b/plugins/inputs/filestat/filestat.go @@ -5,17 +5,12 @@ import ( "fmt" "io" "os" - "path/filepath" - "strings" - - "github.com/gobwas/glob" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal/globpath" "github.com/influxdata/telegraf/plugins/inputs" ) -var sepStr = fmt.Sprintf("%v", string(os.PathSeparator)) - const sampleConfig = ` ## Files to gather stats about. ## These accept standard unix glob matching rules, but with the addition of @@ -29,16 +24,13 @@ type FileStat struct { Md5 bool Files []string - // maps full file paths to glob obj - globs map[string]glob.Glob - // maps full file paths to their root dir - roots map[string]string + // maps full file paths to globmatch obj + globs map[string]*globpath.GlobPath } func NewFileStat() *FileStat { return &FileStat{ - globs: make(map[string]glob.Glob), - roots: make(map[string]string), + globs: make(map[string]*globpath.GlobPath), } } @@ -56,27 +48,14 @@ func (f *FileStat) Gather(acc telegraf.Accumulator) error { // Get the compiled glob object for this filepath g, ok := f.globs[filepath] if !ok { - if g, err = glob.Compile(filepath, os.PathSeparator); err != nil { + if g, err = globpath.Compile(filepath); err != nil { errS += err.Error() + " " continue } f.globs[filepath] = g } - // Get the root directory for this filepath - root, ok := f.roots[filepath] - if !ok { - root = findRootDir(filepath) - f.roots[filepath] = root - } - var matches []string - // Do not walk file tree if we don't have to. - if !hasMeta(filepath) { - matches = []string{filepath} - } else { - matches = walkFilePath(f.roots[filepath], f.globs[filepath]) - } - for _, file := range matches { + for _, file := range g.Match() { tags := map[string]string{ "file": file, } @@ -118,20 +97,6 @@ func (f *FileStat) Gather(acc telegraf.Accumulator) error { return nil } -// walk the filepath from the given root and return a list of files that match -// the given glob. -func walkFilePath(root string, g glob.Glob) []string { - matchedFiles := []string{} - walkfn := func(path string, _ os.FileInfo, _ error) error { - if g.Match(path) { - matchedFiles = append(matchedFiles, path) - } - return nil - } - filepath.Walk(root, walkfn) - return matchedFiles -} - // Read given file and calculate an md5 hash. func getMd5(file string) (string, error) { of, err := os.Open(file) @@ -149,35 +114,6 @@ func getMd5(file string) (string, error) { return fmt.Sprintf("%x", hash.Sum(nil)), nil } -// find the root dir of the given path (could include globs). -// ie: -// /var/log/telegraf.conf -> /var/log/ -// /home/** -> /home/ -// /home/*/** -> /home/ -// /lib/share/*/*/**.txt -> /lib/share/ -func findRootDir(path string) string { - pathItems := strings.Split(path, sepStr) - outpath := sepStr - for i, item := range pathItems { - if i == len(pathItems)-1 { - break - } - if item == "" { - continue - } - if hasMeta(item) { - break - } - outpath += item + sepStr - } - return outpath -} - -// hasMeta reports whether path contains any magic glob characters. -func hasMeta(path string) bool { - return strings.IndexAny(path, "*?[") >= 0 -} - func init() { inputs.Add("filestat", func() telegraf.Input { return NewFileStat() diff --git a/plugins/inputs/filestat/filestat_test.go b/plugins/inputs/filestat/filestat_test.go index f8977c920..a404869d9 100644 --- a/plugins/inputs/filestat/filestat_test.go +++ b/plugins/inputs/filestat/filestat_test.go @@ -164,23 +164,6 @@ func TestGatherSuperAsterisk(t *testing.T) { acc.AssertContainsTaggedFields(t, "filestat", fields3, tags3) } -func TestFindRootDir(t *testing.T) { - tests := []struct { - input string - output string - }{ - {"/var/log/telegraf.conf", "/var/log/"}, - {"/home/**", "/home/"}, - {"/home/*/**", "/home/"}, - {"/lib/share/*/*/**.txt", "/lib/share/"}, - } - - for _, test := range tests { - actual := findRootDir(test.input) - assert.Equal(t, test.output, actual) - } -} - func TestGetMd5(t *testing.T) { dir := getTestdataDir() md5, err := getMd5(dir + "test.conf")