From c98115e7442253da5a1633d1afaa5db24c69bd1d Mon Sep 17 00:00:00 2001 From: Sven Rebhan <36194019+srebhan@users.noreply.github.com> Date: Thu, 8 Dec 2022 17:53:06 +0100 Subject: [PATCH] feat: secret-store implementation (#11232) --- Makefile | 2 +- cmd/telegraf/cmd_secretstore.go | 258 +++++++++ cmd/telegraf/main.go | 34 +- cmd/telegraf/main_test.go | 80 +++ cmd/telegraf/printer.go | 67 ++- cmd/telegraf/telegraf.go | 73 ++- config/config.go | 124 ++++- config/secret.go | 197 +++++++ config/secret_test.go | 561 ++++++++++++++++++++ config/secret_with_mlock.go | 20 + config/secret_without_mlock.go | 15 + config/util.go | 23 + docs/CONFIGURATION.md | 39 ++ docs/LICENSE_OF_DEPENDENCIES.md | 8 + go.mod | 26 +- go.sum | 19 + plugins/inputs/http/http.go | 26 +- plugins/inputs/sql/sql.go | 29 +- plugins/inputs/sql/sql_test.go | 60 +-- plugins/secretstores/all/all.go | 1 + plugins/secretstores/all/jose.go | 5 + plugins/secretstores/all/os.go | 5 + plugins/secretstores/deprecations.go | 6 + plugins/secretstores/jose/README.md | 38 ++ plugins/secretstores/jose/jose.go | 108 ++++ plugins/secretstores/jose/jose_test.go | 204 +++++++ plugins/secretstores/jose/sample.conf | 13 + plugins/secretstores/os/README.md | 115 ++++ plugins/secretstores/os/os.go | 97 ++++ plugins/secretstores/os/os_darwin.go | 39 ++ plugins/secretstores/os/os_linux.go | 27 + plugins/secretstores/os/os_test.go | 89 ++++ plugins/secretstores/os/os_unsupported.go | 1 + plugins/secretstores/os/os_windows.go | 23 + plugins/secretstores/os/sample_darwin.conf | 17 + plugins/secretstores/os/sample_linux.conf | 12 + plugins/secretstores/os/sample_windows.conf | 15 + plugins/secretstores/registry.go | 16 + secretstore.go | 25 + tools/custom_builder/main.go | 1 + 40 files changed, 2422 insertions(+), 96 deletions(-) create mode 100644 cmd/telegraf/cmd_secretstore.go create mode 100644 config/secret.go create mode 100644 config/secret_test.go create mode 100644 config/secret_with_mlock.go create mode 100644 config/secret_without_mlock.go create mode 100644 config/util.go create mode 100644 plugins/secretstores/all/all.go create mode 100644 plugins/secretstores/all/jose.go create mode 100644 plugins/secretstores/all/os.go create mode 100644 plugins/secretstores/deprecations.go create mode 100644 plugins/secretstores/jose/README.md create mode 100644 plugins/secretstores/jose/jose.go create mode 100644 plugins/secretstores/jose/jose_test.go create mode 100644 plugins/secretstores/jose/sample.conf create mode 100644 plugins/secretstores/os/README.md create mode 100644 plugins/secretstores/os/os.go create mode 100644 plugins/secretstores/os/os_darwin.go create mode 100644 plugins/secretstores/os/os_linux.go create mode 100644 plugins/secretstores/os/os_test.go create mode 100644 plugins/secretstores/os/os_unsupported.go create mode 100644 plugins/secretstores/os/os_windows.go create mode 100644 plugins/secretstores/os/sample_darwin.conf create mode 100644 plugins/secretstores/os/sample_linux.conf create mode 100644 plugins/secretstores/os/sample_windows.conf create mode 100644 plugins/secretstores/registry.go create mode 100644 secretstore.go diff --git a/Makefile b/Makefile index f088aa23d..f762c5e55 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ embed_readme_%: go generate -run="readme_config_includer/generator$$" ./plugins/$*/... .PHONY: docs -docs: build_tools embed_readme_inputs embed_readme_outputs embed_readme_processors embed_readme_aggregators +docs: build_tools embed_readme_inputs embed_readme_outputs embed_readme_processors embed_readme_aggregators embed_readme_secretstores .PHONY: build build: diff --git a/cmd/telegraf/cmd_secretstore.go b/cmd/telegraf/cmd_secretstore.go new file mode 100644 index 000000000..dae87f583 --- /dev/null +++ b/cmd/telegraf/cmd_secretstore.go @@ -0,0 +1,258 @@ +// Command handling for secret-stores' "secret" command +package main + +import ( + "errors" + "fmt" + "os" + "sort" + "strings" + + "github.com/influxdata/telegraf/config" + "github.com/urfave/cli/v2" + "golang.org/x/term" +) + +func processFilterOnlySecretStoreFlags(ctx *cli.Context) Filters { + sectionFilters := []string{"inputs", "outputs", "processors", "aggregators"} + inputFilters := []string{"-"} + outputFilters := []string{"-"} + processorFilters := []string{"-"} + aggregatorFilters := []string{"-"} + + // Only load the secret-stores + var secretstore string + if len(ctx.Lineage()) >= 2 { + parent := ctx.Lineage()[1] // ancestor contexts in order from child to parent + secretstore = parent.String("secretstore-filter") + } + + // If both the parent and command filters are defined, append them together + secretstore = appendFilter(secretstore, ctx.String("secretstore-filter")) + secretstoreFilters := deleteEmpty(strings.Split(secretstore, ":")) + return Filters{sectionFilters, inputFilters, outputFilters, aggregatorFilters, processorFilters, secretstoreFilters} +} + +func getSecretStoreCommands(m App) []*cli.Command { + return []*cli.Command{ + { + Name: "secrets", + Usage: "commands for listing, adding and removing secrets on all known secret-stores", + Subcommands: []*cli.Command{ + { + Name: "list", + Usage: "list known secrets and secret-stores", + Description: ` +The 'list' command requires passing in your configuration file +containing the secret-store definitions you want to access. To get a +list of available secret-store plugins, please have a look at +https://github.com/influxdata/telegraf/tree/master/plugins/secretstores. + +For help on how to define secret-stores, check the documentation of the +different plugins. + +Assuming you use the default configuration file location, you can run +the following command to list the keys of all known secrets in ALL +available stores + +> telegraf secrets list + +To get the keys of all known secrets in a particular store, you can run + +> telegraf secrets list mystore + +To also reveal the actual secret, i.e. the value, you can pass the +'--reveal-secret' flag. +`, + ArgsUsage: "[secret-store ID]...[secret-store ID]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "reveal-secret", + Usage: "also print the secret value", + }, + }, + Action: func(cCtx *cli.Context) error { + // Only load the secret-stores + filters := processFilterOnlySecretStoreFlags(cCtx) + g := GlobalFlags{ + config: cCtx.StringSlice("config"), + configDir: cCtx.StringSlice("config-directory"), + plugindDir: cCtx.String("plugin-directory"), + debug: cCtx.Bool("debug"), + } + w := WindowFlags{} + m.Init(nil, filters, g, w) + + args := cCtx.Args() + var storeIDs []string + if args.Present() { + storeIDs = args.Slice() + } else { + ids, err := m.ListSecretStores() + if err != nil { + return fmt.Errorf("unable to determine secret-store IDs: %w", err) + } + storeIDs = ids + } + sort.Strings(storeIDs) + + reveal := cCtx.Bool("reveal-secret") + for _, storeID := range storeIDs { + store, err := m.GetSecretStore(storeID) + if err != nil { + return fmt.Errorf("unable to get secret-store %q: %w", storeID, err) + } + keys, err := store.List() + if err != nil { + return fmt.Errorf("unable to get secrets from store %q: %w", storeID, err) + } + sort.Strings(keys) + + _, _ = fmt.Printf("Known secrets for store %q:\n", storeID) + for _, k := range keys { + var v []byte + if reveal { + if v, err = store.Get(k); err != nil { + return fmt.Errorf("unable to get value of secret %q from store %q: %w", k, storeID, err) + } + } + _, _ = fmt.Printf(" %-30s %s\n", k, string(v)) + config.ReleaseSecret(v) + } + } + + return nil + }, + }, + { + Name: "get", + Usage: "retrieves value of given secret from given store", + Description: ` +The 'get' command requires passing in your configuration file +containing the secret-store definitions you want to access. To get a +list of available secret-store plugins, please have a look at +https://github.com/influxdata/telegraf/tree/master/plugins/secretstores. +and use the 'secrets list' command to get the IDs of available stores and +key(s) of available secrets. + +For help on how to define secret-stores, check the documentation of the +different plugins. + +Assuming you use the default configuration file location, you can run +the following command to retrieve a secret from a secret store +available stores + +> telegraf secrets get mystore mysecretkey + +This will fetch the secret with the key 'mysecretkey' from the secret-store +with the ID 'mystore'. +`, + ArgsUsage: " ", + Action: func(cCtx *cli.Context) error { + // Only load the secret-stores + filters := processFilterOnlySecretStoreFlags(cCtx) + g := GlobalFlags{ + config: cCtx.StringSlice("config"), + configDir: cCtx.StringSlice("config-directory"), + plugindDir: cCtx.String("plugin-directory"), + debug: cCtx.Bool("debug"), + } + w := WindowFlags{} + m.Init(nil, filters, g, w) + + args := cCtx.Args() + if !args.Present() || args.Len() != 2 { + return errors.New("invalid number of arguments") + } + + storeID := args.First() + key := args.Get(1) + + store, err := m.GetSecretStore(storeID) + if err != nil { + return fmt.Errorf("unable to get secret-store: %w", err) + } + value, err := store.Get(key) + if err != nil { + return fmt.Errorf("unable to get secret: %w", err) + } + _, _ = fmt.Printf("%s:%s = %s\n", storeID, key, value) + + return nil + }, + }, + { + Name: "set", + Usage: "create or modify a secret in the given store", + Description: ` +The 'set' command requires passing in your configuration file +containing the secret-store definitions you want to access. To get a +list of available secret-store plugins, please have a look at +https://github.com/influxdata/telegraf/tree/master/plugins/secretstores. +and use the 'secrets list' command to get the IDs of available stores and keys. + +For help on how to define secret-stores, check the documentation of the +different plugins. + +Assuming you use the default configuration file location, you can run +the following command to create a secret in anm available secret-store + +> telegraf secrets set mystore mysecretkey mysecretvalue + +This will create a secret with the key 'mysecretkey' in the secret-store +with the ID 'mystore' with the value being set to 'mysecretvalue'. If a +secret with that key ('mysecretkey') already existed in that store, its +value will be modified. + +When you leave out the value of the secret like + +> telegraf secrets set mystore mysecretkey + +you will be prompted to enter the value of the secret. +`, + ArgsUsage: " ", + Action: func(cCtx *cli.Context) error { + // Only load the secret-stores + filters := processFilterOnlySecretStoreFlags(cCtx) + g := GlobalFlags{ + config: cCtx.StringSlice("config"), + configDir: cCtx.StringSlice("config-directory"), + plugindDir: cCtx.String("plugin-directory"), + debug: cCtx.Bool("debug"), + } + w := WindowFlags{} + m.Init(nil, filters, g, w) + + args := cCtx.Args() + if !args.Present() || args.Len() < 2 { + return errors.New("invalid number of arguments") + } + + storeID := args.First() + key := args.Get(1) + value := args.Get(2) + if value == "" { + fmt.Printf("enter secret: ") + b, err := term.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + return err + } + fmt.Println() + value = string(b) + } + + store, err := m.GetSecretStore(storeID) + if err != nil { + return fmt.Errorf("unable to get secret-store: %w", err) + } + if err := store.Set(key, value); err != nil { + return fmt.Errorf("unable to set secret: %w", err) + } + + return nil + }, + }, + }, + }, + } +} diff --git a/cmd/telegraf/main.go b/cmd/telegraf/main.go index 1797be0e0..95498a2c4 100644 --- a/cmd/telegraf/main.go +++ b/cmd/telegraf/main.go @@ -10,6 +10,7 @@ import ( "github.com/urfave/cli/v2" + "github.com/awnumar/memguard" "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/internal/goplugin" @@ -21,6 +22,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/outputs/all" _ "github.com/influxdata/telegraf/plugins/parsers/all" _ "github.com/influxdata/telegraf/plugins/processors/all" + _ "github.com/influxdata/telegraf/plugins/secretstores/all" ) type TelegrafConfig interface { @@ -29,11 +31,12 @@ type TelegrafConfig interface { } type Filters struct { - section []string - input []string - output []string - aggregator []string - processor []string + section []string + input []string + output []string + aggregator []string + processor []string + secretstore []string } func appendFilter(a, b string) string { @@ -47,7 +50,7 @@ func appendFilter(a, b string) string { } func processFilterFlags(ctx *cli.Context) Filters { - var section, input, output, aggregator, processor string + var section, input, output, aggregator, processor, secretstore string // Support defining filters before and after the command // The old style was: @@ -62,6 +65,7 @@ func processFilterFlags(ctx *cli.Context) Filters { output = parent.String("output-filter") aggregator = parent.String("aggregator-filter") processor = parent.String("processor-filter") + secretstore = parent.String("secretstore-filter") } // If both the parent and command filters are defined, append them together @@ -70,13 +74,15 @@ func processFilterFlags(ctx *cli.Context) Filters { output = appendFilter(output, ctx.String("output-filter")) aggregator = appendFilter(aggregator, ctx.String("aggregator-filter")) processor = appendFilter(processor, ctx.String("processor-filter")) + secretstore = appendFilter(secretstore, ctx.String("secretstore-filter")) sectionFilters := deleteEmpty(strings.Split(section, ":")) inputFilters := deleteEmpty(strings.Split(input, ":")) outputFilters := deleteEmpty(strings.Split(output, ":")) aggregatorFilters := deleteEmpty(strings.Split(aggregator, ":")) processorFilters := deleteEmpty(strings.Split(processor, ":")) - return Filters{sectionFilters, inputFilters, outputFilters, aggregatorFilters, processorFilters} + secretstoreFilters := deleteEmpty(strings.Split(secretstore, ":")) + return Filters{sectionFilters, inputFilters, outputFilters, aggregatorFilters, processorFilters, secretstoreFilters} } func deleteEmpty(s []string) []string { @@ -114,6 +120,10 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi Name: "processor-filter", Usage: "filter the processors to enable, separator is ':'", }, + &cli.StringFlag{ + Name: "secretstore-filter", + Usage: "filter the secret-stores to enable, separator is ':'", + }, } extraFlags := append(pluginFilterFlags, cliFlags()...) @@ -199,6 +209,7 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi filters.output, filters.aggregator, filters.processor, + filters.secretstore, ) return nil } @@ -326,7 +337,7 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi // !!! }, extraFlags...), Action: action, - Commands: []*cli.Command{ + Commands: append([]*cli.Command{ { Name: "config", Usage: "print out full sample configuration to stdout", @@ -343,6 +354,7 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi filters.output, filters.aggregator, filters.processor, + filters.secretstore, ) return nil }, @@ -356,8 +368,14 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi }, }, }, + getSecretStoreCommands(m)..., + ), } + // Make sure we safely erase secrets + memguard.CatchInterrupt() + defer memguard.Purge() + return app.Run(args) } diff --git a/cmd/telegraf/main_test.go b/cmd/telegraf/main_test.go index 8f8ce5c5b..aaa30fa75 100644 --- a/cmd/telegraf/main_test.go +++ b/cmd/telegraf/main_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "errors" "fmt" "io" "os" @@ -18,6 +19,26 @@ import ( "github.com/influxdata/telegraf/plugins/outputs" ) +var secrets = map[string]map[string][]byte{ + "yoda": { + "episode1": []byte("member"), + "episode2": []byte("member"), + "episode3": []byte("member"), + }, + "mace_windu": { + "episode1": []byte("member"), + "episode2": []byte("member"), + "episode3": []byte("member"), + }, + "oppo_rancisis": { + "episode1": []byte("member"), + "episode2": []byte("member"), + }, + "coleman_kcaj": { + "episode3": []byte("member"), + }, +} + type MockTelegraf struct { GlobalFlags WindowFlags @@ -36,6 +57,65 @@ func (m *MockTelegraf) Run() error { return nil } +func (m *MockTelegraf) ListSecretStores() ([]string, error) { + ids := make([]string, 0, len(secrets)) + for k := range secrets { + ids = append(ids, k) + } + return ids, nil +} + +func (m *MockTelegraf) GetSecretStore(id string) (telegraf.SecretStore, error) { + v, found := secrets[id] + if !found { + return nil, errors.New("unknown secret store") + } + s := &MockSecretStore{Secrets: v} + return s, nil +} + +type MockSecretStore struct { + Secrets map[string][]byte +} + +func (s *MockSecretStore) Init() error { + return nil +} + +func (s *MockSecretStore) SampleConfig() string { + return "I'm just a dummy" +} + +func (s *MockSecretStore) Get(key string) ([]byte, error) { + v, found := s.Secrets[key] + if !found { + return nil, errors.New("not found") + } + return v, nil +} + +func (s *MockSecretStore) Set(key, value string) error { + if strings.HasPrefix(key, "darth") { + return errors.New("don't join the dark side") + } + s.Secrets[key] = []byte(value) + return nil +} +func (s *MockSecretStore) List() ([]string, error) { + keys := make([]string, 0, len(s.Secrets)) + for k := range s.Secrets { + keys = append(keys, k) + } + return keys, nil +} + +func (s *MockSecretStore) GetResolver(key string) (telegraf.ResolveFunc, error) { + return func() ([]byte, bool, error) { + v, err := s.Get(key) + return v, false, err + }, nil +} + type MockConfig struct { Buffer io.Writer ExpectedDeprecatedPlugins map[string][]config.PluginDeprecationInfo diff --git a/cmd/telegraf/printer.go b/cmd/telegraf/printer.go index 1aa1e830b..72dc41cb0 100644 --- a/cmd/telegraf/printer.go +++ b/cmd/telegraf/printer.go @@ -12,12 +12,13 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/processors" + "github.com/influxdata/telegraf/plugins/secretstores" ) var ( // Default sections - sectionDefaults = []string{"global_tags", "agent", "outputs", - "processors", "aggregators", "inputs"} + sectionDefaults = []string{"global_tags", "agent", "secretstores", + "outputs", "processors", "aggregators", "inputs"} // Default input plugins inputDefaults = []string{"cpu", "mem", "swap", "system", "kernel", @@ -58,6 +59,13 @@ var globalTagsConfig = ` //go:embed agent.conf var agentConfig string +var secretstoreHeader = ` +############################################################################### +# SECRETSTORE PLUGINS # +############################################################################### + +` + var outputHeader = ` ############################################################################### # OUTPUT PLUGINS # @@ -110,6 +118,7 @@ func printSampleConfig( outputFilters []string, aggregatorFilters []string, processorFilters []string, + secretstoreFilters []string, ) { // print headers outputBuffer.Write([]byte(header)) @@ -119,6 +128,42 @@ func printSampleConfig( } printFilteredGlobalSections(sectionFilters, outputBuffer) + // print secretstore plugins + if sliceContains("secretstores", sectionFilters) { + if len(secretstoreFilters) != 0 { + if len(secretstoreFilters) >= 3 && secretstoreFilters[1] != "none" { + fmt.Print(secretstoreHeader) + } + printFilteredSecretstores(secretstoreFilters, false, outputBuffer) + } else { + fmt.Print(secretstoreHeader) + snames := []string{} + for sname := range secretstores.SecretStores { + snames = append(snames, sname) + } + sort.Strings(snames) + printFilteredSecretstores(snames, true, outputBuffer) + } + } + + // print secretstore plugins + if sliceContains("secretstores", sectionFilters) { + if len(secretstoreFilters) != 0 { + if len(secretstoreFilters) >= 3 && secretstoreFilters[1] != "none" { + fmt.Print(secretstoreHeader) + } + printFilteredSecretstores(secretstoreFilters, false, outputBuffer) + } else { + fmt.Print(secretstoreHeader) + snames := []string{} + for sname := range secretstores.SecretStores { + snames = append(snames, sname) + } + sort.Strings(snames) + printFilteredSecretstores(snames, true, outputBuffer) + } + } + // print output plugins if sliceContains("outputs", sectionFilters) { if len(outputFilters) != 0 { @@ -309,6 +354,24 @@ func printFilteredOutputs(outputFilters []string, commented bool, outputBuffer i } } +func printFilteredSecretstores(secretstoreFilters []string, commented bool, outputBuffer io.Writer) { + // Filter secretstores + var snames []string + for sname := range secretstores.SecretStores { + if sliceContains(sname, secretstoreFilters) { + snames = append(snames, sname) + } + } + sort.Strings(snames) + + // Print SecretStores + for _, sname := range snames { + creator := secretstores.SecretStores[sname] + store := creator("dummy") + printConfig(sname, store, "secretstores", commented, secretstores.Deprecations[sname], outputBuffer) + } +} + func printFilteredGlobalSections(sectionFilters []string, outputBuffer io.Writer) { if sliceContains("global_tags", sectionFilters) { outputBuffer.Write([]byte(globalTagsConfig)) diff --git a/cmd/telegraf/telegraf.go b/cmd/telegraf/telegraf.go index 3e05c64be..445a908fb 100644 --- a/cmd/telegraf/telegraf.go +++ b/cmd/telegraf/telegraf.go @@ -26,6 +26,7 @@ import ( "github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/parsers" "github.com/influxdata/telegraf/plugins/processors" + "github.com/influxdata/telegraf/plugins/secretstores" ) var stop chan struct{} @@ -55,14 +56,19 @@ type WindowFlags struct { type App interface { Init(<-chan error, Filters, GlobalFlags, WindowFlags) Run() error + + // Secret store commands + ListSecretStores() ([]string, error) + GetSecretStore(string) (telegraf.SecretStore, error) } type Telegraf struct { pprofErr <-chan error - inputFilters []string - outputFilters []string - configFiles []string + inputFilters []string + outputFilters []string + configFiles []string + secretstoreFilters []string GlobalFlags WindowFlags @@ -72,10 +78,38 @@ func (t *Telegraf) Init(pprofErr <-chan error, f Filters, g GlobalFlags, w Windo t.pprofErr = pprofErr t.inputFilters = f.input t.outputFilters = f.output + t.secretstoreFilters = f.secretstore t.GlobalFlags = g t.WindowFlags = w } +func (t *Telegraf) ListSecretStores() ([]string, error) { + c, err := t.loadConfiguration() + if err != nil { + return nil, err + } + + ids := make([]string, 0, len(c.SecretStores)) + for k := range c.SecretStores { + ids = append(ids, k) + } + return ids, nil +} + +func (t *Telegraf) GetSecretStore(id string) (telegraf.SecretStore, error) { + c, err := t.loadConfiguration() + if err != nil { + return nil, err + } + + store, found := c.SecretStores[id] + if !found { + return nil, errors.New("unknown secret store") + } + + return store, nil +} + func (t *Telegraf) reloadLoop() error { reload := make(chan bool, 1) reload <- true @@ -161,7 +195,13 @@ func (t *Telegraf) watchLocalConfig(signals chan os.Signal, fConfig string) { signals <- syscall.SIGHUP } -func (t *Telegraf) runAgent(ctx context.Context) error { +func (t *Telegraf) loadConfiguration() (*config.Config, error) { + // If no other options are specified, load the config file and run. + c := config.NewConfig() + c.OutputFilters = t.outputFilters + c.InputFilters = t.inputFilters + c.SecretStoreFilters = t.secretstoreFilters + var configFiles []string // providing no "config" flag should load default config if len(t.config) == 0 { @@ -173,18 +213,21 @@ func (t *Telegraf) runAgent(ctx context.Context) error { for _, fConfigDirectory := range t.configDir { files, err := config.WalkDirectory(fConfigDirectory) if err != nil { - return err + return c, err } configFiles = append(configFiles, files...) } - // If no other options are specified, load the config file and run. - c := config.NewConfig() - c.OutputFilters = t.outputFilters - c.InputFilters = t.inputFilters - t.configFiles = configFiles if err := c.LoadAll(configFiles...); err != nil { + return c, err + } + return c, nil +} + +func (t *Telegraf) runAgent(ctx context.Context) error { + c, err := t.loadConfiguration() + if err != nil { return err } @@ -216,22 +259,23 @@ func (t *Telegraf) runAgent(ctx context.Context) error { LogWithTimezone: c.Agent.LogWithTimezone, } - err := logger.SetupLogging(logConfig) - if err != nil { + if err := logger.SetupLogging(logConfig); err != nil { return err } log.Printf("I! Starting Telegraf %s%s", internal.Version, internal.Customized) - log.Printf("I! Available plugins: %d inputs, %d aggregators, %d processors, %d parsers, %d outputs", + log.Printf("I! Available plugins: %d inputs, %d aggregators, %d processors, %d parsers, %d outputs, %d secret-stores", len(inputs.Inputs), len(aggregators.Aggregators), len(processors.Processors), len(parsers.Parsers), len(outputs.Outputs), + len(secretstores.SecretStores), ) log.Printf("I! Loaded inputs: %s", strings.Join(c.InputNames(), " ")) log.Printf("I! Loaded aggregators: %s", strings.Join(c.AggregatorNames(), " ")) log.Printf("I! Loaded processors: %s", strings.Join(c.ProcessorNames(), " ")) + log.Printf("I! Loaded secretstores: %s", strings.Join(c.SecretstoreNames(), " ")) if !t.once && (t.test || t.testWait != 0) { log.Print("W! " + color.RedString("Outputs are not used in testing mode!")) } else { @@ -251,6 +295,9 @@ func (t *Telegraf) runAgent(ctx context.Context) error { if count, found := c.Deprecations["outputs"]; found && (count[0] > 0 || count[1] > 0) { log.Printf("W! Deprecated outputs: %d and %d options", count[0], count[1]) } + if count, found := c.Deprecations["secretstores"]; found && (count[0] > 0 || count[1] > 0) { + log.Printf("W! Deprecated secretstores: %d and %d options", count[0], count[1]) + } ag := agent.NewAgent(c) diff --git a/config/config.go b/config/config.go index 403a46ef7..81f0ff7c5 100644 --- a/config/config.go +++ b/config/config.go @@ -32,6 +32,7 @@ import ( "github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/parsers" "github.com/influxdata/telegraf/plugins/processors" + "github.com/influxdata/telegraf/plugins/secretstores" "github.com/influxdata/telegraf/plugins/serializers" ) @@ -59,9 +60,12 @@ type Config struct { UnusedFields map[string]bool unusedFieldsMutex *sync.Mutex - Tags map[string]string - InputFilters []string - OutputFilters []string + Tags map[string]string + InputFilters []string + OutputFilters []string + SecretStoreFilters []string + + SecretStores map[string]telegraf.SecretStore Agent *AgentConfig Inputs []*models.RunningInput @@ -108,16 +112,18 @@ func NewConfig() *Config { LogfileRotationMaxArchives: 5, }, - Tags: make(map[string]string), - Inputs: make([]*models.RunningInput, 0), - Outputs: make([]*models.RunningOutput, 0), - Processors: make([]*models.RunningProcessor, 0), - AggProcessors: make([]*models.RunningProcessor, 0), - fileProcessors: make([]*OrderedPlugin, 0), - fileAggProcessors: make([]*OrderedPlugin, 0), - InputFilters: make([]string, 0), - OutputFilters: make([]string, 0), - Deprecations: make(map[string][]int64), + Tags: make(map[string]string), + Inputs: make([]*models.RunningInput, 0), + Outputs: make([]*models.RunningOutput, 0), + Processors: make([]*models.RunningProcessor, 0), + AggProcessors: make([]*models.RunningProcessor, 0), + SecretStores: make(map[string]telegraf.SecretStore), + fileProcessors: make([]*OrderedPlugin, 0), + fileAggProcessors: make([]*OrderedPlugin, 0), + InputFilters: make([]string, 0), + OutputFilters: make([]string, 0), + SecretStoreFilters: make([]string, 0), + Deprecations: make(map[string][]int64), } // Handle unknown version @@ -274,6 +280,15 @@ func (c *Config) OutputNames() []string { return PluginNameCounts(name) } +// SecretstoreNames returns a list of strings of the configured secret-stores. +func (c *Config) SecretstoreNames() []string { + names := make([]string, 0, len(c.SecretStores)) + for name := range c.SecretStores { + names = append(names, name) + } + return PluginNameCounts(names) +} + // PluginNameCounts returns a list of sorted plugin names and their count func PluginNameCounts(plugins []string) []string { names := make(map[string]int) @@ -411,7 +426,8 @@ func (c *Config) LoadAll(configFiles ...string) error { sort.Stable(c.Processors) sort.Stable(c.AggProcessors) - return nil + // Let's link all secrets to their secret-stores + return c.LinkSecrets() } // LoadConfigData loads TOML-formatted config data @@ -575,6 +591,24 @@ func (c *Config) LoadConfigData(data []byte) error { name, pluginName, subTable.Line, keys(c.UnusedFields)) } } + case "secretstores": + for pluginName, pluginVal := range subTable.Fields { + switch pluginSubTable := pluginVal.(type) { + case []*ast.Table: + for _, t := range pluginSubTable { + if err = c.addSecretStore(pluginName, t); err != nil { + return fmt.Errorf("error parsing %s, %s", pluginName, err) + } + } + default: + return fmt.Errorf("unsupported config format: %s", pluginName) + } + if len(c.UnusedFields) > 0 { + msg := "plugin %s.%s: line %d: configuration specified the fields %q, but they weren't used" + return fmt.Errorf(msg, name, pluginName, subTable.Line, keys(c.UnusedFields)) + } + } + // Assume it's an input for legacy config file support if no other // identifiers are present default: @@ -746,6 +780,68 @@ func (c *Config) addAggregator(name string, table *ast.Table) error { return nil } +func (c *Config) addSecretStore(name string, table *ast.Table) error { + if len(c.SecretStoreFilters) > 0 && !sliceContains(name, c.SecretStoreFilters) { + return nil + } + + var storeid string + c.getFieldString(table, "id", &storeid) + + creator, ok := secretstores.SecretStores[name] + if !ok { + // Handle removed, deprecated plugins + if di, deprecated := secretstores.Deprecations[name]; deprecated { + printHistoricPluginDeprecationNotice("secretstores", name, di) + return fmt.Errorf("plugin deprecated") + } + return fmt.Errorf("undefined but requested secretstores: %s", name) + } + store := creator(storeid) + + if err := c.toml.UnmarshalTable(table, store); err != nil { + return err + } + + if err := c.printUserDeprecation("secretstores", name, store); err != nil { + return err + } + + if err := store.Init(); err != nil { + return fmt.Errorf("error initializing secretstore: %w", err) + } + + if _, found := c.SecretStores[storeid]; found { + return fmt.Errorf("duplicate ID %q for secretstore %q", storeid, name) + } + c.SecretStores[storeid] = store + return nil +} + +func (c *Config) LinkSecrets() error { + for _, s := range unlinkedSecrets { + resolvers := make(map[string]telegraf.ResolveFunc) + for _, ref := range s.GetUnlinked() { + // Split the reference and lookup the resolver + storeid, key := splitLink(ref) + store, found := c.SecretStores[storeid] + if !found { + return fmt.Errorf("unknown secret-store for %q", ref) + } + resolver, err := store.GetResolver(key) + if err != nil { + return fmt.Errorf("retrieving resolver for %q failed: %v", ref, err) + } + resolvers[ref] = resolver + } + // Inject the resolver list into the secret + if err := s.Link(resolvers); err != nil { + return fmt.Errorf("retrieving resolver failed: %v", err) + } + } + return nil +} + func (c *Config) probeParser(parentcategory string, parentname string, table *ast.Table) bool { var dataformat string c.getFieldString(table, "data_format", &dataformat) diff --git a/config/secret.go b/config/secret.go new file mode 100644 index 000000000..dd3c38113 --- /dev/null +++ b/config/secret.go @@ -0,0 +1,197 @@ +package config + +import ( + "fmt" + "regexp" + "strings" + + "github.com/awnumar/memguard" + + "github.com/influxdata/telegraf" +) + +// unlinkedSecrets contains the list of secrets that contain +// references not yet linked to their corresponding secret-store. +// Those secrets must later (after reading the config) be linked +// by the config to their respective secret-stores. +// Secrets containing constant strings will not be found in this +// list. +var unlinkedSecrets = make([]*Secret, 0) + +// secretPattern is a regex to extract references to secrets stored +// in a secret-store. +var secretPattern = regexp.MustCompile(`@\{(\w+:\w+)\}`) + +// Secret safely stores sensitive data such as a password or token +type Secret struct { + enclave *memguard.Enclave + resolvers map[string]telegraf.ResolveFunc + // unlinked contains all references in the secret that are not yet + // linked to the corresponding secret store. + unlinked []string +} + +// NewSecret creates a new secret from the given bytes +func NewSecret(b []byte) Secret { + s := Secret{} + s.init(b) + return s +} + +// UnmarshalTOML creates a secret from a toml value. +func (s *Secret) UnmarshalTOML(b []byte) error { + // Unmarshal raw secret from TOML and put it into protected memory + s.init(b) + + // Keep track of secrets that contain references to secret-stores + // for later resolving by the config. + if len(s.unlinked) > 0 { + unlinkedSecrets = append(unlinkedSecrets, s) + } + + return nil +} + +// Initialize the secret content +func (s *Secret) init(b []byte) { + secret := unquoteTomlString(b) + + // Find all parts that need to be resolved and return them + s.unlinked = secretPattern.FindAllString(string(secret), -1) + + // Setup the enclave + s.enclave = memguard.NewEnclave(secret) + s.resolvers = nil +} + +// Destroy the secret content +func (s *Secret) Destroy() { + s.resolvers = nil + s.unlinked = nil + + if s.enclave == nil { + return + } + + // Wipe the secret from memory + lockbuf, err := s.enclave.Open() + if err == nil { + lockbuf.Destroy() + } + s.enclave = nil +} + +// Get return the string representation of the secret +func (s *Secret) Get() ([]byte, error) { + if s.enclave == nil { + return nil, nil + } + + if len(s.unlinked) > 0 { + return nil, fmt.Errorf("unlinked parts in secret: %v", strings.Join(s.unlinked, ";")) + } + + // Decrypt the secret so we can return it + lockbuf, err := s.enclave.Open() + if err != nil { + return nil, fmt.Errorf("opening enclave failed: %v", err) + } + defer lockbuf.Destroy() + secret := lockbuf.Bytes() + + if len(s.resolvers) == 0 { + // Make a copy as we cannot access lockbuf after Destroy, i.e. + // after this function finishes. + newsecret := append([]byte{}, secret...) + return newsecret, protect(newsecret) + } + + replaceErrs := make([]string, 0) + newsecret := secretPattern.ReplaceAllFunc(secret, func(match []byte) []byte { + resolver, found := s.resolvers[string(match)] + if !found { + replaceErrs = append(replaceErrs, fmt.Sprintf("no resolver for %q", match)) + return match + } + replacement, _, err := resolver() + if err != nil { + replaceErrs = append(replaceErrs, fmt.Sprintf("resolving %q failed: %v", match, err)) + return match + } + + return replacement + }) + if len(replaceErrs) > 0 { + memguard.WipeBytes(newsecret) + return nil, fmt.Errorf("replacing secrets failed: %s", strings.Join(replaceErrs, ";")) + } + + return newsecret, protect(newsecret) +} + +// GetUnlinked return the parts of the secret that is not yet linked to a resolver +func (s *Secret) GetUnlinked() []string { + return s.unlinked +} + +// Link used the given resolver map to link the secret parts to their +// secret-store resolvers. +func (s *Secret) Link(resolvers map[string]telegraf.ResolveFunc) error { + // Setup the resolver map + s.resolvers = make(map[string]telegraf.ResolveFunc) + + // Decrypt the secret so we can return it + if s.enclave == nil { + return nil + } + lockbuf, err := s.enclave.Open() + if err != nil { + return fmt.Errorf("opening enclave failed: %v", err) + } + defer lockbuf.Destroy() + secret := lockbuf.Bytes() + + // Iterate through the parts and try to resolve them. For static parts + // we directly replace them, while for dynamic ones we store the resolver. + replaceErrs := make([]string, 0) + newsecret := secretPattern.ReplaceAllFunc(secret, func(match []byte) []byte { + resolver, found := resolvers[string(match)] + if !found { + replaceErrs = append(replaceErrs, fmt.Sprintf("unlinked part %q", match)) + return match + } + replacement, dynamic, err := resolver() + if err != nil { + replaceErrs = append(replaceErrs, fmt.Sprintf("resolving %q failed: %v", match, err)) + return match + } + + // Replace static parts right away + if !dynamic { + return replacement + } + + // Keep the resolver for dynamic secrets + s.resolvers[string(match)] = resolver + return match + }) + if len(replaceErrs) > 0 { + return fmt.Errorf("linking secrets failed: %s", strings.Join(replaceErrs, ";")) + } + + // Store the secret if it has changed + if string(secret) != string(newsecret) { + s.enclave = memguard.NewEnclave(newsecret) + } + + // All linked now + s.unlinked = nil + + return nil +} + +func splitLink(s string) (storeid string, key string) { + // There should _ALWAYS_ be two parts due to the regular expression match + parts := strings.SplitN(s[2:len(s)-1], ":", 2) + return parts[0], parts[1] +} diff --git a/config/secret_test.go b/config/secret_test.go new file mode 100644 index 000000000..a8d9f0e82 --- /dev/null +++ b/config/secret_test.go @@ -0,0 +1,561 @@ +package config + +import ( + "errors" + "os" + "testing" + + "github.com/awnumar/memguard" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" + "github.com/stretchr/testify/require" +) + +func TestSecretConstantManually(t *testing.T) { + mysecret := "a wonderful test" + s := NewSecret([]byte(mysecret)) + defer s.Destroy() + retrieved, err := s.Get() + require.NoError(t, err) + defer ReleaseSecret(retrieved) + require.EqualValues(t, mysecret, retrieved) +} + +func TestLinking(t *testing.T) { + mysecret := "a @{referenced:secret}" + resolvers := map[string]telegraf.ResolveFunc{ + "@{referenced:secret}": func() ([]byte, bool, error) { + return []byte("resolved secret"), false, nil + }, + } + s := NewSecret([]byte(mysecret)) + defer s.Destroy() + require.NoError(t, s.Link(resolvers)) + retrieved, err := s.Get() + require.NoError(t, err) + defer ReleaseSecret(retrieved) + require.EqualValues(t, "a resolved secret", retrieved) +} + +func TestLinkingResolverError(t *testing.T) { + mysecret := "a @{referenced:secret}" + resolvers := map[string]telegraf.ResolveFunc{ + "@{referenced:secret}": func() ([]byte, bool, error) { + return nil, false, errors.New("broken") + }, + } + s := NewSecret([]byte(mysecret)) + defer s.Destroy() + expected := `linking secrets failed: resolving "@{referenced:secret}" failed: broken` + require.EqualError(t, s.Link(resolvers), expected) +} + +func TestGettingUnlinked(t *testing.T) { + mysecret := "a @{referenced:secret}" + s := NewSecret([]byte(mysecret)) + defer s.Destroy() + _, err := s.Get() + require.ErrorContains(t, err, "unlinked parts in secret") +} + +func TestGettingMissingResolver(t *testing.T) { + mysecret := "a @{referenced:secret}" + s := NewSecret([]byte(mysecret)) + defer s.Destroy() + s.unlinked = []string{} + s.resolvers = map[string]telegraf.ResolveFunc{ + "@{a:dummy}": func() ([]byte, bool, error) { + return nil, false, nil + }, + } + _, err := s.Get() + expected := `replacing secrets failed: no resolver for "@{referenced:secret}"` + require.EqualError(t, err, expected) +} + +func TestGettingResolverError(t *testing.T) { + mysecret := "a @{referenced:secret}" + s := NewSecret([]byte(mysecret)) + defer s.Destroy() + s.unlinked = []string{} + s.resolvers = map[string]telegraf.ResolveFunc{ + "@{referenced:secret}": func() ([]byte, bool, error) { + return nil, false, errors.New("broken") + }, + } + _, err := s.Get() + expected := `replacing secrets failed: resolving "@{referenced:secret}" failed: broken` + require.EqualError(t, err, expected) +} + +func TestUninitializedEnclave(t *testing.T) { + s := Secret{} + defer s.Destroy() + require.NoError(t, s.Link(map[string]telegraf.ResolveFunc{})) + retrieved, err := s.Get() + require.NoError(t, err) + require.Empty(t, retrieved) + defer ReleaseSecret(retrieved) +} + +func TestEnclaveOpenError(t *testing.T) { + mysecret := "a @{referenced:secret}" + s := NewSecret([]byte(mysecret)) + defer s.Destroy() + memguard.Purge() + err := s.Link(map[string]telegraf.ResolveFunc{}) + require.ErrorContains(t, err, "opening enclave failed") + + s.unlinked = []string{} + _, err = s.Get() + require.ErrorContains(t, err, "opening enclave failed") +} + +func TestMissingResolver(t *testing.T) { + mysecret := "a @{referenced:secret}" + s := NewSecret([]byte(mysecret)) + defer s.Destroy() + err := s.Link(map[string]telegraf.ResolveFunc{}) + require.ErrorContains(t, err, "linking secrets failed: unlinked part") +} + +func TestSecretConstant(t *testing.T) { + tests := []struct { + name string + cfg []byte + expected string + }{ + { + name: "simple string", + cfg: []byte(` + [[inputs.mockup]] + secret = "a secret" + `), + expected: "a secret", + }, + { + name: "mail address", + cfg: []byte(` + [[inputs.mockup]] + secret = "someone@mock.org" + `), + expected: "someone@mock.org", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewConfig() + require.NoError(t, c.LoadConfigData(tt.cfg)) + require.Len(t, c.Inputs, 1) + + // Create a mockup secretstore + store := &MockupSecretStore{ + Secrets: map[string][]byte{"mock": []byte("fail")}, + } + require.NoError(t, store.Init()) + c.SecretStores["mock"] = store + require.NoError(t, c.LinkSecrets()) + + plugin := c.Inputs[0].Input.(*MockupSecretPlugin) + secret, err := plugin.Secret.Get() + require.NoError(t, err) + defer ReleaseSecret(secret) + + require.EqualValues(t, tt.expected, string(secret)) + }) + } +} + +func TestSecretUnquote(t *testing.T) { + tests := []struct { + name string + cfg []byte + expected string + }{ + { + name: "single quotes", + cfg: []byte(` + [[inputs.mockup]] + secret = 'a secret' + `), + expected: "a secret", + }, + { + name: "double quotes", + cfg: []byte(` + [[inputs.mockup]] + secret = "a secret" + `), + expected: "a secret", + }, + { + name: "triple single quotes", + cfg: []byte(` + [[inputs.mockup]] + secret = '''a secret''' + `), + expected: "a secret", + }, + { + name: "triple double quotes", + cfg: []byte(` + [[inputs.mockup]] + secret = """a secret""" + `), + expected: "a secret", + }, + { + name: "escaped double quotes", + cfg: []byte(` + [[inputs.mockup]] + secret = "\"a secret\"" + `), + expected: `\"a secret\"`, + }, + { + name: "mix double-single quotes (single)", + cfg: []byte(` + [[inputs.mockup]] + secret = "'a secret'" + `), + expected: `'a secret'`, + }, + { + name: "mix single-double quotes (single)", + cfg: []byte(` + [[inputs.mockup]] + secret = '"a secret"' + `), + expected: `"a secret"`, + }, + { + name: "mix double-single quotes (triple-single)", + cfg: []byte(` + [[inputs.mockup]] + secret = """'a secret'""" + `), + expected: `'a secret'`, + }, + { + name: "mix single-double quotes (triple-single)", + cfg: []byte(` + [[inputs.mockup]] + secret = '''"a secret"''' + `), + expected: `"a secret"`, + }, + { + name: "mix double-single quotes (triple)", + cfg: []byte(` + [[inputs.mockup]] + secret = """'''a secret'''""" + `), + expected: `'''a secret'''`, + }, + { + name: "mix single-double quotes (triple)", + cfg: []byte(` + [[inputs.mockup]] + secret = '''"""a secret"""''' + `), + expected: `"""a secret"""`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewConfig() + require.NoError(t, c.LoadConfigData(tt.cfg)) + require.Len(t, c.Inputs, 1) + + // Create a mockup secretstore + store := &MockupSecretStore{ + Secrets: map[string][]byte{}, + } + require.NoError(t, store.Init()) + c.SecretStores["mock"] = store + require.NoError(t, c.LinkSecrets()) + + plugin := c.Inputs[0].Input.(*MockupSecretPlugin) + secret, err := plugin.Secret.Get() + require.NoError(t, err) + defer ReleaseSecret(secret) + + require.EqualValues(t, tt.expected, string(secret)) + }) + } +} + +func TestSecretEnvironmentVariable(t *testing.T) { + cfg := []byte(` +[[inputs.mockup]] + secret = "$SOME_ENV_SECRET" +`) + require.NoError(t, os.Setenv("SOME_ENV_SECRET", "an env secret")) + + c := NewConfig() + err := c.LoadConfigData(cfg) + require.NoError(t, err) + require.Len(t, c.Inputs, 1) + + // Create a mockup secretstore + store := &MockupSecretStore{ + Secrets: map[string][]byte{}, + } + require.NoError(t, store.Init()) + c.SecretStores["mock"] = store + require.NoError(t, c.LinkSecrets()) + + plugin := c.Inputs[0].Input.(*MockupSecretPlugin) + secret, err := plugin.Secret.Get() + require.NoError(t, err) + defer ReleaseSecret(secret) + + require.EqualValues(t, "an env secret", secret) +} + +func TestSecretStoreStatic(t *testing.T) { + cfg := []byte( + ` +[[inputs.mockup]] + secret = "@{mock:secret1}" +[[inputs.mockup]] + secret = "@{mock:secret2}" +[[inputs.mockup]] + secret = "@{mock:a_strange_secret}" +[[inputs.mockup]] + secret = "@{mock:a_wierd_secret}" +`) + + c := NewConfig() + err := c.LoadConfigData(cfg) + require.NoError(t, err) + require.Len(t, c.Inputs, 4) + + // Create a mockup secretstore + store := &MockupSecretStore{ + Secrets: map[string][]byte{ + "secret1": []byte("Ood Bnar"), + "secret2": []byte("Thon"), + "a_strange_secret": []byte("Obi-Wan Kenobi"), + "a_wierd_secret": []byte("Arca Jeth"), + }, + } + require.NoError(t, store.Init()) + c.SecretStores["mock"] = store + require.NoError(t, c.LinkSecrets()) + + expected := []string{"Ood Bnar", "Thon", "Obi-Wan Kenobi", "Arca Jeth"} + for i, input := range c.Inputs { + plugin := input.Input.(*MockupSecretPlugin) + secret, err := plugin.Secret.Get() + require.NoError(t, err) + require.EqualValues(t, expected[i], secret) + ReleaseSecret(secret) + } +} + +func TestSecretStoreInvalidKeys(t *testing.T) { + cfg := []byte( + ` +[[inputs.mockup]] + secret = "@{mock:}" +[[inputs.mockup]] + secret = "@{mock:wild?%go}" +[[inputs.mockup]] + secret = "@{mock:a-strange-secret}" +[[inputs.mockup]] + secret = "@{mock:a wierd secret}" +`) + + c := NewConfig() + err := c.LoadConfigData(cfg) + require.NoError(t, err) + require.Len(t, c.Inputs, 4) + + // Create a mockup secretstore + store := &MockupSecretStore{ + Secrets: map[string][]byte{ + "": []byte("Ood Bnar"), + "wild?%go": []byte("Thon"), + "a-strange-secret": []byte("Obi-Wan Kenobi"), + "a wierd secret": []byte("Arca Jeth"), + }, + } + require.NoError(t, store.Init()) + c.SecretStores["mock"] = store + require.NoError(t, c.LinkSecrets()) + + expected := []string{ + "@{mock:}", + "@{mock:wild?%go}", + "@{mock:a-strange-secret}", + "@{mock:a wierd secret}", + } + for i, input := range c.Inputs { + plugin := input.Input.(*MockupSecretPlugin) + secret, err := plugin.Secret.Get() + require.NoError(t, err) + require.EqualValues(t, expected[i], secret) + ReleaseSecret(secret) + } +} + +func TestSecretStoreInvalidReference(t *testing.T) { + // Make sure we clean-up our mess + defer func() { unlinkedSecrets = make([]*Secret, 0) }() + + cfg := []byte( + ` +[[inputs.mockup]] + secret = "@{mock:test}" +`) + + c := NewConfig() + require.NoError(t, c.LoadConfigData(cfg)) + require.Len(t, c.Inputs, 1) + + // Create a mockup secretstore + store := &MockupSecretStore{ + Secrets: map[string][]byte{"test": []byte("Arca Jeth")}, + } + require.NoError(t, store.Init()) + c.SecretStores["foo"] = store + err := c.LinkSecrets() + require.EqualError(t, err, `unknown secret-store for "@{mock:test}"`) + + for _, input := range c.Inputs { + plugin := input.Input.(*MockupSecretPlugin) + secret, err := plugin.Secret.Get() + require.EqualError(t, err, `unlinked parts in secret: @{mock:test}`) + require.Empty(t, secret) + } +} + +func TestSecretStoreStaticChanging(t *testing.T) { + cfg := []byte( + ` +[[inputs.mockup]] + secret = "@{mock:secret}" +`) + + c := NewConfig() + err := c.LoadConfigData(cfg) + require.NoError(t, err) + require.Len(t, c.Inputs, 1) + + // Create a mockup secretstore + store := &MockupSecretStore{ + Secrets: map[string][]byte{"secret": []byte("Ood Bnar")}, + Dynamic: false, + } + require.NoError(t, store.Init()) + c.SecretStores["mock"] = store + require.NoError(t, c.LinkSecrets()) + + sequence := []string{"Ood Bnar", "Thon", "Obi-Wan Kenobi", "Arca Jeth"} + plugin := c.Inputs[0].Input.(*MockupSecretPlugin) + secret, err := plugin.Secret.Get() + require.NoError(t, err) + defer ReleaseSecret(secret) + + require.EqualValues(t, "Ood Bnar", secret) + + for _, v := range sequence { + store.Secrets["secret"] = []byte(v) + secret, err := plugin.Secret.Get() + require.NoError(t, err) + + // The secret should not change as the store is marked non-dyamic! + require.EqualValues(t, "Ood Bnar", secret) + ReleaseSecret(secret) + } +} + +func TestSecretStoreDynamic(t *testing.T) { + cfg := []byte( + ` +[[inputs.mockup]] + secret = "@{mock:secret}" +`) + + c := NewConfig() + err := c.LoadConfigData(cfg) + require.NoError(t, err) + require.Len(t, c.Inputs, 1) + + // Create a mockup secretstore + store := &MockupSecretStore{ + Secrets: map[string][]byte{"secret": []byte("Ood Bnar")}, + Dynamic: true, + } + require.NoError(t, store.Init()) + c.SecretStores["mock"] = store + require.NoError(t, c.LinkSecrets()) + + sequence := []string{"Ood Bnar", "Thon", "Obi-Wan Kenobi", "Arca Jeth"} + plugin := c.Inputs[0].Input.(*MockupSecretPlugin) + for _, v := range sequence { + store.Secrets["secret"] = []byte(v) + secret, err := plugin.Secret.Get() + require.NoError(t, err) + + // The secret should not change as the store is marked non-dynamic! + require.EqualValues(t, v, secret) + ReleaseSecret(secret) + } +} + +/*** Mockup (input) plugin for testing to avoid cyclic dependencies ***/ +type MockupSecretPlugin struct { + Secret Secret `toml:"secret"` +} + +func (*MockupSecretPlugin) SampleConfig() string { return "Mockup test secret plugin" } +func (*MockupSecretPlugin) Gather(_ telegraf.Accumulator) error { return nil } + +type MockupSecretStore struct { + Secrets map[string][]byte + Dynamic bool +} + +func (s *MockupSecretStore) Init() error { + return nil +} +func (*MockupSecretStore) SampleConfig() string { + return "Mockup test secret plugin" +} + +func (s *MockupSecretStore) Get(key string) ([]byte, error) { + v, found := s.Secrets[key] + if !found { + return nil, errors.New("not found") + } + return v, nil +} + +func (s *MockupSecretStore) Set(key, value string) error { + s.Secrets[key] = []byte(value) + return nil +} + +func (s *MockupSecretStore) List() ([]string, error) { + keys := make([]string, 0, len(s.Secrets)) + for k := range s.Secrets { + keys = append(keys, k) + } + return keys, nil +} +func (s *MockupSecretStore) GetResolver(key string) (telegraf.ResolveFunc, error) { + return func() ([]byte, bool, error) { + v, err := s.Get(key) + return v, s.Dynamic, err + }, nil +} + +// Register the mockup plugin on loading +func init() { + // Register the mockup input plugin for the required names + inputs.Add("mockup", func() telegraf.Input { return &MockupSecretPlugin{} }) +} diff --git a/config/secret_with_mlock.go b/config/secret_with_mlock.go new file mode 100644 index 000000000..02a509493 --- /dev/null +++ b/config/secret_with_mlock.go @@ -0,0 +1,20 @@ +//go:build linux + +package config + +import ( + "syscall" + + "github.com/awnumar/memguard" +) + +func protect(secret []byte) error { + return syscall.Mlock(secret) +} + +func ReleaseSecret(secret []byte) { + memguard.WipeBytes(secret) + if err := syscall.Munlock(secret); err != nil { + panic(err) + } +} diff --git a/config/secret_without_mlock.go b/config/secret_without_mlock.go new file mode 100644 index 000000000..1c4c82c95 --- /dev/null +++ b/config/secret_without_mlock.go @@ -0,0 +1,15 @@ +//go:build !linux + +package config + +import ( + "github.com/awnumar/memguard" +) + +func protect(secret []byte) error { + return nil +} + +func ReleaseSecret(secret []byte) { + memguard.WipeBytes(secret) +} diff --git a/config/util.go b/config/util.go new file mode 100644 index 000000000..53f601a9f --- /dev/null +++ b/config/util.go @@ -0,0 +1,23 @@ +package config + +import "bytes" + +func unquoteTomlString(b []byte) []byte { + if len(b) >= 6 { + if bytes.HasPrefix(b, []byte(`'''`)) && bytes.HasSuffix(b, []byte(`'''`)) { + return b[3 : len(b)-3] + } + if bytes.HasPrefix(b, []byte(`"""`)) && bytes.HasSuffix(b, []byte(`"""`)) { + return b[3 : len(b)-3] + } + } + if len(b) >= 2 { + if bytes.HasPrefix(b, []byte(`'`)) && bytes.HasSuffix(b, []byte(`'`)) { + return b[1 : len(b)-1] + } + if bytes.HasPrefix(b, []byte(`"`)) && bytes.HasSuffix(b, []byte(`"`)) { + return b[1 : len(b)-1] + } + } + return b +} diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index bd1157779..1f6331722 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -157,6 +157,45 @@ parsed: bucket = "replace_with_your_bucket_name" ``` +## Secret-store secrets + +Additional or instead of environment variables, you can use secret-stores +to fill in credentials or similar. To do so, you need to configure one or more +secret-store plugin(s) and then reference the secret in your plugin +configurations. A reference to a secret is specified in form +`@{:}`, where the `secret store id` is the unique +ID you defined for your secret-store and `secret name` is the name of the secret +to use. + +**Example**: + +This example illustrates the use of secret-store(s) in plugins + +```toml +[global_tags] + user = "alice" + +[[secretstores.os]] + id = "local_secrets" + +[[secretstores.jose]] + id = "cloud_secrets" + path = "/etc/telegraf/secrets" + # Optional reference to another secret store to unlock this one. + password = "@{local_secrets:cloud_store_passwd}" + +[[inputs.http]] + urls = ["http://server.company.org/metrics"] + username = "@{local_secrets:company_server_http_metric_user}" + password = "@{local_secrets:company_server_http_metric_pass}" + +[[outputs.influxdb_v2]] + urls = ["https://us-west-2-1.aws.cloud2.influxdata.com"] + token = "@{cloud_secrets:influxdb_token}" + organization = "yourname@yourcompany.com" + bucket = "replace_with_your_bucket_name" +``` + ## Intervals Intervals are durations of time and can be specified for supporting settings by diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index f17b94118..c19ec6eb1 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -6,6 +6,7 @@ following works: - cloud.google.com/go [Apache License 2.0](https://github.com/googleapis/google-cloud-go/blob/master/LICENSE) - code.cloudfoundry.org/clock [Apache License 2.0](https://github.com/cloudfoundry/clock/blob/master/LICENSE) - collectd.org [MIT License](https://git.octo.it/?p=collectd.git;a=blob;f=COPYING;hb=HEAD) +- github.com/99designs/keyring [MIT License](https://github.com/99designs/keyring/blob/master/LICENSE) - github.com/Azure/azure-amqp-common-go [MIT License](https://github.com/Azure/azure-amqp-common-go/blob/master/LICENSE) - github.com/Azure/azure-event-hubs-go [MIT License](https://github.com/Azure/azure-event-hubs-go/blob/master/LICENSE) - github.com/Azure/azure-kusto-go [MIT License](https://github.com/Azure/azure-kusto-go/blob/master/LICENSE) @@ -46,6 +47,8 @@ following works: - github.com/aristanetworks/glog [Apache License 2.0](https://github.com/aristanetworks/glog/blob/master/LICENSE) - github.com/aristanetworks/goarista [Apache License 2.0](https://github.com/aristanetworks/goarista/blob/master/COPYING) - github.com/armon/go-metrics [MIT License](https://github.com/armon/go-metrics/blob/master/LICENSE) +- github.com/awnumar/memcall [Apache License 2.0](https://github.com/awnumar/memcall/blob/master/LICENSE) +- github.com/awnumar/memguard [Apache License 2.0](https://github.com/awnumar/memguard/blob/master/LICENSE) - github.com/aws/aws-sdk-go-v2 [Apache License 2.0](https://github.com/aws/aws-sdk-go-v2/blob/main/LICENSE.txt) - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream [Apache License 2.0](https://github.com/aws/aws-sdk-go-v2/blob/main/aws/protocol/eventstream/LICENSE.txt) - github.com/aws/aws-sdk-go-v2/config [Apache License 2.0](https://github.com/aws/aws-sdk-go-v2/blob/main/config/LICENSE.txt) @@ -89,6 +92,7 @@ following works: - github.com/couchbase/gomemcached [MIT License](https://github.com/couchbase/gomemcached/blob/master/LICENSE) - github.com/couchbase/goutils [Apache License 2.0](https://github.com/couchbase/goutils/blob/master/LICENSE.md) - github.com/cpuguy83/go-md2man [MIT License](https://github.com/cpuguy83/go-md2man/blob/master/LICENSE.md) +- github.com/danieljoos/wincred [MIT License](https://github.com/danieljoos/wincred/blob/master/LICENSE) - github.com/davecgh/go-spew [ISC License](https://github.com/davecgh/go-spew/blob/master/LICENSE) - github.com/denisenkom/go-mssqldb [BSD 3-Clause "New" or "Revised" License](https://github.com/denisenkom/go-mssqldb/blob/master/LICENSE.txt) - github.com/devigned/tab [MIT License](https://github.com/devigned/tab/blob/master/LICENSE) @@ -101,6 +105,7 @@ following works: - github.com/docker/go-connections [Apache License 2.0](https://github.com/docker/go-connections/blob/master/LICENSE) - github.com/docker/go-units [Apache License 2.0](https://github.com/docker/go-units/blob/master/LICENSE) - github.com/doclambda/protobufquery [MIT License](https://github.com/doclambda/protobufquery/blob/master/LICENSE) +- github.com/dvsekhvalnov/jose2go [MIT License](https://github.com/dvsekhvalnov/jose2go/blob/master/LICENSE) - github.com/dynatrace-oss/dynatrace-metric-utils-go [Apache License 2.0](https://github.com/dynatrace-oss/dynatrace-metric-utils-go/blob/master/LICENSE) - github.com/eapache/go-resiliency [MIT License](https://github.com/eapache/go-resiliency/blob/master/LICENSE) - github.com/eapache/go-xerial-snappy [MIT License](https://github.com/eapache/go-xerial-snappy/blob/master/LICENSE) @@ -124,6 +129,7 @@ following works: - github.com/go-stack/stack [MIT License](https://github.com/go-stack/stack/blob/master/LICENSE.md) - github.com/go-stomp/stomp [Apache License 2.0](https://github.com/go-stomp/stomp/blob/master/LICENSE.txt) - github.com/gobwas/glob [MIT License](https://github.com/gobwas/glob/blob/master/LICENSE) +- github.com/godbus/dbus [BSD 2-Clause "Simplified" License](https://github.com/godbus/dbus/blob/master/LICENSE) - github.com/gofrs/uuid [MIT License](https://github.com/gofrs/uuid/blob/master/LICENSE) - github.com/gogo/protobuf [BSD 3-Clause Clear License](https://github.com/gogo/protobuf/blob/master/LICENSE) - github.com/golang-jwt/jwt [MIT License](https://github.com/golang-jwt/jwt/blob/main/LICENSE) @@ -151,6 +157,7 @@ following works: - github.com/gosnmp/gosnmp [BSD 2-Clause "Simplified" License](https://github.com/gosnmp/gosnmp/blob/master/LICENSE) - github.com/grid-x/modbus [BSD 3-Clause "New" or "Revised" License](https://github.com/grid-x/modbus/blob/master/LICENSE) - github.com/grid-x/serial [MIT License](https://github.com/grid-x/serial/blob/master/LICENSE) +- github.com/gsterjov/go-libsecret [MIT License](https://github.com/gsterjov/go-libsecret/blob/master/LICENSE) - github.com/gwos/tcg/sdk [MIT License](https://github.com/gwos/tcg/blob/master/LICENSE) - github.com/hailocab/go-hostpool [MIT License](https://github.com/hailocab/go-hostpool/blob/master/LICENSE) - github.com/harlow/kinesis-consumer [MIT License](https://github.com/harlow/kinesis-consumer/blob/master/LICENSE) @@ -231,6 +238,7 @@ following works: - github.com/modern-go/reflect2 [Apache License 2.0](https://github.com/modern-go/reflect2/blob/master/LICENSE) - github.com/montanaflynn/stats [MIT License](https://github.com/montanaflynn/stats/blob/master/LICENSE) - github.com/morikuni/aec [MIT License](https://github.com/morikuni/aec/blob/master/LICENSE) +- github.com/mtibben/percent [MIT License](https://github.com/mtibben/percent/blob/master/LICENSE) - github.com/multiplay/go-ts3 [BSD 2-Clause "Simplified" License](https://github.com/multiplay/go-ts3/blob/master/LICENSE) - github.com/munnerz/goautoneg [BSD 3-Clause Clear License](https://github.com/munnerz/goautoneg/blob/master/LICENSE) - github.com/naoina/go-stringutil [MIT License](https://github.com/naoina/go-stringutil/blob/master/LICENSE) diff --git a/go.mod b/go.mod index 14d83bc7b..6cb5dcc54 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( cloud.google.com/go/pubsub v1.26.0 cloud.google.com/go/storage v1.23.0 collectd.org v0.5.0 + github.com/99designs/keyring v1.2.1 github.com/Azure/azure-event-hubs-go/v3 v3.3.20 github.com/Azure/azure-kusto-go v0.8.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.4.1 @@ -33,6 +34,7 @@ require ( github.com/apache/thrift v0.16.0 github.com/aristanetworks/goarista v0.0.0-20190325233358-a123909ec740 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 + github.com/awnumar/memguard v0.22.3 github.com/aws/aws-sdk-go-v2 v1.17.1 github.com/aws/aws-sdk-go-v2/config v1.17.8 github.com/aws/aws-sdk-go-v2/credentials v1.13.2 @@ -190,20 +192,12 @@ require ( modernc.org/sqlite v1.19.2 ) -require ( - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect - github.com/gabriel-vasile/mimetype v1.4.0 // indirect - github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect - github.com/uber/jaeger-lib v2.4.1+incompatible // indirect - go.opentelemetry.io/otel/metric v0.32.1 // indirect - go.uber.org/zap v1.22.0 // indirect -) - require ( cloud.google.com/go v0.104.0 // indirect cloud.google.com/go/compute v1.10.0 // indirect cloud.google.com/go/iam v0.5.0 // indirect code.cloudfoundry.org/clock v1.0.0 // indirect + github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/Azure/azure-amqp-common-go/v3 v3.2.3 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go v65.0.0+incompatible // indirect @@ -231,6 +225,7 @@ require ( github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect github.com/aristanetworks/glog v0.0.0-20191112221043-67e8567f59f3 // indirect github.com/armon/go-metrics v0.3.10 // indirect + github.com/awnumar/memcall v0.1.2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.2.0 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.7.1 // indirect @@ -244,6 +239,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.0 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.19.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect github.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-hostpool v0.1.0 // indirect @@ -256,11 +252,13 @@ require ( github.com/couchbase/gomemcached v0.1.3 // indirect github.com/couchbase/goutils v0.1.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/danieljoos/wincred v1.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/devigned/tab v0.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/eapache/go-resiliency v1.3.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect @@ -268,6 +266,7 @@ require ( github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect + github.com/gabriel-vasile/mimetype v1.4.0 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -279,6 +278,7 @@ require ( github.com/go-stack/stack v1.8.1 // indirect github.com/goburrow/modbus v0.1.0 // indirect github.com/goburrow/serial v0.1.1-0.20211022031912-bfb69110f8dd // indirect + github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect @@ -294,6 +294,7 @@ require ( github.com/googleapis/go-type-adapters v1.0.0 // indirect github.com/grid-x/serial v0.0.0-20211107191517-583c7356b3aa // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -348,6 +349,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.6.6 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/mtibben/percent v0.2.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/nats-io/jwt/v2 v2.3.0 // indirect @@ -384,6 +386,8 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.5.0 // indirect + github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect + github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/vishvananda/netlink v1.2.1-beta.2 // indirect github.com/vishvananda/netns v0.0.0-20220913150850-18c4f4234207 github.com/wvanbergen/kazoo-go v0.0.0-20180202103751-f72d8611297a // indirect @@ -400,14 +404,16 @@ require ( go.opentelemetry.io/otel v1.10.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.32.1 // indirect + go.opentelemetry.io/otel/metric v0.32.1 // indirect go.opentelemetry.io/otel/sdk v1.10.0 // indirect go.opentelemetry.io/otel/trace v1.10.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect + go.uber.org/zap v1.22.0 // indirect golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect - golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect + golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/tools v0.1.12 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index 4e868f866..2f8f6c7a4 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,10 @@ collectd.org v0.5.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= contrib.go.opencensus.io/exporter/prometheus v0.4.1/go.mod h1:t9wvfitlUjGXG2IXAZsuFq26mDGid/JwCEXp+gTG/9U= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= +github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= +github.com/99designs/keyring v1.2.1 h1:tYLp1ULvO7i3fI5vE21ReQuj99QFSs7lGm0xWyJo87o= +github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-amqp-common-go/v3 v3.2.3 h1:uDF62mbd9bypXWi19V1bN5NZEO84JqgmI5G73ibAmrk= @@ -345,6 +349,10 @@ github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:W github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/ashanbrown/forbidigo v1.1.0/go.mod h1:vVW7PEdqEFqapJe95xHkTfB1+XvZXBFg8t0sG2FIxmI= github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a/go.mod h1:oG9Dnez7/ESBqc4EdrdNlryeo7d0KcW1ftXHm7nU/UU= +github.com/awnumar/memcall v0.1.2 h1:7gOfDTL+BJ6nnbtAp9+HQzUFjtP1hEseRQq8eP055QY= +github.com/awnumar/memcall v0.1.2/go.mod h1:S911igBPR9CThzd/hYQQmTc9SWNu3ZHIlCGaWsWsoJo= +github.com/awnumar/memguard v0.22.3 h1:b4sgUXtbUjhrGELPbuC62wU+BsPQy+8lkWed9Z+pj0Y= +github.com/awnumar/memguard v0.22.3/go.mod h1:mmGunnffnLHlxE5rRgQc3j+uwPZ27eYb61ccr8Clz2Y= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.19.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -728,6 +736,8 @@ github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/daixiang0/gci v0.2.8/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc= +github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= +github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -801,6 +811,8 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM= +github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/dynatrace-oss/dynatrace-metric-utils-go v0.5.0 h1:wHGPJSXvwKQVf/XfhjUPyrhpcPKWNy8F3ikH+eiwoBg= github.com/dynatrace-oss/dynatrace-metric-utils-go v0.5.0/go.mod h1:PseHFo8Leko7J4A/TfZ6kkHdkzKBLUta6hRZR/OEbbc= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -1118,6 +1130,8 @@ github.com/gocql/gocql v0.0.0-20211222173705-d73e6b1002a7/go.mod h1:3gM2c4D3AnkI github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -1355,6 +1369,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/gwos/tcg/sdk v0.0.0-20220621192633-df0eac0a1a4c h1:pVr0TkSFnMP4BWSsEak/4bxD8/K+foJ9V8DGyZ6PIDE= github.com/gwos/tcg/sdk v0.0.0-20220621192633-df0eac0a1a4c/go.mod h1:4yzxLBACr76Is0AMAkE0F/fqWBk28p2tzeO06yDGR/Y= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= @@ -1916,6 +1932,8 @@ github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1: github.com/mozilla/tls-observatory v0.0.0-20201209171846-0547674fceff/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= +github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/multiplay/go-ts3 v1.0.1 h1:Ja8ho7UzUDNvNCwcDzPEPimLRub7MUqbD+sgMWkcR0A= github.com/multiplay/go-ts3 v1.0.1/go.mod h1:WIP3X0efye5ENZdXLu8LV4woCbPoc41wuMHx3EcU5CI= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -3140,6 +3158,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/plugins/inputs/http/http.go b/plugins/inputs/http/http.go index 4bc5afd0f..aa031e0b3 100644 --- a/plugins/inputs/http/http.go +++ b/plugins/inputs/http/http.go @@ -13,6 +13,7 @@ import ( "sync" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/internal" httpconfig "github.com/influxdata/telegraf/plugins/common/http" "github.com/influxdata/telegraf/plugins/inputs" @@ -30,8 +31,8 @@ type HTTP struct { Headers map[string]string `toml:"headers"` // HTTP Basic Auth Credentials - Username string `toml:"username"` - Password string `toml:"password"` + Username config.Secret `toml:"username"` + Password config.Secret `toml:"password"` // Absolute path to file with Bearer token BearerToken string `toml:"bearer_token"` @@ -134,8 +135,8 @@ func (h *HTTP) gatherURL( } } - if h.Username != "" || h.Password != "" { - request.SetBasicAuth(h.Username, h.Password) + if err := h.setRequestAuth(request); err != nil { + return err } resp, err := h.client.Do(request) @@ -184,6 +185,23 @@ func (h *HTTP) gatherURL( return nil } +func (h *HTTP) setRequestAuth(request *http.Request) error { + username, err := h.Username.Get() + if err != nil { + return fmt.Errorf("getting username failed: %v", err) + } + defer config.ReleaseSecret(username) + password, err := h.Password.Get() + if err != nil { + return fmt.Errorf("getting password failed: %v", err) + } + defer config.ReleaseSecret(password) + if len(username) != 0 || len(password) != 0 { + request.SetBasicAuth(string(username), string(password)) + } + return nil +} + func makeRequestBodyReader(contentEncoding, body string) (io.Reader, error) { if body == "" { return nil, nil diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 7cb2645a3..c5550e213 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -206,7 +206,7 @@ func (q *Query) parse(acc telegraf.Accumulator, rows *dbsql.Rows, t time.Time) ( type SQL struct { Driver string `toml:"driver"` - Dsn string `toml:"dsn"` + Dsn config.Secret `toml:"dsn"` Timeout config.Duration `toml:"timeout"` MaxIdleTime config.Duration `toml:"connection_max_idle_time"` MaxLifetime config.Duration `toml:"connection_max_life_time"` @@ -229,8 +229,8 @@ func (s *SQL) Init() error { return errors.New("missing SQL driver option") } - if s.Dsn == "" { - return errors.New("missing data source name (DSN) option") + if err := s.checkDSN(); err != nil { + return err } if s.Timeout <= 0 { @@ -358,8 +358,13 @@ func (s *SQL) Start(_ telegraf.Accumulator) error { var err error // Connect to the database server - s.Log.Debugf("Connecting to %q...", s.Dsn) - s.db, err = dbsql.Open(s.driverName, s.Dsn) + dsn, err := s.Dsn.Get() + if err != nil { + return fmt.Errorf("getting DSN failed: %v", err) + } + defer config.ReleaseSecret(dsn) + s.Log.Debug("Connecting...") + s.db, err = dbsql.Open(s.driverName, string(dsn)) if err != nil { return err } @@ -371,7 +376,7 @@ func (s *SQL) Start(_ telegraf.Accumulator) error { s.db.SetMaxIdleConns(s.MaxIdleConnections) // Test if the connection can be established - s.Log.Debugf("Testing connectivity...") + s.Log.Debug("Testing connectivity...") ctx, cancel := context.WithTimeout(context.Background(), time.Duration(s.Timeout)) err = s.db.PingContext(ctx) cancel() @@ -465,3 +470,15 @@ func (s *SQL) executeQuery(ctx context.Context, acc telegraf.Accumulator, q Quer return err } + +func (s *SQL) checkDSN() error { + dsn, err := s.Dsn.Get() + if err != nil { + return fmt.Errorf("getting DSN failed: %w", err) + } + defer config.ReleaseSecret(dsn) + if len(dsn) == 0 { + return errors.New("missing data source name (DSN) option") + } + return nil +} diff --git a/plugins/inputs/sql/sql_test.go b/plugins/inputs/sql/sql_test.go index f33898422..2a00828d4 100644 --- a/plugins/inputs/sql/sql_test.go +++ b/plugins/inputs/sql/sql_test.go @@ -12,6 +12,7 @@ import ( "github.com/testcontainers/testcontainers-go/wait" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/testutil" ) @@ -99,14 +100,11 @@ func TestMariaDBIntegration(t *testing.T) { for _, tt := range testset { t.Run(tt.name, func(t *testing.T) { // Setup the plugin-under-test + dsn := fmt.Sprintf("root:%s@tcp(%s:%s)/%s", passwd, container.Address, container.Ports[port], database) + secret := config.NewSecret([]byte(dsn)) plugin := &SQL{ - Driver: "maria", - Dsn: fmt.Sprintf("root:%s@tcp(%s:%s)/%s", - passwd, - container.Address, - container.Ports[port], - database, - ), + Driver: "maria", + Dsn: secret, Queries: tt.queries, Log: logger, } @@ -114,14 +112,11 @@ func TestMariaDBIntegration(t *testing.T) { var acc testutil.Accumulator // Startup the plugin - err := plugin.Init() - require.NoError(t, err) - err = plugin.Start(&acc) - require.NoError(t, err) + require.NoError(t, plugin.Init()) + require.NoError(t, plugin.Start(&acc)) // Gather - err = plugin.Gather(&acc) - require.NoError(t, err) + require.NoError(t, plugin.Gather(&acc)) require.Len(t, acc.Errors, 0) // Stopping the plugin @@ -204,14 +199,11 @@ func TestPostgreSQLIntegration(t *testing.T) { for _, tt := range testset { t.Run(tt.name, func(t *testing.T) { // Setup the plugin-under-test + dsn := fmt.Sprintf("postgres://postgres:%v@%v:%v/%v", passwd, container.Address, container.Ports[port], database) + secret := config.NewSecret([]byte(dsn)) plugin := &SQL{ - Driver: "pgx", - Dsn: fmt.Sprintf("postgres://postgres:%v@%v:%v/%v", - passwd, - container.Address, - container.Ports[port], - database, - ), + Driver: "pgx", + Dsn: secret, Queries: tt.queries, Log: logger, } @@ -219,14 +211,11 @@ func TestPostgreSQLIntegration(t *testing.T) { var acc testutil.Accumulator // Startup the plugin - err := plugin.Init() - require.NoError(t, err) - err = plugin.Start(&acc) - require.NoError(t, err) + require.NoError(t, plugin.Init()) + require.NoError(t, plugin.Start(&acc)) // Gather - err = plugin.Gather(&acc) - require.NoError(t, err) + require.NoError(t, plugin.Gather(&acc)) require.Len(t, acc.Errors, 0) // Stopping the plugin @@ -305,13 +294,11 @@ func TestClickHouseIntegration(t *testing.T) { for _, tt := range testset { t.Run(tt.name, func(t *testing.T) { // Setup the plugin-under-test + dsn := fmt.Sprintf("tcp://%v:%v?username=%v", container.Address, container.Ports[port], user) + secret := config.NewSecret([]byte(dsn)) plugin := &SQL{ - Driver: "clickhouse", - Dsn: fmt.Sprintf("tcp://%v:%v?username=%v", - container.Address, - container.Ports[port], - user, - ), + Driver: "clickhouse", + Dsn: secret, Queries: tt.queries, Log: logger, } @@ -319,14 +306,11 @@ func TestClickHouseIntegration(t *testing.T) { var acc testutil.Accumulator // Startup the plugin - err := plugin.Init() - require.NoError(t, err) - err = plugin.Start(&acc) - require.NoError(t, err) + require.NoError(t, plugin.Init()) + require.NoError(t, plugin.Start(&acc)) // Gather - err = plugin.Gather(&acc) - require.NoError(t, err) + require.NoError(t, plugin.Gather(&acc)) require.Len(t, acc.Errors, 0) // Stopping the plugin diff --git a/plugins/secretstores/all/all.go b/plugins/secretstores/all/all.go new file mode 100644 index 000000000..1a6c64721 --- /dev/null +++ b/plugins/secretstores/all/all.go @@ -0,0 +1 @@ +package all diff --git a/plugins/secretstores/all/jose.go b/plugins/secretstores/all/jose.go new file mode 100644 index 000000000..9613f229a --- /dev/null +++ b/plugins/secretstores/all/jose.go @@ -0,0 +1,5 @@ +//go:build !custom || secretstores || secretstores.jose + +package all + +import _ "github.com/influxdata/telegraf/plugins/secretstores/jose" // register plugin diff --git a/plugins/secretstores/all/os.go b/plugins/secretstores/all/os.go new file mode 100644 index 000000000..6c449aac9 --- /dev/null +++ b/plugins/secretstores/all/os.go @@ -0,0 +1,5 @@ +//go:build !custom || secretstores || secretstores.os + +package all + +import _ "github.com/influxdata/telegraf/plugins/secretstores/os" // register plugin diff --git a/plugins/secretstores/deprecations.go b/plugins/secretstores/deprecations.go new file mode 100644 index 000000000..2ca5872b7 --- /dev/null +++ b/plugins/secretstores/deprecations.go @@ -0,0 +1,6 @@ +package secretstores + +import "github.com/influxdata/telegraf" + +// Deprecations lists the deprecated plugins +var Deprecations = map[string]telegraf.DeprecationInfo{} diff --git a/plugins/secretstores/jose/README.md b/plugins/secretstores/jose/README.md new file mode 100644 index 000000000..1d29890d2 --- /dev/null +++ b/plugins/secretstores/jose/README.md @@ -0,0 +1,38 @@ +# Javascript Object Signing and Encryption Secret-store Plugin + +The `jose` plugin allows to manage and store secrets locally +protected by the [Javascript Object Signing and Encryption][jose] algorithm. + +To manage your secrets of this secret-store, you should use the +[secrets command of Telegraf](/docs/COMMANDS_AND_FLAGS.md#secrets-management). + +## Configuration + +```toml @sample.conf +# File based Javascript Object Signing and Encryption based secret-store +[[secretstores.jose]] + ## Unique identifier for the secret-store. + ## This id can later be used in plugins to reference the secrets + ## in this secret-store via @{:} (mandatory) + id = "secretstore" + + ## Directory for storing the secrets + # path = "secrets" + + ## Password to access the secrets. + ## If no password is specified here, Telegraf will prompt for it at startup time. + # password = "" +``` + +Each secret is stored in an individual file in the subdirectory specified +using the `path` parameter. To access the secrets, a password is required. +This password can be specified using the `password` parameter containing a +string, an environment variable or as a reference to a secret in another +secret store. If `password` is not specified in the config, you will be +prompted for the password at startup. + +__Please note:__ All secrets in this secret store are encrypted using +the same password. If you need individual passwords for each `jose` +secret, please use multiple instances of this plugin. + +[jose]: https://github.com/dvsekhvalnov/jose2go diff --git a/plugins/secretstores/jose/jose.go b/plugins/secretstores/jose/jose.go new file mode 100644 index 000000000..b0ccf3304 --- /dev/null +++ b/plugins/secretstores/jose/jose.go @@ -0,0 +1,108 @@ +//go:generate ../../../tools/readme_config_includer/generator +package jose + +import ( + _ "embed" + "errors" + "fmt" + + "github.com/99designs/keyring" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/plugins/secretstores" +) + +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embed the sampleConfig data. +// +//go:embed sample.conf +var sampleConfig string + +type Jose struct { + ID string `toml:"id"` + Path string `toml:"path"` + Password config.Secret `toml:"password"` + + ring keyring.Keyring +} + +func (*Jose) SampleConfig() string { + return sampleConfig +} + +// Init initializes all internals of the secret-store +func (j *Jose) Init() error { + defer j.Password.Destroy() + + if j.ID == "" { + return errors.New("id missing") + } + + passwd, err := j.Password.Get() + if err != nil { + return fmt.Errorf("getting password failed: %v", err) + } + + // Create the prompt-function in case we need it + promptFunc := keyring.TerminalPrompt + if len(passwd) != 0 { + promptFunc = keyring.FixedStringPrompt(string(passwd)) + } + + // Setup the actual keyring + cfg := keyring.Config{ + AllowedBackends: []keyring.BackendType{keyring.FileBackend}, + FileDir: j.Path, + FilePasswordFunc: promptFunc, + } + kr, err := keyring.Open(cfg) + if err != nil { + return fmt.Errorf("opening keyring failed: %v", err) + } + j.ring = kr + + return nil +} + +// Get searches for the given key and return the secret +func (j *Jose) Get(key string) ([]byte, error) { + item, err := j.ring.Get(key) + if err != nil { + return nil, err + } + + return item.Data, nil +} + +// Set sets the given secret for the given key +func (j *Jose) Set(key, value string) error { + item := keyring.Item{ + Key: key, + Data: []byte(value), + } + + return j.ring.Set(item) +} + +// List lists all known secret keys +func (j *Jose) List() ([]string, error) { + return j.ring.Keys() +} + +// GetResolver returns a function to resolve the given key. +func (j *Jose) GetResolver(key string) (telegraf.ResolveFunc, error) { + resolver := func() ([]byte, bool, error) { + s, err := j.Get(key) + return s, false, err + } + return resolver, nil +} + +// Register the secret-store on load. +func init() { + secretstores.Add("jose", func(id string) telegraf.SecretStore { + return &Jose{ + ID: id, + Path: "secrets", + } + }) +} diff --git a/plugins/secretstores/jose/jose_test.go b/plugins/secretstores/jose/jose_test.go new file mode 100644 index 000000000..a077f80b4 --- /dev/null +++ b/plugins/secretstores/jose/jose_test.go @@ -0,0 +1,204 @@ +package jose + +import ( + "os" + "testing" + + "github.com/influxdata/telegraf/config" + "github.com/stretchr/testify/require" +) + +func TestSampleConfig(t *testing.T) { + plugin := &Jose{} + require.NotEmpty(t, plugin.SampleConfig()) +} + +func TestInitFail(t *testing.T) { + tests := []struct { + name string + plugin *Jose + expected string + }{ + { + name: "invalid id", + plugin: &Jose{}, + expected: "id missing", + }, + { + name: "invalid password", + plugin: &Jose{ + ID: "test", + Password: config.NewSecret([]byte("@{unresolvable:secret}")), + }, + expected: "getting password failed", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.plugin.Init() + require.ErrorContains(t, err, tt.expected) + }) + } +} + +func TestSetListGet(t *testing.T) { + secrets := map[string]string{ + "a secret": "I won't tell", + "another one": "secret", + "foo": "bar", + } + + // Create a temporary directory we can use to store the secrets + testdir, err := os.MkdirTemp("", "jose-*") + require.NoError(t, err) + defer os.RemoveAll(testdir) + + // Initialize the plugin + plugin := &Jose{ + ID: "test", + Password: config.NewSecret([]byte("test")), + Path: testdir, + } + require.NoError(t, plugin.Init()) + + // Store the secrets + for k, v := range secrets { + require.NoError(t, plugin.Set(k, v)) + } + + // Check if the secrets were actually stored + entries, err := os.ReadDir(testdir) + require.NoError(t, err) + require.Len(t, entries, len(secrets)) + for _, e := range entries { + _, found := secrets[e.Name()] + require.True(t, found) + require.False(t, e.IsDir()) + } + + // List the secrets + keys, err := plugin.List() + require.NoError(t, err) + require.Len(t, keys, len(secrets)) + for _, k := range keys { + _, found := secrets[k] + require.True(t, found) + } + + // Get the secrets + require.Len(t, keys, len(secrets)) + for _, k := range keys { + value, err := plugin.Get(k) + require.NoError(t, err) + v, found := secrets[k] + require.True(t, found) + require.Equal(t, v, string(value)) + } +} + +func TestResolver(t *testing.T) { + secretKey := "a secret" + secretVal := "I won't tell" + + // Create a temporary directory we can use to store the secrets + testdir, err := os.MkdirTemp("", "jose-*") + require.NoError(t, err) + defer os.RemoveAll(testdir) + + // Initialize the plugin + plugin := &Jose{ + ID: "test", + Password: config.NewSecret([]byte("test")), + Path: testdir, + } + require.NoError(t, plugin.Init()) + require.NoError(t, plugin.Set(secretKey, secretVal)) + + // Get the resolver + resolver, err := plugin.GetResolver(secretKey) + require.NoError(t, err) + require.NotNil(t, resolver) + s, dynamic, err := resolver() + require.NoError(t, err) + require.False(t, dynamic) + require.Equal(t, secretVal, string(s)) +} + +func TestResolverInvalid(t *testing.T) { + secretKey := "a secret" + secretVal := "I won't tell" + + // Create a temporary directory we can use to store the secrets + testdir, err := os.MkdirTemp("", "jose-*") + require.NoError(t, err) + defer os.RemoveAll(testdir) + + // Initialize the plugin + plugin := &Jose{ + ID: "test", + Password: config.NewSecret([]byte("test")), + Path: testdir, + } + require.NoError(t, plugin.Init()) + require.NoError(t, plugin.Set(secretKey, secretVal)) + + // Get the resolver + resolver, err := plugin.GetResolver("foo") + require.NoError(t, err) + require.NotNil(t, resolver) + _, _, err = resolver() + require.Error(t, err) +} + +func TestGetNonExistant(t *testing.T) { + secretKey := "a secret" + secretVal := "I won't tell" + + // Create a temporary directory we can use to store the secrets + testdir, err := os.MkdirTemp("", "jose-*") + require.NoError(t, err) + defer os.RemoveAll(testdir) + + // Initialize the plugin + plugin := &Jose{ + ID: "test", + Password: config.NewSecret([]byte("test")), + Path: testdir, + } + require.NoError(t, plugin.Init()) + require.NoError(t, plugin.Set(secretKey, secretVal)) + + // Get the resolver + _, err = plugin.Get("foo") + require.EqualError(t, err, "The specified item could not be found in the keyring") +} + +func TestGetInvalidPassword(t *testing.T) { + secretKey := "a secret" + secretVal := "I won't tell" + + // Create a temporary directory we can use to store the secrets + testdir, err := os.MkdirTemp("", "jose-*") + require.NoError(t, err) + defer os.RemoveAll(testdir) + + // Initialize the stored secrets + creator := &Jose{ + ID: "test", + Password: config.NewSecret([]byte("test")), + Path: testdir, + } + require.NoError(t, creator.Init()) + require.NoError(t, creator.Set(secretKey, secretVal)) + + // Initialize the plugin with a wrong password + // and try to access an existing secret + plugin := &Jose{ + ID: "test", + Password: config.NewSecret([]byte("lala")), + Path: testdir, + } + require.NoError(t, plugin.Init()) + _, err = plugin.Get(secretKey) + require.ErrorContains(t, err, "integrity check failed") +} diff --git a/plugins/secretstores/jose/sample.conf b/plugins/secretstores/jose/sample.conf new file mode 100644 index 000000000..844eb4800 --- /dev/null +++ b/plugins/secretstores/jose/sample.conf @@ -0,0 +1,13 @@ +# File based Javascript Object Signing and Encryption based secret-store +[[secretstores.jose]] + ## Unique identifier for the secret-store. + ## This id can later be used in plugins to reference the secrets + ## in this secret-store via @{:} (mandatory) + id = "secretstore" + + ## Directory for storing the secrets + # path = "secrets" + + ## Password to access the secrets. + ## If no password is specified here, Telegraf will prompt for it at startup time. + # password = "" diff --git a/plugins/secretstores/os/README.md b/plugins/secretstores/os/README.md new file mode 100644 index 000000000..af89c6cfc --- /dev/null +++ b/plugins/secretstores/os/README.md @@ -0,0 +1,115 @@ +# OS Secret-store Plugin + +The `os` plugin allows to manage and store secrets using the native Operating +System keyring. For Windows this plugin uses the credential manager, on Linux +the kernel keyring is used and on MacOS we use the Keychain implementation. + +To manage your secrets you can either use the +[secrets command of Telegraf](/docs/COMMANDS_AND_FLAGS.md#secrets-management) +or the tools that natively comes with your operating system. + +## Configuration + +The configuration differs slightly depending on the Operating System. We first +describe the common options here and the refer to the individual interpretation +or options in the following sections. + +All secret-store implementations require an `id` to be able to reference the +store when specifying the secret. The `id` needs to be unique in the +configuration. + +For all operating systems, the keyring name can be chosen using the `keyring` +parameter. However, the interpretation is slightly different on the individual +implementations. + +The `dynamic` flag allows to indicate secrets that change during the runtime of +Telegraf. I.e. when set to `true`, the secret will be read from the secret-store +on every access by a plugin. If set to `false`, all secrets in the secret store +are assumed to be static and are only read once at startup of Telegraf. + +### Docker + +Access to the kernel keyring is __disabled by default__ in docker containers +(see [documentation](https://docs.docker.com/engine/security/seccomp/)). +In this case you will get an +`opening keyring failed: Specified keyring backend not available` error! + +You can enable access to the kernel keyring, but as the keyring is __not__ +namespaced, you should be aware of the security implication! One implication +is for example that keys added in one container are accessible by __all__ +other containers running on the same host, not only within the same container. + +### Windows + +```toml @sample_windows.conf +# Operating System native secret-store +[[secretstores.os]] + ## Unique identifier for the secret-store. + ## This id can later be used in plugins to reference the secrets + ## in this secret-store via @{:} (mandatory) + id = "secretstore" + + ## Keyring of the secrets + ## In Windows, keys follow a fixed pattern in the form `::`. Please keep this in mind + ## when creating secrets with the Windows credential tool. + # keyring = "telegraf" + # collection = "" + + ## Allow dynamic secrets that are updated during runtime of telegraf + # dynamic = false +``` + +On Windows you can use the Credential Manager Control panel or +[Telegraf](../../../cmd/telegraf/README.md) to manage your secrets. +Please use _generic credentials_ and respect the special +`::` format of the secret key. The +secret value needs to be stored in the `Password` field. + +### Linux + +```toml @sample_linux.conf +# Operating System native secret-store +[[secretstores.os]] + ## Unique identifier for the secret-store. + ## This id can later be used in plugins to reference the secrets + ## in this secret-store via @{:} (mandatory) + id = "secretstore" + + ## Keyring name used for the secrets + # keyring = "telegraf" + + ## Allow dynamic secrets that are updated during runtime of telegraf + # dynamic = false +``` + +On Linux the kernel keyring in the `user` scope is used to store the +secrets. The `collection` setting is ignored on Linux. + +### MacOS + +```toml @sample_darwin.conf +# Operating System native secret-store +[[secretstores.os]] + ## Unique identifier for the secret-store. + ## This id can later be used in plugins to reference the secrets + ## in this secret-store via @{:} (mandatory) + id = "secretstore" + + ## MacOS' Keychain name and service name + # keyring = "telegraf" + # collection = "" + + ## MacOS' Keychain password + ## If no password is specified here, Telegraf will prompt for it at startup time. + # password = "" + + ## Allow dynamic secrets that are updated during runtime of telegraf + # dynamic = false +``` + +On MacOS the Keychain implementation is used. Here the `keyring` parameter +corresponds to the Keychain name and the `collection` to the optional Keychain +service name. Additionally a password is required to access the Keychain. +The `password` itself is also a secret and can be a string, an environment +variable or a reference to a secret stored in another secret-store. +If `password` is omitted, you will be prompted for the password on startup. diff --git a/plugins/secretstores/os/os.go b/plugins/secretstores/os/os.go new file mode 100644 index 000000000..73dd99fc1 --- /dev/null +++ b/plugins/secretstores/os/os.go @@ -0,0 +1,97 @@ +//go:build darwin || linux || windows +// +build darwin linux windows + +//go:generate ../../../tools/readme_config_includer/generator +package os + +import ( + "errors" + "fmt" + + "github.com/99designs/keyring" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/plugins/secretstores" +) + +type OS struct { + ID string `toml:"id"` + Keyring string `toml:"keyring"` + Collection string `toml:"collection"` + Dynamic bool `toml:"dynamic"` + Password config.Secret `toml:"password"` + + ring keyring.Keyring +} + +func (*OS) SampleConfig() string { + return sampleConfig +} + +// Init initializes all internals of the secret-store +func (o *OS) Init() error { + defer o.Password.Destroy() + + if o.ID == "" { + return errors.New("id missing") + } + + // Set defaults + if o.Keyring == "" { + o.Keyring = "telegraf" + } + + // Setup the actual keyring + cfg, err := o.createKeyringConfig() + if err != nil { + return fmt.Errorf("getting keyring config failed: %v", err) + } + kr, err := keyring.Open(cfg) + if err != nil { + return fmt.Errorf("opening keyring failed: %v", err) + } + o.ring = kr + + return nil +} + +// Get searches for the given key and return the secret +func (o *OS) Get(key string) ([]byte, error) { + item, err := o.ring.Get(key) + if err != nil { + return nil, err + } + + return item.Data, nil +} + +// Set sets the given secret for the given key +func (o *OS) Set(key, value string) error { + item := keyring.Item{ + Key: key, + Data: []byte(value), + } + + return o.ring.Set(item) +} + +// List lists all known secret keys +func (o *OS) List() ([]string, error) { + return o.ring.Keys() +} + +// GetResolver returns a function to resolve the given key. +func (o *OS) GetResolver(key string) (telegraf.ResolveFunc, error) { + resolver := func() ([]byte, bool, error) { + s, err := o.Get(key) + return s, o.Dynamic, err + } + return resolver, nil +} + +// Register the secret-store on load. +func init() { + secretstores.Add("os", func(id string) telegraf.SecretStore { + return &OS{ID: id} + }) +} diff --git a/plugins/secretstores/os/os_darwin.go b/plugins/secretstores/os/os_darwin.go new file mode 100644 index 000000000..26ae211b6 --- /dev/null +++ b/plugins/secretstores/os/os_darwin.go @@ -0,0 +1,39 @@ +//go:build darwin +// +build darwin + +package os + +import ( + _ "embed" + "fmt" + + "github.com/99designs/keyring" + + "github.com/influxdata/telegraf/config" +) + +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embed the sampleConfig data. +// +//go:embed sample_darwin.conf +var sampleConfig string + +func (o *OS) createKeyringConfig() (keyring.Config, error) { + passwd, err := o.Password.Get() + if err != nil { + return keyring.Config{}, fmt.Errorf("getting password failed: %v", err) + } + defer config.ReleaseSecret(passwd) + + // Create the prompt-function in case we need it + promptFunc := keyring.TerminalPrompt + if len(passwd) != 0 { + promptFunc = keyring.FixedStringPrompt(string(passwd)) + } + + return keyring.Config{ + ServiceName: o.Collection, + AllowedBackends: []keyring.BackendType{keyring.KeychainBackend}, + KeychainName: o.Keyring, + KeychainPasswordFunc: promptFunc, + }, nil +} diff --git a/plugins/secretstores/os/os_linux.go b/plugins/secretstores/os/os_linux.go new file mode 100644 index 000000000..9e078f256 --- /dev/null +++ b/plugins/secretstores/os/os_linux.go @@ -0,0 +1,27 @@ +//go:build linux +// +build linux + +package os + +import ( + _ "embed" + + "github.com/99designs/keyring" +) + +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embed the sampleConfig data. +// +//go:embed sample_linux.conf +var sampleConfig string + +func (o *OS) createKeyringConfig() (keyring.Config, error) { + if o.Keyring == "" { + o.Keyring = "telegraf" + } + return keyring.Config{ + ServiceName: o.Keyring, + AllowedBackends: []keyring.BackendType{keyring.KeyCtlBackend}, + KeyCtlScope: "user", + KeyCtlPerm: 0x3f3f0000, // "alswrvalswrv------------" + }, nil +} diff --git a/plugins/secretstores/os/os_test.go b/plugins/secretstores/os/os_test.go new file mode 100644 index 000000000..73aee721a --- /dev/null +++ b/plugins/secretstores/os/os_test.go @@ -0,0 +1,89 @@ +//go:build darwin || linux || windows +// +build darwin linux windows + +package os + +import ( + "testing" + + "github.com/influxdata/telegraf/internal/choice" + "github.com/stretchr/testify/require" +) + +// In docker, access to the keyring is disabled by default see +// https://docs.docker.com/engine/security/seccomp/. +// You will see the following error then. +const dockerErr = "opening keyring failed: Specified keyring backend not available" + +func TestSampleConfig(t *testing.T) { + plugin := &OS{} + require.NotEmpty(t, plugin.SampleConfig()) +} + +func TestInitFail(t *testing.T) { + tests := []struct { + name string + plugin *OS + expected string + }{ + { + name: "invalid id", + plugin: &OS{}, + expected: "id missing", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.plugin.Init() + require.ErrorContains(t, err, tt.expected) + }) + } +} + +func TestResolverInvalid(t *testing.T) { + plugin := &OS{ID: "test"} + + // In docker, access to the keyring is disabled by default + // see https://docs.docker.com/engine/security/seccomp/. + err := plugin.Init() + if err != nil && err.Error() == dockerErr { + t.Skip("Kernel keyring not available!") + } + require.NoError(t, err) + + // Make sure the key does not exist and try to read that key + testKey := "foobar secret key" + keys, err := plugin.List() + require.NoError(t, err) + for choice.Contains(testKey, keys) { + testKey += "x" + } + // Get the resolver + resolver, err := plugin.GetResolver(testKey) + require.NoError(t, err) + require.NotNil(t, resolver) + _, _, err = resolver() + require.Error(t, err) +} + +func TestGetNonExisting(t *testing.T) { + plugin := &OS{ID: "test"} + + // In docker, access to the keyring is disabled by default + // see https://docs.docker.com/engine/security/seccomp/. + err := plugin.Init() + if err != nil && err.Error() == dockerErr { + t.Skip("Kernel keyring not available!") + } + require.NoError(t, err) + + // Make sure the key does not exist and try to read that key + testKey := "foobar secret key" + keys, err := plugin.List() + require.NoError(t, err) + for choice.Contains(testKey, keys) { + testKey += "x" + } + _, err = plugin.Get(testKey) + require.EqualError(t, err, "The specified item could not be found in the keyring") +} diff --git a/plugins/secretstores/os/os_unsupported.go b/plugins/secretstores/os/os_unsupported.go new file mode 100644 index 000000000..cf1f67e04 --- /dev/null +++ b/plugins/secretstores/os/os_unsupported.go @@ -0,0 +1 @@ +package os diff --git a/plugins/secretstores/os/os_windows.go b/plugins/secretstores/os/os_windows.go new file mode 100644 index 000000000..806ba35d5 --- /dev/null +++ b/plugins/secretstores/os/os_windows.go @@ -0,0 +1,23 @@ +//go:build windows +// +build windows + +package os + +import ( + _ "embed" + + "github.com/99designs/keyring" +) + +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embed the sampleConfig data. +// +//go:embed sample_windows.conf +var sampleConfig string + +func (o *OS) createKeyringConfig() (keyring.Config, error) { + return keyring.Config{ + ServiceName: o.Keyring, + AllowedBackends: []keyring.BackendType{keyring.WinCredBackend}, + WinCredPrefix: o.Collection, + }, nil +} diff --git a/plugins/secretstores/os/sample_darwin.conf b/plugins/secretstores/os/sample_darwin.conf new file mode 100644 index 000000000..d91df3437 --- /dev/null +++ b/plugins/secretstores/os/sample_darwin.conf @@ -0,0 +1,17 @@ +# Operating System native secret-store +[[secretstores.os]] + ## Unique identifier for the secret-store. + ## This id can later be used in plugins to reference the secrets + ## in this secret-store via @{:} (mandatory) + id = "secretstore" + + ## MacOS' Keychain name and service name + # keyring = "telegraf" + # collection = "" + + ## MacOS' Keychain password + ## If no password is specified here, Telegraf will prompt for it at startup time. + # password = "" + + ## Allow dynamic secrets that are updated during runtime of telegraf + # dynamic = false diff --git a/plugins/secretstores/os/sample_linux.conf b/plugins/secretstores/os/sample_linux.conf new file mode 100644 index 000000000..87d993d0f --- /dev/null +++ b/plugins/secretstores/os/sample_linux.conf @@ -0,0 +1,12 @@ +# Operating System native secret-store +[[secretstores.os]] + ## Unique identifier for the secret-store. + ## This id can later be used in plugins to reference the secrets + ## in this secret-store via @{:} (mandatory) + id = "secretstore" + + ## Keyring name used for the secrets + # keyring = "telegraf" + + ## Allow dynamic secrets that are updated during runtime of telegraf + # dynamic = false diff --git a/plugins/secretstores/os/sample_windows.conf b/plugins/secretstores/os/sample_windows.conf new file mode 100644 index 000000000..def9b074e --- /dev/null +++ b/plugins/secretstores/os/sample_windows.conf @@ -0,0 +1,15 @@ +# Operating System native secret-store +[[secretstores.os]] + ## Unique identifier for the secret-store. + ## This id can later be used in plugins to reference the secrets + ## in this secret-store via @{:} (mandatory) + id = "secretstore" + + ## Keyring of the secrets + ## In Windows, keys follow a fixed pattern in the form `::`. Please keep this in mind + ## when creating secrets with the Windows credential tool. + # keyring = "telegraf" + # collection = "" + + ## Allow dynamic secrets that are updated during runtime of telegraf + # dynamic = false diff --git a/plugins/secretstores/registry.go b/plugins/secretstores/registry.go new file mode 100644 index 000000000..6dc8e5e81 --- /dev/null +++ b/plugins/secretstores/registry.go @@ -0,0 +1,16 @@ +package secretstores + +import ( + "github.com/influxdata/telegraf" +) + +// Creator is the function to create a new parser +type Creator func(id string) telegraf.SecretStore + +// SecretStores contains the registry of all known secret-stores +var SecretStores = map[string]Creator{} + +// Add adds a secret-store to the registry. Usually this function is called in the plugin's init function +func Add(name string, creator Creator) { + SecretStores[name] = creator +} diff --git a/secretstore.go b/secretstore.go new file mode 100644 index 000000000..69e92baff --- /dev/null +++ b/secretstore.go @@ -0,0 +1,25 @@ +package telegraf + +// SecretStore is an interface defining functions that a secret-store plugin must satisfy. +type SecretStore interface { + Initializer + PluginDescriber + + // Get searches for the given key and return the secret + Get(key string) ([]byte, error) + + // Set sets the given secret for the given key + Set(key, value string) error + + // List lists all known secret keys + List() ([]string, error) + + // GetResolver returns a function to resolve the given key. + GetResolver(key string) (ResolveFunc, error) +} + +// ResolveFunc is a function to resolve the secret. +// The returned flag indicates if the resolver is static (false), i.e. +// the secret will not change over time, or dynamic (true) to handle +// secrets that change over time (e.g. TOTP). +type ResolveFunc func() ([]byte, bool, error) diff --git a/tools/custom_builder/main.go b/tools/custom_builder/main.go index 7545f195f..69206e78f 100644 --- a/tools/custom_builder/main.go +++ b/tools/custom_builder/main.go @@ -18,6 +18,7 @@ var categories = []string{ "outputs", "parsers", "processors", + "secretstores", } const description = `