From c3e53193d27e93a02ab5a4b270fb84e92917656e Mon Sep 17 00:00:00 2001 From: Sven Rebhan <36194019+srebhan@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:52:22 +0200 Subject: [PATCH] feat(logging): Implement structured logging (#15751) Co-authored-by: Thomas Casteleyn --- CHANGELOG.md | 14 + cmd/telegraf/agent.conf | 23 +- cmd/telegraf/telegraf.go | 1 + config/config.go | 15 +- config/migration.go | 25 +- docs/CONFIGURATION.md | 12 +- logger.go | 3 + logger/event_logger.go | 4 +- logger/event_logger_test.go | 4 +- logger/handler.go | 36 +- logger/logger.go | 159 ++++----- logger/logger_test.go | 12 + logger/structured_logger.go | 79 +++++ logger/structured_logger_test.go | 322 ++++++++++++++++++ logger/{default_logger.go => text_logger.go} | 19 +- ...ult_logger_test.go => text_logger_test.go} | 122 ++++--- migrations/all/global_agent.go | 5 + migrations/global_agent/migration.go | 75 ++++ migrations/global_agent/migration_test.go | 104 ++++++ .../global_agent/testcases/default.conf | 100 ++++++ .../logtarget_eventlog/expected.conf | 10 + .../logtarget_eventlog/telegraf.conf | 102 ++++++ .../logtarget_eventlog_collision.conf | 102 ++++++ .../expected.conf | 10 + .../telegraf.conf | 102 ++++++ .../testcases/logtarget_file/expected.conf | 10 + .../testcases/logtarget_file/telegraf.conf | 102 ++++++ .../logtarget_file_no_logfile/expected.conf | 9 + .../logtarget_file_no_logfile/telegraf.conf | 102 ++++++ .../testcases/logtarget_stderr/expected.conf | 9 + .../testcases/logtarget_stderr/telegraf.conf | 102 ++++++ .../expected.conf | 9 + .../telegraf.conf | 102 ++++++ migrations/registry.go | 15 +- migrations/utils.go | 15 + plugins/outputs/postgresql/postgresql_test.go | 3 +- testutil/capturelog.go | 16 +- testutil/log.go | 16 +- 38 files changed, 1764 insertions(+), 206 deletions(-) create mode 100644 logger/structured_logger.go create mode 100644 logger/structured_logger_test.go rename logger/{default_logger.go => text_logger.go} (64%) rename logger/{default_logger_test.go => text_logger_test.go} (57%) create mode 100644 migrations/all/global_agent.go create mode 100644 migrations/global_agent/migration.go create mode 100644 migrations/global_agent/migration_test.go create mode 100644 migrations/global_agent/testcases/default.conf create mode 100644 migrations/global_agent/testcases/logtarget_eventlog/expected.conf create mode 100644 migrations/global_agent/testcases/logtarget_eventlog/telegraf.conf create mode 100644 migrations/global_agent/testcases/logtarget_eventlog_collision.conf create mode 100644 migrations/global_agent/testcases/logtarget_eventlog_with_logfile/expected.conf create mode 100644 migrations/global_agent/testcases/logtarget_eventlog_with_logfile/telegraf.conf create mode 100644 migrations/global_agent/testcases/logtarget_file/expected.conf create mode 100644 migrations/global_agent/testcases/logtarget_file/telegraf.conf create mode 100644 migrations/global_agent/testcases/logtarget_file_no_logfile/expected.conf create mode 100644 migrations/global_agent/testcases/logtarget_file_no_logfile/telegraf.conf create mode 100644 migrations/global_agent/testcases/logtarget_stderr/expected.conf create mode 100644 migrations/global_agent/testcases/logtarget_stderr/telegraf.conf create mode 100644 migrations/global_agent/testcases/logtarget_stderr_with_logfile/expected.conf create mode 100644 migrations/global_agent/testcases/logtarget_stderr_with_logfile/telegraf.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index fe7a0c9d5..b6f64d3b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,20 @@ # Changelog +## Unreleased + +### Important Changes + +- This release contains a logging overhaul as well as some new features for + logging (see PRs [#15556](https://github.com/influxdata/telegraf/pull/15556), + [#15629](https://github.com/influxdata/telegraf/pull/15629), + [#15677](https://github.com/influxdata/telegraf/pull/15677), + [#15695](https://github.com/influxdata/telegraf/pull/15695) and + [#15751](https://github.com/influxdata/telegraf/pull/15751)). + As a consequence the redunant `logtarget` setting is deprecated, `stderr` is + used if no `logfile` is provided, otherwise messages are logged to the given + file. For using the Windows `eventlog` set `logformat = "eventlog"`! + ## v1.31.3 [2024-08-12] ### Bugfixes diff --git a/cmd/telegraf/agent.conf b/cmd/telegraf/agent.conf index bb9ea0ab8..12fd81ac4 100644 --- a/cmd/telegraf/agent.conf +++ b/cmd/telegraf/agent.conf @@ -53,13 +53,12 @@ ## Log only error level messages. # quiet = false - ## Log target controls the destination for logs and can be one of "file", - ## "stderr" or, on Windows, "eventlog". When set to "file", the output file - ## is determined by the "logfile" setting. - # logtarget = "file" + ## Log format controls the way messages are logged and can be one of "text", + ## "structured" or, on Windows, "eventlog". + # logformat = "text" - ## Name of the file to be logged to when using the "file" logtarget. If set to - ## the empty string then logs are written to stderr. + ## Name of the file to be logged to or stderr if unset or empty. This + ## setting is ignored for the "eventlog" format. # logfile = "" ## The logfile will be rotated after the time interval specified. When set @@ -80,9 +79,9 @@ # log_with_timezone = "" ## Override default hostname, if empty use os.Hostname() - hostname = "" + # hostname = "" ## If set to true, do no set the "host" tag in the telegraf agent. - omit_hostname = false + # omit_hostname = false ## Method of translating SNMP objects. Can be "netsnmp" (deprecated) which ## translates by calling external programs snmptranslate and snmptable, @@ -95,7 +94,7 @@ ## the state in the file will be restored for the plugins. # statefile = "" - ## Flag to skip running processors after aggregators - ## By default, processors are run a second time after aggregators. Changing - ## this setting to true will skip the second run of processors. - # skip_processors_after_aggregators = false + ## Flag to skip running processors after aggregators + ## By default, processors are run a second time after aggregators. Changing + ## this setting to true will skip the second run of processors. + # skip_processors_after_aggregators = false diff --git a/cmd/telegraf/telegraf.go b/cmd/telegraf/telegraf.go index c6fa58093..def6de8e1 100644 --- a/cmd/telegraf/telegraf.go +++ b/cmd/telegraf/telegraf.go @@ -362,6 +362,7 @@ func (t *Telegraf) runAgent(ctx context.Context, reloadConfig bool) error { Debug: c.Agent.Debug || t.debug, Quiet: c.Agent.Quiet || t.quiet, LogTarget: c.Agent.LogTarget, + LogFormat: c.Agent.LogFormat, Logfile: c.Agent.Logfile, RotationInterval: time.Duration(c.Agent.LogfileRotationInterval), RotationMaxSize: int64(c.Agent.LogfileRotationMaxSize), diff --git a/config/config.go b/config/config.go index aad8af1e1..5ac7700a2 100644 --- a/config/config.go +++ b/config/config.go @@ -123,7 +123,7 @@ func NewConfig() *Config { Interval: Duration(10 * time.Second), RoundInterval: true, FlushInterval: Duration(10 * time.Second), - LogTarget: "file", + LogFormat: "text", LogfileRotationMaxArchives: 5, }, @@ -226,12 +226,15 @@ type AgentConfig struct { Quiet bool `toml:"quiet"` // Log target controls the destination for logs and can be one of "file", - // "stderr" or, on Windows, "eventlog". When set to "file", the output file - // is determined by the "logfile" setting. - LogTarget string `toml:"logtarget"` + // "stderr" or, on Windows, "eventlog". When set to "file", the output file + // is determined by the "logfile" setting + LogTarget string `toml:"logtarget" deprecated:"1.32.0;1.40.0;use 'logformat' and 'logfile' instead"` - // Name of the file to be logged to when using the "file" logtarget. If set to - // the empty string then logs are written to stderr. + // Log format controls the way messages are logged and can be one of "text", + // "structured" or, on Windows, "eventlog". + LogFormat string `toml:"logformat"` + + // Name of the file to be logged to or stderr if empty. Ignored for "eventlog" format. Logfile string `toml:"logfile"` // The file will be rotated after the time interval specified. When set diff --git a/config/migration.go b/config/migration.go index 520e1be14..837c4b6b8 100644 --- a/config/migration.go +++ b/config/migration.go @@ -151,8 +151,31 @@ func ApplyMigrations(data []byte) ([]byte, uint64, error) { return nil, 0, fmt.Errorf("assigning text failed: %w", err) } - // Do the actual plugin migration(s) var applied uint64 + // Do the actual global section migration(s) + for idx, s := range sections { + if strings.Contains(s.name, ".") { + continue + } + log.Printf("D! applying global migrations to section %q in line %d...", s.name, s.begin) + for _, migrate := range migrations.GlobalMigrations { + result, msg, err := migrate(s.name, s.content) + if err != nil { + if errors.Is(err, migrations.ErrNotApplicable) { + continue + } + return nil, 0, fmt.Errorf("migrating options of %q (line %d) failed: %w", s.name, s.begin, err) + } + if msg != "" { + log.Printf("I! Global section %q in line %d: %s", s.name, s.begin, msg) + } + s.raw = bytes.NewBuffer(result) + applied++ + } + sections[idx] = s + } + + // Do the actual plugin migration(s) for idx, s := range sections { migrate, found := migrations.PluginMigrations[s.name] if !found { diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index d67d5d5e4..313c6fa91 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -296,14 +296,14 @@ The agent table configures Telegraf and the defaults used across all plugins. - **quiet**: Log only error level messages. -- **logtarget**: - Log target controls the destination for logs and can be one of "file", - "stderr" or, on Windows, "eventlog". When set to "file", the output file is - determined by the "logfile" setting. +- **logformat**: + Log format controls the way messages are logged and can be one of "text", + "structured" or, on Windows, "eventlog". The output file (if any) is + determined by the `logfile` setting. - **logfile**: - Name of the file to be logged to when using the "file" logtarget. If set to - the empty string then logs are written to stderr. + Name of the file to be logged to or stderr if unset or empty. This + setting is ignored for the "eventlog" format. - **logfile_rotation_interval**: The logfile will be rotated after the time interval specified. When set to diff --git a/logger.go b/logger.go index 500b04b16..382d0dec8 100644 --- a/logger.go +++ b/logger.go @@ -75,6 +75,9 @@ type Logger interface { //nolint:interfacebloat // All functions are required // Level returns the configured log-level of the logger Level() LogLevel + // AddAttribute allows to add a key-value attribute to the logging output + AddAttribute(key string, value interface{}) + // Errorf logs an error message, patterned after log.Printf. Errorf(format string, args ...interface{}) // Error logs an error message, patterned after log.Print. diff --git a/logger/event_logger.go b/logger/event_logger.go index b8f2e5e51..c5847c93b 100644 --- a/logger/event_logger.go +++ b/logger/event_logger.go @@ -27,7 +27,7 @@ func (l *eventLogger) Close() error { return l.eventlog.Close() } -func (l *eventLogger) Print(level telegraf.LogLevel, _ time.Time, prefix string, args ...interface{}) { +func (l *eventLogger) Print(level telegraf.LogLevel, _ time.Time, prefix string, _ map[string]interface{}, args ...interface{}) { // Skip debug and beyond as they cannot be logged if level >= telegraf.Debug { return @@ -47,6 +47,8 @@ func (l *eventLogger) Print(level telegraf.LogLevel, _ time.Time, prefix string, if err != nil { l.errlog.Printf("E! Writing log message failed: %v", err) } + + // TODO attributes... } func createEventLogger(cfg *Config) (sink, error) { diff --git a/logger/event_logger_test.go b/logger/event_logger_test.go index 96734fc9c..f1ac4ef2a 100644 --- a/logger/event_logger_test.go +++ b/logger/event_logger_test.go @@ -54,7 +54,7 @@ func TestEventLogIntegration(t *testing.T) { t.Skip("Skipping integration test in short mode") } config := &Config{ - LogTarget: "eventlog", + LogFormat: "eventlog", Logfile: "", } require.NoError(t, SetupLogging(config)) @@ -76,7 +76,7 @@ func TestRestrictedEventLogIntegration(t *testing.T) { } config := &Config{ - LogTarget: "eventlog", + LogFormat: "eventlog", Quiet: true, } require.NoError(t, SetupLogging(config)) diff --git a/logger/handler.go b/logger/handler.go index bd96c5aab..7f23b32b0 100644 --- a/logger/handler.go +++ b/logger/handler.go @@ -6,6 +6,7 @@ import ( "io" "log" "os" + "strings" "sync" "time" @@ -13,10 +14,11 @@ import ( ) type entry struct { - timestamp time.Time - level telegraf.LogLevel - prefix string - args []interface{} + timestamp time.Time + level telegraf.LogLevel + prefix string + attributes map[string]interface{} + args []interface{} } type handler struct { @@ -60,7 +62,7 @@ func (h *handler) switchSink(impl sink, level telegraf.LogLevel, tz *time.Locati current := h.earlylogs.Front() for current != nil { e := current.Value.(*entry) - h.impl.Print(e.level, e.timestamp.In(h.timezone), e.prefix, e.args...) + h.impl.Print(e.level, e.timestamp.In(h.timezone), e.prefix, e.attributes, e.args...) next := current.Next() h.earlylogs.Remove(current) current = next @@ -69,12 +71,13 @@ func (h *handler) switchSink(impl sink, level telegraf.LogLevel, tz *time.Locati h.Unlock() } -func (h *handler) add(level telegraf.LogLevel, ts time.Time, prefix string, args ...interface{}) *entry { +func (h *handler) add(level telegraf.LogLevel, ts time.Time, prefix string, attr map[string]interface{}, args ...interface{}) *entry { e := &entry{ - timestamp: ts, - level: level, - prefix: prefix, - args: args, + timestamp: ts, + level: level, + prefix: prefix, + attributes: attr, + args: args, } h.Lock() @@ -109,7 +112,16 @@ type redirectLogger struct { writer io.Writer } -func (l *redirectLogger) Print(level telegraf.LogLevel, ts time.Time, prefix string, args ...interface{}) { - msg := append([]interface{}{ts.In(time.UTC).Format(time.RFC3339), " ", level.Indicator(), " ", prefix}, args...) +func (l *redirectLogger) Print(level telegraf.LogLevel, ts time.Time, prefix string, attr map[string]interface{}, args ...interface{}) { + var attrMsg string + if len(attr) > 0 { + var parts []string + for k, v := range attr { + parts = append(parts, fmt.Sprintf("%s=%v", k, v)) + } + attrMsg = " (" + strings.Join(parts, ",") + ")" + } + + msg := append([]interface{}{ts.In(time.UTC).Format(time.RFC3339), " ", level.Indicator(), " ", prefix + attrMsg}, args...) fmt.Fprintln(l.writer, msg...) } diff --git a/logger/logger.go b/logger/logger.go index 50840700a..535f4150b 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -1,6 +1,7 @@ package logger import ( + "errors" "fmt" "io" "log" @@ -21,13 +22,7 @@ var ( // sink interface that has to be implemented by a logging sink type sink interface { - Print(telegraf.LogLevel, time.Time, string, ...interface{}) -} - -// Attr represents an attribute appended to structured logging -type Attr struct { - Key string - Value interface{} + Print(telegraf.LogLevel, time.Time, string, map[string]interface{}, ...interface{}) } // logger is the actual implementation of the telegraf logger interface @@ -36,46 +31,25 @@ type logger struct { category string name string alias string - suffix string - prefix string - - onError []func() + prefix string + onError []func() + attributes map[string]interface{} } // New creates a new logging instance to be used in models func New(category, name, alias string) *logger { l := &logger{ - category: category, - name: name, - alias: alias, + category: category, + name: name, + alias: alias, + attributes: map[string]interface{}{"category": category, "plugin": name}, } - l.formatPrefix() - - return l -} - -// SubLogger creates a new logger with the given name added as suffix -func (l *logger) SubLogger(name string) telegraf.Logger { - suffix := l.suffix - if suffix != "" && name != "" { - suffix += "." + if alias != "" { + l.attributes["alias"] = alias } - suffix += name - nl := &logger{ - level: l.level, - category: l.category, - name: l.name, - alias: l.alias, - suffix: suffix, - } - nl.formatPrefix() - - return nl -} - -func (l *logger) formatPrefix() { + // Format the prefix l.prefix = l.category if l.prefix != "" && l.name != "" { @@ -88,13 +62,11 @@ func (l *logger) formatPrefix() { } l.prefix += l.alias - if l.suffix != "" { - l.prefix += "(" + l.suffix + ")" - } - if l.prefix != "" { l.prefix = "[" + l.prefix + "] " } + + return l } // Level returns the current log-level of the logger @@ -105,27 +77,14 @@ func (l *logger) Level() telegraf.LogLevel { return instance.level } -// SetLevel overrides the current log-level of the logger -func (l *logger) SetLevel(level telegraf.LogLevel) { - l.level = &level -} - -// SetLevel changes the log-level to the given one -func (l *logger) SetLogLevel(name string) error { - if name == "" { - return nil +// AddAttribute allows to add a key-value attribute to the logging output +func (l *logger) AddAttribute(key string, value interface{}) { + // Do not allow to overwrite general keys + switch key { + case "category", "plugin", "alias": + default: + l.attributes[key] = value } - level := telegraf.LogLevelFromString(name) - if level == telegraf.None { - return fmt.Errorf("invalid log-level %q", name) - } - l.SetLevel(level) - return nil -} - -// Register a callback triggered when errors are about to be written to the log -func (l *logger) RegisterErrorCallback(f func()) { - l.onError = append(l.onError, f) } // Error logging including callbacks @@ -179,7 +138,7 @@ func (l *logger) Trace(args ...interface{}) { func (l *logger) Print(level telegraf.LogLevel, ts time.Time, args ...interface{}) { // Check if we are in early logging state and store the message in this case if instance.impl == nil { - instance.add(level, ts, l.prefix, args...) + instance.add(level, ts, l.prefix, l.attributes, args...) } // Skip all messages with insufficient log-levels @@ -187,24 +146,45 @@ func (l *logger) Print(level telegraf.LogLevel, ts time.Time, args ...interface{ return } if instance.impl != nil { - instance.impl.Print(level, ts.In(instance.timezone), l.prefix, args...) + instance.impl.Print(level, ts.In(instance.timezone), l.prefix, l.attributes, args...) } else { msg := append([]interface{}{ts.In(instance.timezone).Format(time.RFC3339), " ", level.Indicator(), " ", l.prefix}, args...) instance.earlysink.Print(msg...) } } +// SetLevel overrides the current log-level of the logger +func (l *logger) SetLevel(level telegraf.LogLevel) { + l.level = &level +} + +// SetLevel changes the log-level to the given one +func (l *logger) SetLogLevel(name string) error { + if name == "" { + return nil + } + level := telegraf.LogLevelFromString(name) + if level == telegraf.None { + return fmt.Errorf("invalid log-level %q", name) + } + l.SetLevel(level) + return nil +} + +// Register a callback triggered when errors are about to be written to the log +func (l *logger) RegisterErrorCallback(f func()) { + l.onError = append(l.onError, f) +} + type Config struct { // will set the log level to DEBUG Debug bool // will set the log level to ERROR Quiet bool - //stderr, stdout, file or eventlog (Windows only) + // format and target of log messages LogTarget string - // will direct the logging output to a file. Empty string is - // interpreted as stderr. If there is an error opening the file the - // logger will fall back to stderr - Logfile string + LogFormat string + Logfile string // will rotate when current file at the specified time interval RotationInterval time.Duration // will rotate when current file size exceeds this parameter. @@ -222,6 +202,31 @@ type Config struct { // SetupLogging configures the logging output. func SetupLogging(cfg *Config) error { + // Issue deprecation warning for option + switch cfg.LogTarget { + case "": + // Best-case no target set or file already migrated... + case "stderr": + msg := "Agent setting %q is deprecated, please leave %q empty and remove this setting!" + deprecation := "The setting will be removed in v1.40.0." + log.Printf("W! "+msg+" "+deprecation, "logtarget", "logfile") + cfg.Logfile = "" + case "file": + msg := "Agent setting %q is deprecated, please just set %q and remove this setting!" + deprecation := "The setting will be removed in v1.40.0." + log.Printf("W! "+msg+" "+deprecation, "logtarget", "logfile") + case "eventlog": + msg := "Agent setting %q is deprecated, please set %q to %q and remove this setting!" + deprecation := "The setting will be removed in v1.40.0." + log.Printf("W! "+msg+" "+deprecation, "logtarget", "logformat", "eventlog") + if cfg.LogFormat != "" && cfg.LogFormat != "eventlog" { + return errors.New("contradicting setting between 'logtarget' and 'logformat'") + } + cfg.LogFormat = "eventlog" + default: + return fmt.Errorf("invalid deprecated 'logtarget' setting %q", cfg.LogTarget) + } + if cfg.Debug { cfg.logLevel = telegraf.Debug } @@ -236,8 +241,8 @@ func SetupLogging(cfg *Config) error { cfg.InstanceName = "telegraf" } - if cfg.LogTarget == "" || cfg.LogTarget == "file" && cfg.Logfile == "" { - cfg.LogTarget = "stderr" + if cfg.LogFormat == "" { + cfg.LogFormat = "text" } // Get configured timezone @@ -250,13 +255,12 @@ func SetupLogging(cfg *Config) error { return fmt.Errorf("setting logging timezone failed: %w", err) } - // Get the logging factory - creator, ok := registry[cfg.LogTarget] - if !ok { - return fmt.Errorf("unsupported log target: %s, using stderr", cfg.LogTarget) + // Get the logging factory and create the root instance + creator, found := registry[cfg.LogFormat] + if !found { + return fmt.Errorf("unsupported log-format: %s", cfg.LogFormat) } - // Create the root logging instance l, err := creator(cfg) if err != nil { return err @@ -268,7 +272,8 @@ func SetupLogging(cfg *Config) error { } // Update the logging instance - instance.switchSink(l, cfg.logLevel, tz, cfg.LogTarget == "stderr") + skipEarlyLogs := cfg.LogFormat == "text" && cfg.Logfile == "" + instance.switchSink(l, cfg.logLevel, tz, skipEarlyLogs) return nil } diff --git a/logger/logger_test.go b/logger/logger_test.go index fc2cafbfb..cca66fe1e 100644 --- a/logger/logger_test.go +++ b/logger/logger_test.go @@ -1,12 +1,24 @@ package logger import ( + "os" "testing" "github.com/influxdata/telegraf/selfstat" "github.com/stretchr/testify/require" ) +func TestTextLogTargetDefault(t *testing.T) { + instance = defaultHandler() + cfg := &Config{ + Quiet: true, + } + require.NoError(t, SetupLogging(cfg)) + logger, ok := instance.impl.(*textLogger) + require.Truef(t, ok, "logging instance is not a default-logger but %T", instance.impl) + require.Equal(t, logger.logger.Writer(), os.Stderr) +} + func TestErrorCounting(t *testing.T) { reg := selfstat.Register( "gather", diff --git a/logger/structured_logger.go b/logger/structured_logger.go new file mode 100644 index 000000000..dfe7dc275 --- /dev/null +++ b/logger/structured_logger.go @@ -0,0 +1,79 @@ +package logger + +import ( + "context" + "fmt" + "io" + "log" + "log/slog" + "os" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal/rotate" +) + +type structuredLogger struct { + handler slog.Handler + output io.Writer + + errlog *log.Logger +} + +func (l *structuredLogger) Close() error { + if closer, ok := l.output.(io.Closer); ok { + return closer.Close() + } + return nil +} + +func (l *structuredLogger) Print(level telegraf.LogLevel, ts time.Time, _ string, attr map[string]interface{}, args ...interface{}) { + record := slog.Record{ + Time: ts, + Message: fmt.Sprint(args...), + Level: slog.Level(level), + } + for k, v := range attr { + record.Add(k, v) + } + if err := l.handler.Handle(context.Background(), record); err != nil { + l.errlog.Printf("E! Writing log message failed: %v", err) + } +} + +var defaultStructuredHandlerOptions = &slog.HandlerOptions{ + Level: slog.Level(-99), + ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr { + // Translate the Telegraf log-levels to strings + if attr.Key == slog.LevelKey { + if level, ok := attr.Value.Any().(slog.Level); ok { + attr.Value = slog.StringValue(telegraf.LogLevel(level).String()) + } + } + return attr + }, +} + +func init() { + add("structured", func(cfg *Config) (sink, error) { + var writer io.Writer = os.Stderr + if cfg.Logfile != "" { + w, err := rotate.NewFileWriter( + cfg.Logfile, + cfg.RotationInterval, + cfg.RotationMaxSize, + cfg.RotationMaxArchives, + ) + if err != nil { + return nil, err + } + writer = w + } + + return &structuredLogger{ + handler: slog.NewJSONHandler(writer, defaultStructuredHandlerOptions), + output: writer, + errlog: log.New(os.Stderr, "", 0), + }, nil + }) +} diff --git a/logger/structured_logger_test.go b/logger/structured_logger_test.go new file mode 100644 index 000000000..89208563a --- /dev/null +++ b/logger/structured_logger_test.go @@ -0,0 +1,322 @@ +package logger + +import ( + "encoding/json" + "io" + "log" + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/influxdata/telegraf" + "github.com/stretchr/testify/require" +) + +func TestStructuredStderr(t *testing.T) { + instance = defaultHandler() + cfg := &Config{ + LogFormat: "structured", + Quiet: true, + } + require.NoError(t, SetupLogging(cfg)) + logger, ok := instance.impl.(*structuredLogger) + require.Truef(t, ok, "logging instance is not a structured-logger but %T", instance.impl) + require.Equal(t, logger.output, os.Stderr) +} + +func TestStructuredFile(t *testing.T) { + tmpfile, err := os.CreateTemp("", "") + require.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + cfg := &Config{ + Logfile: tmpfile.Name(), + LogFormat: "structured", + RotationMaxArchives: -1, + } + require.NoError(t, SetupLogging(cfg)) + + log.Printf("I! TEST") + log.Printf("D! TEST") // <- should be ignored + + buf, err := os.ReadFile(tmpfile.Name()) + require.NoError(t, err) + + expected := map[string]interface{}{ + "level": "INFO", + "msg": "TEST", + } + + var actual map[string]interface{} + require.NoError(t, json.Unmarshal(buf, &actual)) + + require.Contains(t, actual, "time") + require.NotEmpty(t, actual["time"]) + delete(actual, "time") + require.Equal(t, expected, actual) +} + +func TestStructuredFileDebug(t *testing.T) { + tmpfile, err := os.CreateTemp("", "") + require.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + cfg := &Config{ + Logfile: tmpfile.Name(), + LogFormat: "structured", + RotationMaxArchives: -1, + Debug: true, + } + require.NoError(t, SetupLogging(cfg)) + + log.Printf("D! TEST") + + buf, err := os.ReadFile(tmpfile.Name()) + require.NoError(t, err) + + expected := map[string]interface{}{ + "level": "DEBUG", + "msg": "TEST", + } + + var actual map[string]interface{} + require.NoError(t, json.Unmarshal(buf, &actual)) + + require.Contains(t, actual, "time") + require.NotEmpty(t, actual["time"]) + delete(actual, "time") + require.Equal(t, expected, actual) +} + +func TestStructuredFileError(t *testing.T) { + tmpfile, err := os.CreateTemp("", "") + require.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + cfg := &Config{ + Logfile: tmpfile.Name(), + LogFormat: "structured", + RotationMaxArchives: -1, + Quiet: true, + } + require.NoError(t, SetupLogging(cfg)) + + log.Printf("E! TEST") + log.Printf("I! TEST") // <- should be ignored + + buf, err := os.ReadFile(tmpfile.Name()) + require.NoError(t, err) + require.Greater(t, len(buf), 19) + + expected := map[string]interface{}{ + "level": "ERROR", + "msg": "TEST", + } + + var actual map[string]interface{} + require.NoError(t, json.Unmarshal(buf, &actual)) + + require.Contains(t, actual, "time") + require.NotEmpty(t, actual["time"]) + delete(actual, "time") + require.Equal(t, expected, actual) +} + +func TestStructuredAddDefaultLogLevel(t *testing.T) { + tmpfile, err := os.CreateTemp("", "") + require.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + cfg := &Config{ + Logfile: tmpfile.Name(), + LogFormat: "structured", + RotationMaxArchives: -1, + Debug: true, + } + require.NoError(t, SetupLogging(cfg)) + + log.Printf("TEST") + + buf, err := os.ReadFile(tmpfile.Name()) + require.NoError(t, err) + + expected := map[string]interface{}{ + "level": "INFO", + "msg": "TEST", + } + + var actual map[string]interface{} + require.NoError(t, json.Unmarshal(buf, &actual)) + + require.Contains(t, actual, "time") + require.NotEmpty(t, actual["time"]) + delete(actual, "time") + require.Equal(t, expected, actual) +} + +func TestStructuredDerivedLogger(t *testing.T) { + instance = defaultHandler() + + tmpfile, err := os.CreateTemp("", "") + require.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + cfg := &Config{ + Logfile: tmpfile.Name(), + LogFormat: "structured", + RotationMaxArchives: -1, + Debug: true, + } + require.NoError(t, SetupLogging(cfg)) + + l := New("testing", "test", "") + l.Info("TEST") + + buf, err := os.ReadFile(tmpfile.Name()) + require.NoError(t, err) + + expected := map[string]interface{}{ + "level": "INFO", + "msg": "TEST", + "category": "testing", + "plugin": "test", + } + + var actual map[string]interface{} + require.NoError(t, json.Unmarshal(buf, &actual)) + + require.Contains(t, actual, "time") + require.NotEmpty(t, actual["time"]) + delete(actual, "time") + require.Equal(t, expected, actual) +} + +func TestStructuredDerivedLoggerWithAttributes(t *testing.T) { + instance = defaultHandler() + + tmpfile, err := os.CreateTemp("", "") + require.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + cfg := &Config{ + Logfile: tmpfile.Name(), + LogFormat: "structured", + RotationMaxArchives: -1, + Debug: true, + } + require.NoError(t, SetupLogging(cfg)) + + l := New("testing", "test", "myalias") + l.AddAttribute("alias", "foo") // Should be ignored + l.AddAttribute("device_id", 123) + + l.Info("TEST") + + buf, err := os.ReadFile(tmpfile.Name()) + require.NoError(t, err) + + expected := map[string]interface{}{ + "level": "INFO", + "msg": "TEST", + "category": "testing", + "plugin": "test", + "alias": "myalias", + "device_id": float64(123), + } + + var actual map[string]interface{} + require.NoError(t, json.Unmarshal(buf, &actual)) + + require.Contains(t, actual, "time") + require.NotEmpty(t, actual["time"]) + delete(actual, "time") + require.Equal(t, expected, actual) +} + +func TestStructuredWriteToTruncatedFile(t *testing.T) { + tmpfile, err := os.CreateTemp("", "") + require.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + cfg := &Config{ + Logfile: tmpfile.Name(), + LogFormat: "structured", + RotationMaxArchives: -1, + Debug: true, + } + require.NoError(t, SetupLogging(cfg)) + + log.Printf("TEST") + + buf, err := os.ReadFile(tmpfile.Name()) + require.NoError(t, err) + + expected := map[string]interface{}{ + "level": "INFO", + "msg": "TEST", + } + + var actual map[string]interface{} + require.NoError(t, json.Unmarshal(buf, &actual)) + + require.Contains(t, actual, "time") + require.NotEmpty(t, actual["time"]) + delete(actual, "time") + require.Equal(t, expected, actual) + + require.NoError(t, os.Truncate(tmpfile.Name(), 0)) + + log.Printf("SHOULD BE FIRST") + + buf, err = os.ReadFile(tmpfile.Name()) + require.NoError(t, err) + + expected = map[string]interface{}{ + "level": "INFO", + "msg": "SHOULD BE FIRST", + } + + require.NoError(t, json.Unmarshal(buf, &actual)) + + require.Contains(t, actual, "time") + require.NotEmpty(t, actual["time"]) + delete(actual, "time") + require.Equal(t, expected, actual) +} + +func TestStructuredWriteToFileInRotation(t *testing.T) { + tempDir := t.TempDir() + cfg := &Config{ + Logfile: filepath.Join(tempDir, "test.log"), + LogFormat: "structured", + RotationMaxArchives: -1, + RotationMaxSize: 30, + } + require.NoError(t, SetupLogging(cfg)) + + // Close the writer here, otherwise the temp folder cannot be deleted because the current log file is in use. + defer CloseLogging() //nolint:errcheck // We cannot do anything if this fails + + log.Printf("I! TEST 1") // Writes 31 bytes, will rotate + log.Printf("I! TEST") // Writes 29 byes, no rotation expected + + files, err := os.ReadDir(tempDir) + require.NoError(t, err) + require.Len(t, files, 2) +} + +func BenchmarkTelegrafStructuredLogWrite(b *testing.B) { + // Discard all logging output + l := &structuredLogger{ + handler: slog.NewJSONHandler(io.Discard, defaultStructuredHandlerOptions), + output: io.Discard, + errlog: log.New(os.Stderr, "", 0), + } + + ts := time.Now() + for i := 0; i < b.N; i++ { + l.Print(telegraf.Debug, ts, "", nil, "test") + } +} diff --git a/logger/default_logger.go b/logger/text_logger.go similarity index 64% rename from logger/default_logger.go rename to logger/text_logger.go index bc6fee3de..6cfba4ac5 100644 --- a/logger/default_logger.go +++ b/logger/text_logger.go @@ -16,11 +16,11 @@ const ( LogTargetStderr = "stderr" ) -type defaultLogger struct { +type textLogger struct { logger *log.Logger } -func (l *defaultLogger) Close() error { +func (l *textLogger) Close() error { writer := l.logger.Writer() // Close the writer if possible and avoid closing stderr @@ -34,18 +34,14 @@ func (l *defaultLogger) Close() error { return errors.New("the underlying writer cannot be closed") } -func (l *defaultLogger) SetOutput(w io.Writer) { - l.logger.SetOutput(w) -} - -func (l *defaultLogger) Print(level telegraf.LogLevel, ts time.Time, prefix string, args ...interface{}) { +func (l *textLogger) Print(level telegraf.LogLevel, ts time.Time, prefix string, _ map[string]interface{}, args ...interface{}) { msg := append([]interface{}{ts.Format(time.RFC3339), " ", level.Indicator(), " ", prefix}, args...) l.logger.Print(msg...) } -func createDefaultLogger(cfg *Config) (sink, error) { +func createTextLogger(cfg *Config) (sink, error) { var writer io.Writer = os.Stderr - if cfg.LogTarget == "file" && cfg.Logfile != "" { + if cfg.Logfile != "" { w, err := rotate.NewFileWriter( cfg.Logfile, cfg.RotationInterval, @@ -58,10 +54,9 @@ func createDefaultLogger(cfg *Config) (sink, error) { writer = w } - return &defaultLogger{logger: log.New(writer, "", 0)}, nil + return &textLogger{logger: log.New(writer, "", 0)}, nil } func init() { - add("stderr", createDefaultLogger) - add("file", createDefaultLogger) + add("text", createTextLogger) } diff --git a/logger/default_logger_test.go b/logger/text_logger_test.go similarity index 57% rename from logger/default_logger_test.go rename to logger/text_logger_test.go index a1bf9fc8f..3a3535649 100644 --- a/logger/default_logger_test.go +++ b/logger/text_logger_test.go @@ -12,37 +12,26 @@ import ( "github.com/stretchr/testify/require" ) -func TestLogTargetDefault(t *testing.T) { +func TestTextStderr(t *testing.T) { instance = defaultHandler() cfg := &Config{ - Quiet: true, - } - require.NoError(t, SetupLogging(cfg)) - logger, ok := instance.impl.(*defaultLogger) - require.True(t, ok, "logging instance is not a default-logger") - require.Equal(t, logger.logger.Writer(), os.Stderr) -} - -func TestLogTargetStderr(t *testing.T) { - instance = defaultHandler() - cfg := &Config{ - LogTarget: "stderr", + LogFormat: "text", Quiet: true, } require.NoError(t, SetupLogging(cfg)) - logger, ok := instance.impl.(*defaultLogger) - require.True(t, ok, "logging instance is not a default-logger") + logger, ok := instance.impl.(*textLogger) + require.Truef(t, ok, "logging instance is not a text-logger but %T", instance.impl) require.Equal(t, logger.logger.Writer(), os.Stderr) } -func TestLogTargetFile(t *testing.T) { +func TestTextFile(t *testing.T) { tmpfile, err := os.CreateTemp("", "") require.NoError(t, err) defer os.Remove(tmpfile.Name()) cfg := &Config{ Logfile: tmpfile.Name(), - LogTarget: "file", + LogFormat: "text", RotationMaxArchives: -1, } require.NoError(t, SetupLogging(cfg)) @@ -53,17 +42,17 @@ func TestLogTargetFile(t *testing.T) { buf, err := os.ReadFile(tmpfile.Name()) require.NoError(t, err) require.Greater(t, len(buf), 19) - require.Equal(t, buf[19:], []byte("Z I! TEST\n")) + require.Equal(t, "Z I! TEST\n", string(buf[19:])) } -func TestLogTargetFileDebug(t *testing.T) { +func TestTextFileDebug(t *testing.T) { tmpfile, err := os.CreateTemp("", "") require.NoError(t, err) defer os.Remove(tmpfile.Name()) cfg := &Config{ Logfile: tmpfile.Name(), - LogTarget: "file", + LogFormat: "text", RotationMaxArchives: -1, Debug: true, } @@ -74,17 +63,17 @@ func TestLogTargetFileDebug(t *testing.T) { buf, err := os.ReadFile(tmpfile.Name()) require.NoError(t, err) require.Greater(t, len(buf), 19) - require.Equal(t, buf[19:], []byte("Z D! TEST\n")) + require.Equal(t, "Z D! TEST\n", string(buf[19:])) } -func TestLogTargetFileError(t *testing.T) { +func TestTextFileError(t *testing.T) { tmpfile, err := os.CreateTemp("", "") require.NoError(t, err) defer os.Remove(tmpfile.Name()) cfg := &Config{ Logfile: tmpfile.Name(), - LogTarget: "file", + LogFormat: "text", RotationMaxArchives: -1, Quiet: true, } @@ -96,17 +85,17 @@ func TestLogTargetFileError(t *testing.T) { buf, err := os.ReadFile(tmpfile.Name()) require.NoError(t, err) require.Greater(t, len(buf), 19) - require.Equal(t, buf[19:], []byte("Z E! TEST\n")) + require.Equal(t, "Z E! TEST\n", string(buf[19:])) } -func TestAddDefaultLogLevel(t *testing.T) { +func TestTextAddDefaultLogLevel(t *testing.T) { tmpfile, err := os.CreateTemp("", "") require.NoError(t, err) defer os.Remove(tmpfile.Name()) cfg := &Config{ Logfile: tmpfile.Name(), - LogTarget: "file", + LogFormat: "text", RotationMaxArchives: -1, Debug: true, } @@ -117,17 +106,17 @@ func TestAddDefaultLogLevel(t *testing.T) { buf, err := os.ReadFile(tmpfile.Name()) require.NoError(t, err) require.Greater(t, len(buf), 19) - require.Equal(t, buf[19:], []byte("Z I! TEST\n")) + require.Equal(t, "Z I! TEST\n", string(buf[19:])) } -func TestWriteToTruncatedFile(t *testing.T) { +func TestTextWriteToTruncatedFile(t *testing.T) { tmpfile, err := os.CreateTemp("", "") require.NoError(t, err) defer os.Remove(tmpfile.Name()) cfg := &Config{ Logfile: tmpfile.Name(), - LogTarget: "file", + LogFormat: "text", RotationMaxArchives: -1, Debug: true, } @@ -138,24 +127,22 @@ func TestWriteToTruncatedFile(t *testing.T) { buf, err := os.ReadFile(tmpfile.Name()) require.NoError(t, err) require.Greater(t, len(buf), 19) - require.Equal(t, buf[19:], []byte("Z I! TEST\n")) + require.Equal(t, "Z I! TEST\n", string(buf[19:])) - tmpf, err := os.OpenFile(tmpfile.Name(), os.O_RDWR|os.O_TRUNC, 0640) - require.NoError(t, err) - require.NoError(t, tmpf.Close()) + require.NoError(t, os.Truncate(tmpfile.Name(), 0)) log.Printf("SHOULD BE FIRST") buf, err = os.ReadFile(tmpfile.Name()) require.NoError(t, err) - require.Equal(t, buf[19:], []byte("Z I! SHOULD BE FIRST\n")) + require.Equal(t, "Z I! SHOULD BE FIRST\n", string(buf[19:])) } -func TestWriteToFileInRotation(t *testing.T) { +func TestTextWriteToFileInRotation(t *testing.T) { tempDir := t.TempDir() cfg := &Config{ Logfile: filepath.Join(tempDir, "test.log"), - LogTarget: "file", + LogFormat: "text", RotationMaxArchives: -1, RotationMaxSize: 30, } @@ -172,16 +159,69 @@ func TestWriteToFileInRotation(t *testing.T) { require.Len(t, files, 2) } -func BenchmarkTelegrafLogWrite(b *testing.B) { - l, err := createDefaultLogger(&Config{}) +func TestTextWriteDerivedLogger(t *testing.T) { + instance = defaultHandler() + + tmpfile, err := os.CreateTemp("", "") + require.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + cfg := &Config{ + Logfile: tmpfile.Name(), + LogFormat: "text", + RotationMaxArchives: -1, + Debug: true, + } + require.NoError(t, SetupLogging(cfg)) + + l := New("testing", "test", "") + l.Info("TEST") + + buf, err := os.ReadFile(tmpfile.Name()) + require.NoError(t, err) + require.Greater(t, len(buf), 19) + require.Equal(t, "Z I! [testing.test] TEST\n", string(buf[19:])) +} + +func TestTextWriteDerivedLoggerWithAttributes(t *testing.T) { + instance = defaultHandler() + + tmpfile, err := os.CreateTemp("", "") + require.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + cfg := &Config{ + Logfile: tmpfile.Name(), + LogFormat: "text", + RotationMaxArchives: -1, + Debug: true, + } + require.NoError(t, SetupLogging(cfg)) + + l := New("testing", "test", "myalias") + + // All attributes should be ignored + l.AddAttribute("alias", "foo") + l.AddAttribute("device_id", 123) + + l.Info("TEST") + + buf, err := os.ReadFile(tmpfile.Name()) + require.NoError(t, err) + require.Greater(t, len(buf), 19) + require.Equal(t, "Z I! [testing.test::myalias] TEST\n", string(buf[19:])) +} + +func BenchmarkTelegrafTextLogWrite(b *testing.B) { + l, err := createTextLogger(&Config{}) require.NoError(b, err) // Discard all logging output - dl := l.(*defaultLogger) - dl.SetOutput(io.Discard) + dl := l.(*textLogger) + dl.logger.SetOutput(io.Discard) ts := time.Now() for i := 0; i < b.N; i++ { - dl.Print(telegraf.Debug, ts, "", "test") + dl.Print(telegraf.Debug, ts, "", nil, "test") } } diff --git a/migrations/all/global_agent.go b/migrations/all/global_agent.go new file mode 100644 index 000000000..368f164cc --- /dev/null +++ b/migrations/all/global_agent.go @@ -0,0 +1,5 @@ +//go:build !custom || migrations + +package all + +import _ "github.com/influxdata/telegraf/migrations/global_agent" // register migration diff --git a/migrations/global_agent/migration.go b/migrations/global_agent/migration.go new file mode 100644 index 000000000..c93a7fc29 --- /dev/null +++ b/migrations/global_agent/migration.go @@ -0,0 +1,75 @@ +package global_agent + +import ( + "errors" + + "github.com/influxdata/toml" + "github.com/influxdata/toml/ast" + + "github.com/influxdata/telegraf/migrations" +) + +// Migration function +func migrate(name string, tbl *ast.Table) ([]byte, string, error) { + // Migrate the agent section only... + if name != "agent" { + return nil, "", migrations.ErrNotApplicable + } + + // Decode the old data structure + var agent map[string]interface{} + if err := toml.UnmarshalTable(tbl, &agent); err != nil { + return nil, "", err + } + + // Check for deprecated option(s) and migrate them + var applied bool + + // Migrate log settings + var logtarget string + var logtargetFound bool + if raw, found := agent["logtarget"]; found { + if v, ok := raw.(string); ok { + logtarget = v + logtargetFound = true + } + } + + var logformat string + var logformatFound bool + if raw, found := agent["logformat"]; found { + if v, ok := raw.(string); ok { + logformat = v + logformatFound = true + } + } + + if logtargetFound { + switch logtarget { + case "stderr": + delete(agent, "logfile") + case "file": + case "eventlog": + if logformatFound && logformat != "eventlog" { + return nil, "", errors.New("contradicting setting for 'logtarget' and 'logformat'") + } + agent["logformat"] = "eventlog" + delete(agent, "logfile") + } + applied = true + delete(agent, "logtarget") + } + + // No options migrated so we can exit early + if !applied { + return nil, "", migrations.ErrNotApplicable + } + + output, err := toml.Marshal(map[string]map[string]interface{}{"agent": agent}) + return output, "", err +} + +// Register the migration function for the plugin type +func init() { + migrations.AddGlobalMigration(migrate) +} diff --git a/migrations/global_agent/migration_test.go b/migrations/global_agent/migration_test.go new file mode 100644 index 000000000..1533f6fe1 --- /dev/null +++ b/migrations/global_agent/migration_test.go @@ -0,0 +1,104 @@ +package global_agent_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/influxdata/telegraf/config" + _ "github.com/influxdata/telegraf/migrations/global_agent" // register migration +) + +func TestNoMigration(t *testing.T) { + fn := filepath.Join("testcases", "default.conf") + + // Read the input data + input, remote, err := config.LoadConfigFile(fn) + require.NoError(t, err) + require.False(t, remote) + require.NotEmpty(t, input) + + // Expect the output to be equal to the input + expectedBuffer, err := os.ReadFile(fn) + require.NoError(t, err) + expected := config.NewConfig() + require.NoError(t, expected.LoadConfigData(expectedBuffer)) + require.NotNil(t, expected.Agent) + + // Migrate + output, n, err := config.ApplyMigrations(input) + require.NoError(t, err) + require.NotEmpty(t, output) + require.Zero(t, n) + + actual := config.NewConfig() + require.NoError(t, actual.LoadConfigData(output)) + require.NotNil(t, actual.Agent) + + // Test the output + require.EqualValues(t, expected.Agent, actual.Agent, string(output)) + require.Equal(t, string(expectedBuffer), string(output)) +} + +func TestLogTargetEventlogCollision(t *testing.T) { + fn := filepath.Join("testcases", "logtarget_eventlog_collision.conf") + + // Read the input data + input, remote, err := config.LoadConfigFile(fn) + require.NoError(t, err) + require.False(t, remote) + require.NotEmpty(t, input) + + // Migrate + _, n, err := config.ApplyMigrations(input) + require.ErrorContains(t, err, "contradicting setting for 'logtarget' and 'logformat'") + require.Zero(t, n) +} + +func TestCases(t *testing.T) { + // Get all directories in testdata + folders, err := os.ReadDir("testcases") + require.NoError(t, err) + + for _, f := range folders { + // Only handle folders + if !f.IsDir() { + continue + } + + t.Run(f.Name(), func(t *testing.T) { + testcasePath := filepath.Join("testcases", f.Name()) + inputFile := filepath.Join(testcasePath, "telegraf.conf") + expectedFile := filepath.Join(testcasePath, "expected.conf") + + // Read the expected output + expected := config.NewConfig() + require.NoError(t, expected.LoadConfig(expectedFile)) + require.NotNil(t, expected.Agent) + + // Read the input data + input, remote, err := config.LoadConfigFile(inputFile) + require.NoError(t, err) + require.False(t, remote) + require.NotEmpty(t, input) + + // Migrate + output, n, err := config.ApplyMigrations(input) + require.NoError(t, err) + require.NotEmpty(t, output) + require.Positive(t, n, "expected migration application but none applied") + actual := config.NewConfig() + require.NoError(t, actual.LoadConfigData(output)) + require.NotNil(t, actual.Agent) + + // Test the output + require.EqualValues(t, expected.Agent, actual.Agent, string(output)) + + expectedBuffer, err := os.ReadFile(expectedFile) + require.NoError(t, err) + require.Equal(t, string(expectedBuffer), string(output)) + }) + } +} diff --git a/migrations/global_agent/testcases/default.conf b/migrations/global_agent/testcases/default.conf new file mode 100644 index 000000000..12fd81ac4 --- /dev/null +++ b/migrations/global_agent/testcases/default.conf @@ -0,0 +1,100 @@ +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Collection offset is used to shift the collection by the given amount. + ## This can be be used to avoid many plugins querying constraint devices + ## at the same time by manually scheduling them in time. + # collection_offset = "0s" + + ## Default flushing interval for all outputs. Maximum flush_interval will be + ## flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## Collected metrics are rounded to the precision specified. Precision is + ## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s). + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + ## + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s: + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + precision = "0s" + + ## Log at debug level. + # debug = false + ## Log only error level messages. + # quiet = false + + ## Log format controls the way messages are logged and can be one of "text", + ## "structured" or, on Windows, "eventlog". + # logformat = "text" + + ## Name of the file to be logged to or stderr if unset or empty. This + ## setting is ignored for the "eventlog" format. + # logfile = "" + + ## The logfile will be rotated after the time interval specified. When set + ## to 0 no time based rotation is performed. Logs are rotated only when + ## written to, if there is no log activity rotation may be delayed. + # logfile_rotation_interval = "0h" + + ## The logfile will be rotated when it becomes larger than the specified + ## size. When set to 0 no size based rotation is performed. + # logfile_rotation_max_size = "0MB" + + ## Maximum number of rotated archives to keep, any older logs are deleted. + ## If set to -1, no archives are removed. + # logfile_rotation_max_archives = 5 + + ## Pick a timezone to use when logging or type 'local' for local time. + ## Example: America/Chicago + # log_with_timezone = "" + + ## Override default hostname, if empty use os.Hostname() + # hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + # omit_hostname = false + + ## Method of translating SNMP objects. Can be "netsnmp" (deprecated) which + ## translates by calling external programs snmptranslate and snmptable, + ## or "gosmi" which translates using the built-in gosmi library. + # snmp_translator = "netsnmp" + + ## Name of the file to load the state of plugins from and store the state to. + ## If uncommented and not empty, this file will be used to save the state of + ## stateful plugins on termination of Telegraf. If the file exists on start, + ## the state in the file will be restored for the plugins. + # statefile = "" + + ## Flag to skip running processors after aggregators + ## By default, processors are run a second time after aggregators. Changing + ## this setting to true will skip the second run of processors. + # skip_processors_after_aggregators = false diff --git a/migrations/global_agent/testcases/logtarget_eventlog/expected.conf b/migrations/global_agent/testcases/logtarget_eventlog/expected.conf new file mode 100644 index 000000000..6e0514bbb --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_eventlog/expected.conf @@ -0,0 +1,10 @@ +[agent] +collection_jitter = "0s" +flush_interval = "10s" +flush_jitter = "0s" +interval = "10s" +logformat = "eventlog" +metric_batch_size = 1000 +metric_buffer_limit = 10000 +precision = "0s" +round_interval = true diff --git a/migrations/global_agent/testcases/logtarget_eventlog/telegraf.conf b/migrations/global_agent/testcases/logtarget_eventlog/telegraf.conf new file mode 100644 index 000000000..666153d6b --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_eventlog/telegraf.conf @@ -0,0 +1,102 @@ +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Collection offset is used to shift the collection by the given amount. + ## This can be be used to avoid many plugins querying constraint devices + ## at the same time by manually scheduling them in time. + # collection_offset = "0s" + + ## Default flushing interval for all outputs. Maximum flush_interval will be + ## flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## Collected metrics are rounded to the precision specified. Precision is + ## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s). + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + ## + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s: + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + precision = "0s" + + ## Log at debug level. + # debug = false + ## Log only error level messages. + # quiet = false + + ## Log format controls the way messages are logged and can be one of "text", + ## "structured" or, on Windows, "eventlog". + # logformat = "text" + + logtarget = "eventlog" + + ## Name of the file to be logged to or stderr if unset or empty. This + ## setting is ignored for the "eventlog" format. + # logfile = "lala.log" + + ## The logfile will be rotated after the time interval specified. When set + ## to 0 no time based rotation is performed. Logs are rotated only when + ## written to, if there is no log activity rotation may be delayed. + # logfile_rotation_interval = "0h" + + ## The logfile will be rotated when it becomes larger than the specified + ## size. When set to 0 no size based rotation is performed. + # logfile_rotation_max_size = "0MB" + + ## Maximum number of rotated archives to keep, any older logs are deleted. + ## If set to -1, no archives are removed. + # logfile_rotation_max_archives = 5 + + ## Pick a timezone to use when logging or type 'local' for local time. + ## Example: America/Chicago + # log_with_timezone = "" + + ## Override default hostname, if empty use os.Hostname() + # hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + # omit_hostname = false + + ## Method of translating SNMP objects. Can be "netsnmp" (deprecated) which + ## translates by calling external programs snmptranslate and snmptable, + ## or "gosmi" which translates using the built-in gosmi library. + # snmp_translator = "netsnmp" + + ## Name of the file to load the state of plugins from and store the state to. + ## If uncommented and not empty, this file will be used to save the state of + ## stateful plugins on termination of Telegraf. If the file exists on start, + ## the state in the file will be restored for the plugins. + # statefile = "" + + ## Flag to skip running processors after aggregators + ## By default, processors are run a second time after aggregators. Changing + ## this setting to true will skip the second run of processors. + # skip_processors_after_aggregators = false diff --git a/migrations/global_agent/testcases/logtarget_eventlog_collision.conf b/migrations/global_agent/testcases/logtarget_eventlog_collision.conf new file mode 100644 index 000000000..5be34e3c7 --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_eventlog_collision.conf @@ -0,0 +1,102 @@ +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Collection offset is used to shift the collection by the given amount. + ## This can be be used to avoid many plugins querying constraint devices + ## at the same time by manually scheduling them in time. + # collection_offset = "0s" + + ## Default flushing interval for all outputs. Maximum flush_interval will be + ## flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## Collected metrics are rounded to the precision specified. Precision is + ## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s). + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + ## + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s: + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + precision = "0s" + + ## Log at debug level. + # debug = false + ## Log only error level messages. + # quiet = false + + ## Log format controls the way messages are logged and can be one of "text", + ## "structured" or, on Windows, "eventlog". + logformat = "text" + + logtarget = "eventlog" + + ## Name of the file to be logged to or stderr if unset or empty. This + ## setting is ignored for the "eventlog" format. + # logfile = "" + + ## The logfile will be rotated after the time interval specified. When set + ## to 0 no time based rotation is performed. Logs are rotated only when + ## written to, if there is no log activity rotation may be delayed. + # logfile_rotation_interval = "0h" + + ## The logfile will be rotated when it becomes larger than the specified + ## size. When set to 0 no size based rotation is performed. + # logfile_rotation_max_size = "0MB" + + ## Maximum number of rotated archives to keep, any older logs are deleted. + ## If set to -1, no archives are removed. + # logfile_rotation_max_archives = 5 + + ## Pick a timezone to use when logging or type 'local' for local time. + ## Example: America/Chicago + # log_with_timezone = "" + + ## Override default hostname, if empty use os.Hostname() + # hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + # omit_hostname = false + + ## Method of translating SNMP objects. Can be "netsnmp" (deprecated) which + ## translates by calling external programs snmptranslate and snmptable, + ## or "gosmi" which translates using the built-in gosmi library. + # snmp_translator = "netsnmp" + + ## Name of the file to load the state of plugins from and store the state to. + ## If uncommented and not empty, this file will be used to save the state of + ## stateful plugins on termination of Telegraf. If the file exists on start, + ## the state in the file will be restored for the plugins. + # statefile = "" + + ## Flag to skip running processors after aggregators + ## By default, processors are run a second time after aggregators. Changing + ## this setting to true will skip the second run of processors. + # skip_processors_after_aggregators = false diff --git a/migrations/global_agent/testcases/logtarget_eventlog_with_logfile/expected.conf b/migrations/global_agent/testcases/logtarget_eventlog_with_logfile/expected.conf new file mode 100644 index 000000000..6e0514bbb --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_eventlog_with_logfile/expected.conf @@ -0,0 +1,10 @@ +[agent] +collection_jitter = "0s" +flush_interval = "10s" +flush_jitter = "0s" +interval = "10s" +logformat = "eventlog" +metric_batch_size = 1000 +metric_buffer_limit = 10000 +precision = "0s" +round_interval = true diff --git a/migrations/global_agent/testcases/logtarget_eventlog_with_logfile/telegraf.conf b/migrations/global_agent/testcases/logtarget_eventlog_with_logfile/telegraf.conf new file mode 100644 index 000000000..8695a700f --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_eventlog_with_logfile/telegraf.conf @@ -0,0 +1,102 @@ +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Collection offset is used to shift the collection by the given amount. + ## This can be be used to avoid many plugins querying constraint devices + ## at the same time by manually scheduling them in time. + # collection_offset = "0s" + + ## Default flushing interval for all outputs. Maximum flush_interval will be + ## flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## Collected metrics are rounded to the precision specified. Precision is + ## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s). + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + ## + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s: + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + precision = "0s" + + ## Log at debug level. + # debug = false + ## Log only error level messages. + # quiet = false + + ## Log format controls the way messages are logged and can be one of "text", + ## "structured" or, on Windows, "eventlog". + # logformat = "text" + + logtarget = "eventlog" + + ## Name of the file to be logged to or stderr if unset or empty. This + ## setting is ignored for the "eventlog" format. + logfile = "lala.log" + + ## The logfile will be rotated after the time interval specified. When set + ## to 0 no time based rotation is performed. Logs are rotated only when + ## written to, if there is no log activity rotation may be delayed. + # logfile_rotation_interval = "0h" + + ## The logfile will be rotated when it becomes larger than the specified + ## size. When set to 0 no size based rotation is performed. + # logfile_rotation_max_size = "0MB" + + ## Maximum number of rotated archives to keep, any older logs are deleted. + ## If set to -1, no archives are removed. + # logfile_rotation_max_archives = 5 + + ## Pick a timezone to use when logging or type 'local' for local time. + ## Example: America/Chicago + # log_with_timezone = "" + + ## Override default hostname, if empty use os.Hostname() + # hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + # omit_hostname = false + + ## Method of translating SNMP objects. Can be "netsnmp" (deprecated) which + ## translates by calling external programs snmptranslate and snmptable, + ## or "gosmi" which translates using the built-in gosmi library. + # snmp_translator = "netsnmp" + + ## Name of the file to load the state of plugins from and store the state to. + ## If uncommented and not empty, this file will be used to save the state of + ## stateful plugins on termination of Telegraf. If the file exists on start, + ## the state in the file will be restored for the plugins. + # statefile = "" + + ## Flag to skip running processors after aggregators + ## By default, processors are run a second time after aggregators. Changing + ## this setting to true will skip the second run of processors. + # skip_processors_after_aggregators = false diff --git a/migrations/global_agent/testcases/logtarget_file/expected.conf b/migrations/global_agent/testcases/logtarget_file/expected.conf new file mode 100644 index 000000000..176dab47b --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_file/expected.conf @@ -0,0 +1,10 @@ +[agent] +collection_jitter = "0s" +flush_interval = "10s" +flush_jitter = "0s" +interval = "10s" +logfile = "lala.log" +metric_batch_size = 1000 +metric_buffer_limit = 10000 +precision = "0s" +round_interval = true diff --git a/migrations/global_agent/testcases/logtarget_file/telegraf.conf b/migrations/global_agent/testcases/logtarget_file/telegraf.conf new file mode 100644 index 000000000..b4483921e --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_file/telegraf.conf @@ -0,0 +1,102 @@ +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Collection offset is used to shift the collection by the given amount. + ## This can be be used to avoid many plugins querying constraint devices + ## at the same time by manually scheduling them in time. + # collection_offset = "0s" + + ## Default flushing interval for all outputs. Maximum flush_interval will be + ## flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## Collected metrics are rounded to the precision specified. Precision is + ## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s). + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + ## + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s: + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + precision = "0s" + + ## Log at debug level. + # debug = false + ## Log only error level messages. + # quiet = false + + ## Log format controls the way messages are logged and can be one of "text", + ## "structured" or, on Windows, "eventlog". + # logformat = "text" + + logtarget = "file" + + ## Name of the file to be logged to or stderr if unset or empty. This + ## setting is ignored for the "eventlog" format. + logfile = "lala.log" + + ## The logfile will be rotated after the time interval specified. When set + ## to 0 no time based rotation is performed. Logs are rotated only when + ## written to, if there is no log activity rotation may be delayed. + # logfile_rotation_interval = "0h" + + ## The logfile will be rotated when it becomes larger than the specified + ## size. When set to 0 no size based rotation is performed. + # logfile_rotation_max_size = "0MB" + + ## Maximum number of rotated archives to keep, any older logs are deleted. + ## If set to -1, no archives are removed. + # logfile_rotation_max_archives = 5 + + ## Pick a timezone to use when logging or type 'local' for local time. + ## Example: America/Chicago + # log_with_timezone = "" + + ## Override default hostname, if empty use os.Hostname() + # hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + # omit_hostname = false + + ## Method of translating SNMP objects. Can be "netsnmp" (deprecated) which + ## translates by calling external programs snmptranslate and snmptable, + ## or "gosmi" which translates using the built-in gosmi library. + # snmp_translator = "netsnmp" + + ## Name of the file to load the state of plugins from and store the state to. + ## If uncommented and not empty, this file will be used to save the state of + ## stateful plugins on termination of Telegraf. If the file exists on start, + ## the state in the file will be restored for the plugins. + # statefile = "" + + ## Flag to skip running processors after aggregators + ## By default, processors are run a second time after aggregators. Changing + ## this setting to true will skip the second run of processors. + # skip_processors_after_aggregators = false diff --git a/migrations/global_agent/testcases/logtarget_file_no_logfile/expected.conf b/migrations/global_agent/testcases/logtarget_file_no_logfile/expected.conf new file mode 100644 index 000000000..07c2131ab --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_file_no_logfile/expected.conf @@ -0,0 +1,9 @@ +[agent] +collection_jitter = "0s" +flush_interval = "10s" +flush_jitter = "0s" +interval = "10s" +metric_batch_size = 1000 +metric_buffer_limit = 10000 +precision = "0s" +round_interval = true diff --git a/migrations/global_agent/testcases/logtarget_file_no_logfile/telegraf.conf b/migrations/global_agent/testcases/logtarget_file_no_logfile/telegraf.conf new file mode 100644 index 000000000..5293c8b2b --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_file_no_logfile/telegraf.conf @@ -0,0 +1,102 @@ +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Collection offset is used to shift the collection by the given amount. + ## This can be be used to avoid many plugins querying constraint devices + ## at the same time by manually scheduling them in time. + # collection_offset = "0s" + + ## Default flushing interval for all outputs. Maximum flush_interval will be + ## flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## Collected metrics are rounded to the precision specified. Precision is + ## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s). + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + ## + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s: + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + precision = "0s" + + ## Log at debug level. + # debug = false + ## Log only error level messages. + # quiet = false + + ## Log format controls the way messages are logged and can be one of "text", + ## "structured" or, on Windows, "eventlog". + # logformat = "text" + + logtarget = "file" + + ## Name of the file to be logged to or stderr if unset or empty. This + ## setting is ignored for the "eventlog" format. + # logfile = "lala.log" + + ## The logfile will be rotated after the time interval specified. When set + ## to 0 no time based rotation is performed. Logs are rotated only when + ## written to, if there is no log activity rotation may be delayed. + # logfile_rotation_interval = "0h" + + ## The logfile will be rotated when it becomes larger than the specified + ## size. When set to 0 no size based rotation is performed. + # logfile_rotation_max_size = "0MB" + + ## Maximum number of rotated archives to keep, any older logs are deleted. + ## If set to -1, no archives are removed. + # logfile_rotation_max_archives = 5 + + ## Pick a timezone to use when logging or type 'local' for local time. + ## Example: America/Chicago + # log_with_timezone = "" + + ## Override default hostname, if empty use os.Hostname() + # hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + # omit_hostname = false + + ## Method of translating SNMP objects. Can be "netsnmp" (deprecated) which + ## translates by calling external programs snmptranslate and snmptable, + ## or "gosmi" which translates using the built-in gosmi library. + # snmp_translator = "netsnmp" + + ## Name of the file to load the state of plugins from and store the state to. + ## If uncommented and not empty, this file will be used to save the state of + ## stateful plugins on termination of Telegraf. If the file exists on start, + ## the state in the file will be restored for the plugins. + # statefile = "" + + ## Flag to skip running processors after aggregators + ## By default, processors are run a second time after aggregators. Changing + ## this setting to true will skip the second run of processors. + # skip_processors_after_aggregators = false diff --git a/migrations/global_agent/testcases/logtarget_stderr/expected.conf b/migrations/global_agent/testcases/logtarget_stderr/expected.conf new file mode 100644 index 000000000..07c2131ab --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_stderr/expected.conf @@ -0,0 +1,9 @@ +[agent] +collection_jitter = "0s" +flush_interval = "10s" +flush_jitter = "0s" +interval = "10s" +metric_batch_size = 1000 +metric_buffer_limit = 10000 +precision = "0s" +round_interval = true diff --git a/migrations/global_agent/testcases/logtarget_stderr/telegraf.conf b/migrations/global_agent/testcases/logtarget_stderr/telegraf.conf new file mode 100644 index 000000000..c7a56a81e --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_stderr/telegraf.conf @@ -0,0 +1,102 @@ +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Collection offset is used to shift the collection by the given amount. + ## This can be be used to avoid many plugins querying constraint devices + ## at the same time by manually scheduling them in time. + # collection_offset = "0s" + + ## Default flushing interval for all outputs. Maximum flush_interval will be + ## flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## Collected metrics are rounded to the precision specified. Precision is + ## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s). + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + ## + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s: + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + precision = "0s" + + ## Log at debug level. + # debug = false + ## Log only error level messages. + # quiet = false + + ## Log format controls the way messages are logged and can be one of "text", + ## "structured" or, on Windows, "eventlog". + # logformat = "text" + + logtarget = "stderr" + + ## Name of the file to be logged to or stderr if unset or empty. This + ## setting is ignored for the "eventlog" format. + # logfile = "lala.log" + + ## The logfile will be rotated after the time interval specified. When set + ## to 0 no time based rotation is performed. Logs are rotated only when + ## written to, if there is no log activity rotation may be delayed. + # logfile_rotation_interval = "0h" + + ## The logfile will be rotated when it becomes larger than the specified + ## size. When set to 0 no size based rotation is performed. + # logfile_rotation_max_size = "0MB" + + ## Maximum number of rotated archives to keep, any older logs are deleted. + ## If set to -1, no archives are removed. + # logfile_rotation_max_archives = 5 + + ## Pick a timezone to use when logging or type 'local' for local time. + ## Example: America/Chicago + # log_with_timezone = "" + + ## Override default hostname, if empty use os.Hostname() + # hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + # omit_hostname = false + + ## Method of translating SNMP objects. Can be "netsnmp" (deprecated) which + ## translates by calling external programs snmptranslate and snmptable, + ## or "gosmi" which translates using the built-in gosmi library. + # snmp_translator = "netsnmp" + + ## Name of the file to load the state of plugins from and store the state to. + ## If uncommented and not empty, this file will be used to save the state of + ## stateful plugins on termination of Telegraf. If the file exists on start, + ## the state in the file will be restored for the plugins. + # statefile = "" + + ## Flag to skip running processors after aggregators + ## By default, processors are run a second time after aggregators. Changing + ## this setting to true will skip the second run of processors. + # skip_processors_after_aggregators = false diff --git a/migrations/global_agent/testcases/logtarget_stderr_with_logfile/expected.conf b/migrations/global_agent/testcases/logtarget_stderr_with_logfile/expected.conf new file mode 100644 index 000000000..07c2131ab --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_stderr_with_logfile/expected.conf @@ -0,0 +1,9 @@ +[agent] +collection_jitter = "0s" +flush_interval = "10s" +flush_jitter = "0s" +interval = "10s" +metric_batch_size = 1000 +metric_buffer_limit = 10000 +precision = "0s" +round_interval = true diff --git a/migrations/global_agent/testcases/logtarget_stderr_with_logfile/telegraf.conf b/migrations/global_agent/testcases/logtarget_stderr_with_logfile/telegraf.conf new file mode 100644 index 000000000..5059141ea --- /dev/null +++ b/migrations/global_agent/testcases/logtarget_stderr_with_logfile/telegraf.conf @@ -0,0 +1,102 @@ +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Collection offset is used to shift the collection by the given amount. + ## This can be be used to avoid many plugins querying constraint devices + ## at the same time by manually scheduling them in time. + # collection_offset = "0s" + + ## Default flushing interval for all outputs. Maximum flush_interval will be + ## flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## Collected metrics are rounded to the precision specified. Precision is + ## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s). + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + ## + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s: + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + precision = "0s" + + ## Log at debug level. + # debug = false + ## Log only error level messages. + # quiet = false + + ## Log format controls the way messages are logged and can be one of "text", + ## "structured" or, on Windows, "eventlog". + # logformat = "text" + + logtarget = "stderr" + + ## Name of the file to be logged to or stderr if unset or empty. This + ## setting is ignored for the "eventlog" format. + logfile = "lala.log" + + ## The logfile will be rotated after the time interval specified. When set + ## to 0 no time based rotation is performed. Logs are rotated only when + ## written to, if there is no log activity rotation may be delayed. + # logfile_rotation_interval = "0h" + + ## The logfile will be rotated when it becomes larger than the specified + ## size. When set to 0 no size based rotation is performed. + # logfile_rotation_max_size = "0MB" + + ## Maximum number of rotated archives to keep, any older logs are deleted. + ## If set to -1, no archives are removed. + # logfile_rotation_max_archives = 5 + + ## Pick a timezone to use when logging or type 'local' for local time. + ## Example: America/Chicago + # log_with_timezone = "" + + ## Override default hostname, if empty use os.Hostname() + # hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + # omit_hostname = false + + ## Method of translating SNMP objects. Can be "netsnmp" (deprecated) which + ## translates by calling external programs snmptranslate and snmptable, + ## or "gosmi" which translates using the built-in gosmi library. + # snmp_translator = "netsnmp" + + ## Name of the file to load the state of plugins from and store the state to. + ## If uncommented and not empty, this file will be used to save the state of + ## stateful plugins on termination of Telegraf. If the file exists on start, + ## the state in the file will be restored for the plugins. + # statefile = "" + + ## Flag to skip running processors after aggregators + ## By default, processors are run a second time after aggregators. Changing + ## this setting to true will skip the second run of processors. + # skip_processors_after_aggregators = false diff --git a/migrations/registry.go b/migrations/registry.go index db88dc16b..d492ed13e 100644 --- a/migrations/registry.go +++ b/migrations/registry.go @@ -39,17 +39,10 @@ func AddGeneralMigration(f GeneralMigrationFunc) { GeneralMigrations = append(GeneralMigrations, f) } -type pluginTOMLStruct map[string]map[string][]interface{} +type GlobalMigrationFunc func(string, *ast.Table) ([]byte, string, error) -func CreateTOMLStruct(category, name string) pluginTOMLStruct { - return map[string]map[string][]interface{}{ - category: { - name: make([]interface{}, 0), - }, - } -} +var GlobalMigrations []GlobalMigrationFunc -func (p *pluginTOMLStruct) Add(category, name string, plugin interface{}) { - cfg := map[string]map[string][]interface{}(*p) - cfg[category][name] = append(cfg[category][name], plugin) +func AddGlobalMigration(f GlobalMigrationFunc) { + GlobalMigrations = append(GlobalMigrations, f) } diff --git a/migrations/utils.go b/migrations/utils.go index bc211a4f2..f78dbb8ff 100644 --- a/migrations/utils.go +++ b/migrations/utils.go @@ -4,6 +4,21 @@ import ( "fmt" ) +type pluginTOMLStruct map[string]map[string][]interface{} + +func CreateTOMLStruct(category, name string) pluginTOMLStruct { + return map[string]map[string][]interface{}{ + category: { + name: make([]interface{}, 0), + }, + } +} + +func (p *pluginTOMLStruct) Add(category, name string, plugin interface{}) { + cfg := map[string]map[string][]interface{}(*p) + cfg[category][name] = append(cfg[category][name], plugin) +} + func AsStringSlice(raw interface{}) ([]string, error) { rawList, ok := raw.([]interface{}) if !ok { diff --git a/plugins/outputs/postgresql/postgresql_test.go b/plugins/outputs/postgresql/postgresql_test.go index 228344cb3..38dffff80 100644 --- a/plugins/outputs/postgresql/postgresql_test.go +++ b/plugins/outputs/postgresql/postgresql_test.go @@ -65,7 +65,8 @@ func (la *LogAccumulator) Level() telegraf.LogLevel { return telegraf.Debug } -func (*LogAccumulator) RegisterErrorCallback(func()) {} +// Unused +func (*LogAccumulator) AddAttribute(string, interface{}) {} func (la *LogAccumulator) append(level pgx.LogLevel, format string, args []interface{}) { la.tb.Helper() diff --git a/testutil/capturelog.go b/testutil/capturelog.go index 4ed50f5fa..0d204ead9 100644 --- a/testutil/capturelog.go +++ b/testutil/capturelog.go @@ -50,58 +50,50 @@ func (l *CaptureLogger) loga(level byte, args ...any) { l.print(Entry{level, l.Name, fmt.Sprint(args...)}) } -func (l *CaptureLogger) Level() telegraf.LogLevel { +// We always want to output at debug level during testing to find issues easier +func (*CaptureLogger) Level() telegraf.LogLevel { return telegraf.Debug } -func (*CaptureLogger) RegisterErrorCallback(func()) {} +// Adding attributes is not supported by the test-logger +func (*CaptureLogger) AddAttribute(string, interface{}) {} -// Errorf logs an error message, patterned after log.Printf. func (l *CaptureLogger) Errorf(format string, args ...interface{}) { l.logf(LevelError, format, args...) } -// Error logs an error message, patterned after log.Print. func (l *CaptureLogger) Error(args ...interface{}) { l.loga(LevelError, args...) } -// Warnf logs a warning message, patterned after log.Printf. func (l *CaptureLogger) Warnf(format string, args ...interface{}) { l.logf(LevelWarn, format, args...) } -// Warn logs a warning message, patterned after log.Print. func (l *CaptureLogger) Warn(args ...interface{}) { l.loga(LevelWarn, args...) } -// Infof logs an information message, patterned after log.Printf. func (l *CaptureLogger) Infof(format string, args ...interface{}) { l.logf(LevelInfo, format, args...) } -// Info logs an information message, patterned after log.Print. func (l *CaptureLogger) Info(args ...interface{}) { l.loga(LevelInfo, args...) } -// Debugf logs a debug message, patterned after log.Printf. func (l *CaptureLogger) Debugf(format string, args ...interface{}) { l.logf(LevelDebug, format, args...) } -// Debug logs a debug message, patterned after log.Print. func (l *CaptureLogger) Debug(args ...interface{}) { l.loga(LevelDebug, args...) } -// Tracef logs a trace message, patterned after log.Printf. func (l *CaptureLogger) Tracef(format string, args ...interface{}) { l.logf(LevelTrace, format, args...) } -// Trace logs a trace message, patterned after log.Print. func (l *CaptureLogger) Trace(args ...interface{}) { l.loga(LevelTrace, args...) } diff --git a/testutil/log.go b/testutil/log.go index 89801676d..e8d212101 100644 --- a/testutil/log.go +++ b/testutil/log.go @@ -8,67 +8,59 @@ import ( var _ telegraf.Logger = &Logger{} -// Logger defines a logging structure for plugins. type Logger struct { Name string // Name is the plugin name, will be printed in the `[]`. Quiet bool } -func (l Logger) Level() telegraf.LogLevel { +// We always want to output at debug level during testing to find issues easier +func (Logger) Level() telegraf.LogLevel { return telegraf.Debug } -func (Logger) RegisterErrorCallback(func()) {} +// Adding attributes is not supported by the test-logger +func (Logger) AddAttribute(string, interface{}) {} -// Errorf logs an error message, patterned after log.Printf. func (l Logger) Errorf(format string, args ...interface{}) { log.Printf("E! ["+l.Name+"] "+format, args...) } -// Error logs an error message, patterned after log.Print. func (l Logger) Error(args ...interface{}) { log.Print(append([]interface{}{"E! [" + l.Name + "] "}, args...)...) } -// Warnf logs a warning message, patterned after log.Printf. func (l Logger) Warnf(format string, args ...interface{}) { log.Printf("W! ["+l.Name+"] "+format, args...) } -// Warn logs a warning message, patterned after log.Print. func (l Logger) Warn(args ...interface{}) { log.Print(append([]interface{}{"W! [" + l.Name + "] "}, args...)...) } -// Infof logs an information message, patterned after log.Printf. func (l Logger) Infof(format string, args ...interface{}) { if !l.Quiet { log.Printf("I! ["+l.Name+"] "+format, args...) } } -// Info logs an information message, patterned after log.Print. func (l Logger) Info(args ...interface{}) { if !l.Quiet { log.Print(append([]interface{}{"I! [" + l.Name + "] "}, args...)...) } } -// Debugf logs a debug message, patterned after log.Printf. func (l Logger) Debugf(format string, args ...interface{}) { if !l.Quiet { log.Printf("D! ["+l.Name+"] "+format, args...) } } -// Debug logs a debug message, patterned after log.Print. func (l Logger) Debug(args ...interface{}) { if !l.Quiet { log.Print(append([]interface{}{"D! [" + l.Name + "] "}, args...)...) } } -// Tracef logs a trace message, patterned after log.Printf. func (l Logger) Tracef(format string, args ...interface{}) { if !l.Quiet { log.Printf("T! ["+l.Name+"] "+format, args...)