diff --git a/config/config.go b/config/config.go index 36b65aa01..d6f1a9fa7 100644 --- a/config/config.go +++ b/config/config.go @@ -693,8 +693,17 @@ func (c *Config) probeParser(table *ast.Table) bool { var dataformat string c.getFieldString(table, "data_format", &dataformat) - _, ok := parsers.Parsers[dataformat] - return ok + creator, ok := parsers.Parsers[dataformat] + if !ok { + return false + } + + // Try to parse the options to detect if any of them is misspelled + // We don't actually use the parser, so no need to check the error. + parser := creator("") + _ = c.toml.UnmarshalTable(table, parser) + + return true } func (c *Config) addParser(parentcategory, parentname string, table *ast.Table) (*models.RunningParser, error) { @@ -754,6 +763,7 @@ func (c *Config) addProcessor(name string, table *ast.Table) error { // for the input both need to miss the entry. We count the // missing entries at the end. missCount := make(map[string]int) + missCountThreshold := 0 c.setLocalMissingTomlFieldTracker(missCount) defer c.resetMissingTomlFieldTracker() @@ -772,6 +782,7 @@ func (c *Config) addProcessor(name string, table *ast.Table) error { // it can accept arbitrary data-formats, so build the requested parser and // set it. if t, ok := processor.(telegraf.ParserPlugin); ok { + missCountThreshold = 2 parser, err := c.addParser("processors", name, table) if err != nil { return fmt.Errorf("adding parser failed: %w", err) @@ -780,6 +791,7 @@ func (c *Config) addProcessor(name string, table *ast.Table) error { } if t, ok := processor.(telegraf.ParserFuncPlugin); ok { + missCountThreshold = 2 if !c.probeParser(table) { return errors.New("parser not found") } @@ -804,6 +816,16 @@ func (c *Config) addProcessor(name string, table *ast.Table) error { rf = models.NewRunningProcessor(streamingProcessor, processorConfig) c.AggProcessors = append(c.AggProcessors, rf) + // Check the number of misses against the threshold + for key, count := range missCount { + if count <= missCountThreshold { + continue + } + if err := c.missingTomlField(nil, key); err != nil { + return err + } + } + return nil } @@ -884,6 +906,7 @@ func (c *Config) addInput(name string, table *ast.Table) error { // for the input both need to miss the entry. We count the // missing entries at the end. missCount := make(map[string]int) + missCountThreshold := 0 c.setLocalMissingTomlFieldTracker(missCount) defer c.resetMissingTomlFieldTracker() @@ -902,6 +925,7 @@ func (c *Config) addInput(name string, table *ast.Table) error { // If the input has a SetParser or SetParserFunc function, it can accept // arbitrary data-formats, so build the requested parser and set it. if t, ok := input.(telegraf.ParserPlugin); ok { + missCountThreshold = 1 parser, err := c.addParser("inputs", name, table) if err != nil { return fmt.Errorf("adding parser failed: %w", err) @@ -912,6 +936,7 @@ func (c *Config) addInput(name string, table *ast.Table) error { // Keep the old interface for backward compatibility if t, ok := input.(parsers.ParserInput); ok { // DEPRECATED: Please switch your plugin to telegraf.ParserPlugin. + missCountThreshold = 1 parser, err := c.addParser("inputs", name, table) if err != nil { return fmt.Errorf("adding parser failed: %w", err) @@ -920,6 +945,7 @@ func (c *Config) addInput(name string, table *ast.Table) error { } if t, ok := input.(telegraf.ParserFuncPlugin); ok { + missCountThreshold = 1 if !c.probeParser(table) { return errors.New("parser not found") } @@ -930,6 +956,7 @@ func (c *Config) addInput(name string, table *ast.Table) error { if t, ok := input.(parsers.ParserFuncInput); ok { // DEPRECATED: Please switch your plugin to telegraf.ParserFuncPlugin. + missCountThreshold = 1 if !c.probeParser(table) { return errors.New("parser not found") } @@ -963,7 +990,7 @@ func (c *Config) addInput(name string, table *ast.Table) error { // Check the number of misses against the threshold for key, count := range missCount { - if count <= 1 { + if count <= missCountThreshold { continue } if err := c.missingTomlField(nil, key); err != nil { @@ -1243,11 +1270,27 @@ func (c *Config) missingTomlField(_ reflect.Type, key string) error { } func (c *Config) setLocalMissingTomlFieldTracker(counter map[string]int) { - f := func(_ reflect.Type, key string) error { - if c, ok := counter[key]; ok { - counter[key] = c + 1 - } else { + f := func(t reflect.Type, key string) error { + // Check if we are in a root element that might share options among + // each other. Those root elements are plugins of all types. + // All other elements are subtables of their respective plugin and + // should just be hit once anyway. Therefore, we mark them with a + // high number to handle them correctly later. + pt := reflect.PtrTo(t) + root := pt.Implements(reflect.TypeOf((*telegraf.Input)(nil)).Elem()) + root = root || pt.Implements(reflect.TypeOf((*telegraf.ServiceInput)(nil)).Elem()) + root = root || pt.Implements(reflect.TypeOf((*telegraf.Output)(nil)).Elem()) + root = root || pt.Implements(reflect.TypeOf((*telegraf.Aggregator)(nil)).Elem()) + root = root || pt.Implements(reflect.TypeOf((*telegraf.Processor)(nil)).Elem()) + root = root || pt.Implements(reflect.TypeOf((*telegraf.Parser)(nil)).Elem()) + + c, ok := counter[key] + if !root { + counter[key] = 100 + } else if !ok { counter[key] = 1 + } else { + counter[key] = c + 1 } return nil } diff --git a/config/config_test.go b/config/config_test.go index 3b3592017..7ceb75baf 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -285,10 +285,70 @@ func TestConfig_LoadSpecialTypes(t *testing.T) { } func TestConfig_FieldNotDefined(t *testing.T) { - c := NewConfig() - err := c.LoadConfig("./testdata/invalid_field.toml") - require.Error(t, err, "invalid field name") - require.Equal(t, "error loading config file ./testdata/invalid_field.toml: plugin inputs.http_listener_v2: line 1: configuration specified the fields [\"not_a_field\"], but they weren't used", err.Error()) + tests := []struct { + name string + filename string + expected string + }{ + { + name: "in input plugin without parser", + filename: "./testdata/invalid_field.toml", + expected: `line 1: configuration specified the fields ["not_a_field"], but they weren't used`, + }, + { + name: "in input plugin with parser", + filename: "./testdata/invalid_field_with_parser.toml", + expected: `line 1: configuration specified the fields ["not_a_field"], but they weren't used`, + }, + { + name: "in input plugin with parser func", + filename: "./testdata/invalid_field_with_parserfunc.toml", + expected: `line 1: configuration specified the fields ["not_a_field"], but they weren't used`, + }, + { + name: "in parser of input plugin", + filename: "./testdata/invalid_field_in_parser_table.toml", + expected: `line 1: configuration specified the fields ["not_a_field"], but they weren't used`, + }, + { + name: "in parser of input plugin with parser-func", + filename: "./testdata/invalid_field_in_parserfunc_table.toml", + expected: `line 1: configuration specified the fields ["not_a_field"], but they weren't used`, + }, + { + name: "in processor plugin without parser", + filename: "./testdata/invalid_field_processor.toml", + expected: `line 1: configuration specified the fields ["not_a_field"], but they weren't used`, + }, + { + name: "in processor plugin with parser", + filename: "./testdata/invalid_field_processor_with_parser.toml", + expected: `line 1: configuration specified the fields ["not_a_field"], but they weren't used`, + }, + { + name: "in processor plugin with parser func", + filename: "./testdata/invalid_field_processor_with_parserfunc.toml", + expected: `line 1: configuration specified the fields ["not_a_field"], but they weren't used`, + }, + { + name: "in parser of processor plugin", + filename: "./testdata/invalid_field_processor_in_parser_table.toml", + expected: `line 1: configuration specified the fields ["not_a_field"], but they weren't used`, + }, + { + name: "in parser of processor plugin with parser-func", + filename: "./testdata/invalid_field_processor_in_parserfunc_table.toml", + expected: `line 1: configuration specified the fields ["not_a_field"], but they weren't used`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewConfig() + err := c.LoadConfig(tt.filename) + require.ErrorContains(t, err, tt.expected) + }) + } } func TestConfig_WrongFieldType(t *testing.T) { @@ -843,6 +903,36 @@ func (m *MockupInputPlugin) SetParser(parser telegraf.Parser) { m.parser = parser } +/*** Mockup INPUT plugin with ParserFunc interface ***/ +type MockupInputPluginParserFunc struct { + parserFunc telegraf.ParserFunc +} + +func (m *MockupInputPluginParserFunc) SampleConfig() string { + return "Mockup test input plugin" +} +func (m *MockupInputPluginParserFunc) Gather(_ telegraf.Accumulator) error { + return nil +} +func (m *MockupInputPluginParserFunc) SetParserFunc(pf telegraf.ParserFunc) { + m.parserFunc = pf +} + +/*** Mockup INPUT plugin without ParserFunc interface ***/ +type MockupInputPluginParserOnly struct { + parser telegraf.Parser +} + +func (m *MockupInputPluginParserOnly) SampleConfig() string { + return "Mockup test input plugin" +} +func (m *MockupInputPluginParserOnly) Gather(_ telegraf.Accumulator) error { + return nil +} +func (m *MockupInputPluginParserOnly) SetParser(p telegraf.Parser) { + m.parser = p +} + /*** Mockup PROCESSOR plugin for testing to avoid cyclic dependencies ***/ type MockupProcessorPluginParser struct { Parser telegraf.Parser @@ -871,6 +961,73 @@ func (m *MockupProcessorPluginParser) SetParserFunc(f telegraf.ParserFunc) { m.ParserFunc = f } +/*** Mockup PROCESSOR plugin without parser ***/ +type MockupProcessorPlugin struct{} + +func (m *MockupProcessorPlugin) Start(_ telegraf.Accumulator) error { + return nil +} +func (m *MockupProcessorPlugin) Stop() error { + return nil +} +func (m *MockupProcessorPlugin) SampleConfig() string { + return "Mockup test processor plugin with parser" +} +func (m *MockupProcessorPlugin) Apply(_ ...telegraf.Metric) []telegraf.Metric { + return nil +} +func (m *MockupProcessorPlugin) Add(_ telegraf.Metric, _ telegraf.Accumulator) error { + return nil +} + +/*** Mockup PROCESSOR plugin with parser ***/ +type MockupProcessorPluginParserOnly struct { + Parser telegraf.Parser +} + +func (m *MockupProcessorPluginParserOnly) Start(_ telegraf.Accumulator) error { + return nil +} +func (m *MockupProcessorPluginParserOnly) Stop() error { + return nil +} +func (m *MockupProcessorPluginParserOnly) SampleConfig() string { + return "Mockup test processor plugin with parser" +} +func (m *MockupProcessorPluginParserOnly) Apply(_ ...telegraf.Metric) []telegraf.Metric { + return nil +} +func (m *MockupProcessorPluginParserOnly) Add(_ telegraf.Metric, _ telegraf.Accumulator) error { + return nil +} +func (m *MockupProcessorPluginParserOnly) SetParser(parser telegraf.Parser) { + m.Parser = parser +} + +/*** Mockup PROCESSOR plugin with parser-function ***/ +type MockupProcessorPluginParserFunc struct { + Parser telegraf.ParserFunc +} + +func (m *MockupProcessorPluginParserFunc) Start(_ telegraf.Accumulator) error { + return nil +} +func (m *MockupProcessorPluginParserFunc) Stop() error { + return nil +} +func (m *MockupProcessorPluginParserFunc) SampleConfig() string { + return "Mockup test processor plugin with parser" +} +func (m *MockupProcessorPluginParserFunc) Apply(_ ...telegraf.Metric) []telegraf.Metric { + return nil +} +func (m *MockupProcessorPluginParserFunc) Add(_ telegraf.Metric, _ telegraf.Accumulator) error { + return nil +} +func (m *MockupProcessorPluginParserFunc) SetParserFunc(pf telegraf.ParserFunc) { + m.Parser = pf +} + /*** Mockup OUTPUT plugin for testing to avoid cyclic dependencies ***/ type MockupOuputPlugin struct { URL string `toml:"url"` @@ -903,6 +1060,12 @@ func init() { inputs.Add("parser_test_old", func() telegraf.Input { return &MockupInputPluginParserOld{} }) + inputs.Add("parser", func() telegraf.Input { + return &MockupInputPluginParserOnly{} + }) + inputs.Add("parser_func", func() telegraf.Input { + return &MockupInputPluginParserFunc{} + }) inputs.Add("exec", func() telegraf.Input { return &MockupInputPlugin{Timeout: Duration(time.Second * 5)} }) @@ -916,10 +1079,19 @@ func init() { return &MockupInputPlugin{} }) - // Register the mockup output plugin for the required names + // Register the mockup processor plugin for the required names processors.Add("parser_test", func() telegraf.Processor { return &MockupProcessorPluginParser{} }) + processors.Add("processor", func() telegraf.Processor { + return &MockupProcessorPlugin{} + }) + processors.Add("processor_parser", func() telegraf.Processor { + return &MockupProcessorPluginParserOnly{} + }) + processors.Add("processor_parserfunc", func() telegraf.Processor { + return &MockupProcessorPluginParserFunc{} + }) // Register the mockup output plugin for the required names outputs.Add("azure_monitor", func() telegraf.Output { diff --git a/config/testdata/invalid_field_in_parser_table.toml b/config/testdata/invalid_field_in_parser_table.toml new file mode 100644 index 000000000..b2d73015f --- /dev/null +++ b/config/testdata/invalid_field_in_parser_table.toml @@ -0,0 +1,5 @@ +[[inputs.parser]] + data_format = "xpath_json" + + [[inputs.parser.xpath]] + not_a_field = true diff --git a/config/testdata/invalid_field_in_parserfunc_table.toml b/config/testdata/invalid_field_in_parserfunc_table.toml new file mode 100644 index 000000000..36b2b201c --- /dev/null +++ b/config/testdata/invalid_field_in_parserfunc_table.toml @@ -0,0 +1,5 @@ +[[inputs.parser_func]] + data_format = "xpath_json" + + [[inputs.parser_func.xpath]] + not_a_field = true diff --git a/config/testdata/invalid_field_processor.toml b/config/testdata/invalid_field_processor.toml new file mode 100644 index 000000000..ec72055e3 --- /dev/null +++ b/config/testdata/invalid_field_processor.toml @@ -0,0 +1,2 @@ +[[processors.processor]] + not_a_field = true diff --git a/config/testdata/invalid_field_processor_in_parser.toml b/config/testdata/invalid_field_processor_in_parser.toml new file mode 100644 index 000000000..39877a7f6 --- /dev/null +++ b/config/testdata/invalid_field_processor_in_parser.toml @@ -0,0 +1,3 @@ +[[processors.processor_parser]] + not_a_field = true + data_format = "influx" diff --git a/config/testdata/invalid_field_processor_in_parser_table.toml b/config/testdata/invalid_field_processor_in_parser_table.toml new file mode 100644 index 000000000..1a583d9ae --- /dev/null +++ b/config/testdata/invalid_field_processor_in_parser_table.toml @@ -0,0 +1,5 @@ +[[processors.processor_parser]] + data_format = "xpath_json" + + [[processors.processor_parser.xpath]] + not_a_field = true diff --git a/config/testdata/invalid_field_processor_in_parserfunc.toml b/config/testdata/invalid_field_processor_in_parserfunc.toml new file mode 100644 index 000000000..f7e6a4310 --- /dev/null +++ b/config/testdata/invalid_field_processor_in_parserfunc.toml @@ -0,0 +1,3 @@ +[[processors.processor_parserfunc]] + not_a_field = true + data_format = "influx" diff --git a/config/testdata/invalid_field_processor_in_parserfunc_table.toml b/config/testdata/invalid_field_processor_in_parserfunc_table.toml new file mode 100644 index 000000000..36b2b201c --- /dev/null +++ b/config/testdata/invalid_field_processor_in_parserfunc_table.toml @@ -0,0 +1,5 @@ +[[inputs.parser_func]] + data_format = "xpath_json" + + [[inputs.parser_func.xpath]] + not_a_field = true diff --git a/config/testdata/invalid_field_processor_with_parser.toml b/config/testdata/invalid_field_processor_with_parser.toml new file mode 100644 index 000000000..39877a7f6 --- /dev/null +++ b/config/testdata/invalid_field_processor_with_parser.toml @@ -0,0 +1,3 @@ +[[processors.processor_parser]] + not_a_field = true + data_format = "influx" diff --git a/config/testdata/invalid_field_processor_with_parserfunc.toml b/config/testdata/invalid_field_processor_with_parserfunc.toml new file mode 100644 index 000000000..f7e6a4310 --- /dev/null +++ b/config/testdata/invalid_field_processor_with_parserfunc.toml @@ -0,0 +1,3 @@ +[[processors.processor_parserfunc]] + not_a_field = true + data_format = "influx" diff --git a/config/testdata/invalid_field_with_parser.toml b/config/testdata/invalid_field_with_parser.toml new file mode 100644 index 000000000..bb366c846 --- /dev/null +++ b/config/testdata/invalid_field_with_parser.toml @@ -0,0 +1,3 @@ +[[inputs.parser]] + not_a_field = true + data_format = "influx" diff --git a/config/testdata/invalid_field_with_parserfunc.toml b/config/testdata/invalid_field_with_parserfunc.toml new file mode 100644 index 000000000..dc771dad7 --- /dev/null +++ b/config/testdata/invalid_field_with_parserfunc.toml @@ -0,0 +1,3 @@ +[[inputs.parser_func]] + not_a_field = true + data_format = "influx"