CL-Softwares
/
telegraf
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

docs: json_v2 improved var naming and comments (#9907)

This commit is contained in:
Sebastian Spaink 2021-10-12 16:07:34 -05:00 committed by GitHub
parent 66da86017f
commit 0be92db8af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 55 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
plugins/parsers/json_v2/parser.go
View File

@ -12,19 +12,27 @@ import (
"github.com/tidwall/gjson" "github.com/tidwall/gjson"
) )
// Parser adheres to the parser interface, contains the parser configuration, and data required to parse JSON
type Parser struct { type Parser struct {
InputJSON []byte // These struct fields are common for a parser
Configs []Config Configs []Config
DefaultTags map[string]string DefaultTags map[string]string
Log telegraf.Logger Log telegraf.Logger
Timestamp time.Time
// **** The struct fields bellow this comment are used for processing indvidual configs ****
// measurementName is the the name of the current config used in each line protocol
measurementName string measurementName string
// timestamp is the timestamp used in each line protocol, defaults to time.Now()
timestamp time.Time
// **** Specific for object configuration ****
// subPathResults contains the results of sub-gjson path expressions provided in fields/tags table within object config
subPathResults []PathResult
// iterateObjects dictates if ExpandArray function will handle objects
iterateObjects bool iterateObjects bool
// objectConfig contains the config for an object, some info is needed while iterating over the gjson results
currentSettings JSONObject objectConfig JSONObject
pathResults []PathResult
} }
type PathResult struct { type PathResult struct {
@ -83,28 +91,27 @@ type MetricNode struct {
} }
func (p *Parser) Parse(input []byte) ([]telegraf.Metric, error) { func (p *Parser) Parse(input []byte) ([]telegraf.Metric, error) {
p.InputJSON = input
// Only valid JSON is supported // Only valid JSON is supported
if !gjson.Valid(string(p.InputJSON)) { if !gjson.Valid(string(input)) {
return nil, fmt.Errorf("Invalid JSON provided, unable to parse") return nil, fmt.Errorf("invalid JSON provided, unable to parse")
} }
var metrics []telegraf.Metric var metrics []telegraf.Metric
for _, c := range p.Configs { for _, c := range p.Configs {
// Measurement name configuration // Measurement name can either be hardcoded, or parsed from the JSON using a GJSON path expression
p.measurementName = c.MeasurementName p.measurementName = c.MeasurementName
if c.MeasurementNamePath != "" { if c.MeasurementNamePath != "" {
result := gjson.GetBytes(p.InputJSON, c.MeasurementNamePath) result := gjson.GetBytes(input, c.MeasurementNamePath)
if !result.IsArray() && !result.IsObject() { if !result.IsArray() && !result.IsObject() {
p.measurementName = result.String() p.measurementName = result.String()
} }
} }
// Timestamp configuration // timestamp defaults to current time, or can be parsed from the JSON using a GJSON path expression
p.Timestamp = time.Now() p.timestamp = time.Now()
if c.TimestampPath != "" { if c.TimestampPath != "" {
result := gjson.GetBytes(p.InputJSON, c.TimestampPath) result := gjson.GetBytes(input, c.TimestampPath)
if !result.IsArray() && !result.IsObject() { if !result.IsArray() && !result.IsObject() {
if c.TimestampFormat == "" { if c.TimestampFormat == "" {
err := fmt.Errorf("use of 'timestamp_query' requires 'timestamp_format'") err := fmt.Errorf("use of 'timestamp_query' requires 'timestamp_format'")
@ -112,24 +119,24 @@ func (p *Parser) Parse(input []byte) ([]telegraf.Metric, error) {
} }
var err error var err error
p.Timestamp, err = internal.ParseTimestamp(c.TimestampFormat, result.Value(), c.TimestampTimezone) p.timestamp, err = internal.ParseTimestamp(c.TimestampFormat, result.Value(), c.TimestampTimezone)
if err != nil { if err != nil {
return nil, err return nil, err
} }
} }
} }
fields, err := p.processMetric(c.Fields, false) fields, err := p.processMetric(input, c.Fields, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
tags, err := p.processMetric(c.Tags, true) tags, err := p.processMetric(input, c.Tags, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
objects, err := p.processObjects(c.JSONObjects) objects, err := p.processObjects(input, c.JSONObjects)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -155,7 +162,7 @@ func (p *Parser) Parse(input []byte) ([]telegraf.Metric, error) {
// processMetric will iterate over all 'field' or 'tag' configs and create metrics for each // processMetric will iterate over all 'field' or 'tag' configs and create metrics for each
// A field/tag can either be a single value or an array of values, each resulting in its own metric // A field/tag can either be a single value or an array of values, each resulting in its own metric
// For multiple configs, a set of metrics is created from the cartesian product of each separate config // For multiple configs, a set of metrics is created from the cartesian product of each separate config
func (p *Parser) processMetric(data []DataSet, tag bool) ([]telegraf.Metric, error) { func (p *Parser) processMetric(input []byte, data []DataSet, tag bool) ([]telegraf.Metric, error) {
if len(data) == 0 { if len(data) == 0 {
return nil, nil return nil, nil
} }
@ -167,7 +174,7 @@ func (p *Parser) processMetric(data []DataSet, tag bool) ([]telegraf.Metric, err
if c.Path == "" { if c.Path == "" {
return nil, fmt.Errorf("GJSON path is required") return nil, fmt.Errorf("GJSON path is required")
} }
result := gjson.GetBytes(p.InputJSON, c.Path) result := gjson.GetBytes(input, c.Path)
if result.IsObject() { if result.IsObject() {
p.Log.Debugf("Found object in the path: %s, ignoring it please use 'object' to gather metrics from objects", c.Path) p.Log.Debugf("Found object in the path: %s, ignoring it please use 'object' to gather metrics from objects", c.Path)
@ -191,7 +198,7 @@ func (p *Parser) processMetric(data []DataSet, tag bool) ([]telegraf.Metric, err
p.measurementName, p.measurementName,
map[string]string{}, map[string]string{},
map[string]interface{}{}, map[string]interface{}{},
p.Timestamp, p.timestamp,
), ),
Result: result, Result: result,
} }
@ -251,7 +258,7 @@ func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) {
p.Log.Debugf("Found object in query ignoring it please use 'object' to gather metrics from objects") p.Log.Debugf("Found object in query ignoring it please use 'object' to gather metrics from objects")
return results, nil return results, nil
} }
if result.IncludeCollection == nil && (len(p.currentSettings.FieldPaths) > 0 || len(p.currentSettings.TagPaths) > 0) { if result.IncludeCollection == nil && (len(p.objectConfig.FieldPaths) > 0 || len(p.objectConfig.TagPaths) > 0) {
result.IncludeCollection = p.existsInpathResults(result.Index, result.Raw) result.IncludeCollection = p.existsInpathResults(result.Index, result.Raw)
} }
r, err := p.combineObject(result) r, err := p.combineObject(result)
@ -264,7 +271,7 @@ func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) {
if result.IsArray() { if result.IsArray() {
var err error var err error
if result.IncludeCollection == nil && (len(p.currentSettings.FieldPaths) > 0 || len(p.currentSettings.TagPaths) > 0) { if result.IncludeCollection == nil && (len(p.objectConfig.FieldPaths) > 0 || len(p.objectConfig.TagPaths) > 0) {
result.IncludeCollection = p.existsInpathResults(result.Index, result.Raw) result.IncludeCollection = p.existsInpathResults(result.Index, result.Raw)
} }
result.ForEach(func(_, val gjson.Result) bool { result.ForEach(func(_, val gjson.Result) bool {
@ -272,7 +279,7 @@ func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) {
p.measurementName, p.measurementName,
map[string]string{}, map[string]string{},
map[string]interface{}{}, map[string]interface{}{},
p.Timestamp, p.timestamp,
) )
if val.IsObject() { if val.IsObject() {
if p.iterateObjects { if p.iterateObjects {
@ -280,7 +287,7 @@ func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) {
n.ParentIndex += val.Index n.ParentIndex += val.Index
n.Metric = m n.Metric = m
n.Result = val n.Result = val
if n.IncludeCollection == nil && (len(p.currentSettings.FieldPaths) > 0 || len(p.currentSettings.TagPaths) > 0) { if n.IncludeCollection == nil && (len(p.objectConfig.FieldPaths) > 0 || len(p.objectConfig.TagPaths) > 0) {
n.IncludeCollection = p.existsInpathResults(n.Index, n.Raw) n.IncludeCollection = p.existsInpathResults(n.Index, n.Raw)
} }
r, err := p.combineObject(n) r, err := p.combineObject(n)
@ -310,7 +317,7 @@ func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) {
n.ParentIndex += val.Index n.ParentIndex += val.Index
n.Metric = m n.Metric = m
n.Result = val n.Result = val
if n.IncludeCollection == nil && (len(p.currentSettings.FieldPaths) > 0 || len(p.currentSettings.TagPaths) > 0) { if n.IncludeCollection == nil && (len(p.objectConfig.FieldPaths) > 0 || len(p.objectConfig.TagPaths) > 0) {
n.IncludeCollection = p.existsInpathResults(n.Index, n.Raw) n.IncludeCollection = p.existsInpathResults(n.Index, n.Raw)
} }
r, err := p.expandArray(n) r, err := p.expandArray(n)
@ -324,12 +331,12 @@ func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) {
return nil, err return nil, err
} }
} else { } else {
if result.SetName == p.currentSettings.TimestampKey { if result.SetName == p.objectConfig.TimestampKey {
if p.currentSettings.TimestampFormat == "" { if p.objectConfig.TimestampFormat == "" {
err := fmt.Errorf("use of 'timestamp_query' requires 'timestamp_format'") err := fmt.Errorf("use of 'timestamp_query' requires 'timestamp_format'")
return nil, err return nil, err
} }
timestamp, err := internal.ParseTimestamp(p.currentSettings.TimestampFormat, result.Value(), p.currentSettings.TimestampTimezone) timestamp, err := internal.ParseTimestamp(p.objectConfig.TimestampFormat, result.Value(), p.objectConfig.TimestampTimezone)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -341,7 +348,7 @@ func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) {
outputName := result.OutputName outputName := result.OutputName
desiredType := result.DesiredType desiredType := result.DesiredType
if len(p.currentSettings.FieldPaths) > 0 || len(p.currentSettings.TagPaths) > 0 { if len(p.objectConfig.FieldPaths) > 0 || len(p.objectConfig.TagPaths) > 0 {
var pathResult *PathResult var pathResult *PathResult
// When IncludeCollection isn't nil, that means the current result is included in the collection. // When IncludeCollection isn't nil, that means the current result is included in the collection.
if result.IncludeCollection != nil { if result.IncludeCollection != nil {
@ -386,7 +393,7 @@ func (p *Parser) expandArray(result MetricNode) ([]telegraf.Metric, error) {
} }
func (p *Parser) existsInpathResults(index int, raw string) *PathResult { func (p *Parser) existsInpathResults(index int, raw string) *PathResult {
for _, f := range p.pathResults { for _, f := range p.subPathResults {
if f.result.Index == 0 { if f.result.Index == 0 {
for _, i := range f.result.Indexes { for _, i := range f.result.Indexes {
if i == index { if i == index {
@ -401,23 +408,23 @@ func (p *Parser) existsInpathResults(index int, raw string) *PathResult {
} }
// processObjects will iterate over all 'object' configs and create metrics for each // processObjects will iterate over all 'object' configs and create metrics for each
func (p *Parser) processObjects(objects []JSONObject) ([]telegraf.Metric, error) { func (p *Parser) processObjects(input []byte, objects []JSONObject) ([]telegraf.Metric, error) {
p.iterateObjects = true p.iterateObjects = true
var t []telegraf.Metric var t []telegraf.Metric
for _, c := range objects { for _, c := range objects {
p.currentSettings = c p.objectConfig = c
if c.Path == "" { if c.Path == "" {
return nil, fmt.Errorf("GJSON path is required") return nil, fmt.Errorf("GJSON path is required")
} }
result := gjson.GetBytes(p.InputJSON, c.Path) result := gjson.GetBytes(input, c.Path)
scopedJSON := []byte(result.Raw) scopedJSON := []byte(result.Raw)
for _, f := range c.FieldPaths { for _, f := range c.FieldPaths {
var r PathResult var r PathResult
r.result = gjson.GetBytes(scopedJSON, f.Path) r.result = gjson.GetBytes(scopedJSON, f.Path)
r.DataSet = f r.DataSet = f
p.pathResults = append(p.pathResults, r) p.subPathResults = append(p.subPathResults, r)
} }
for _, f := range c.TagPaths { for _, f := range c.TagPaths {
@ -425,7 +432,7 @@ func (p *Parser) processObjects(objects []JSONObject) ([]telegraf.Metric, error)
r.result = gjson.GetBytes(scopedJSON, f.Path) r.result = gjson.GetBytes(scopedJSON, f.Path)
r.DataSet = f r.DataSet = f
r.tag = true r.tag = true
p.pathResults = append(p.pathResults, r) p.subPathResults = append(p.subPathResults, r)
} }
if result.Type == gjson.Null { if result.Type == gjson.Null {
@ -438,7 +445,7 @@ func (p *Parser) processObjects(objects []JSONObject) ([]telegraf.Metric, error)
p.measurementName, p.measurementName,
map[string]string{}, map[string]string{},
map[string]interface{}{}, map[string]interface{}{},
p.Timestamp, p.timestamp,
), ),
Result: result, Result: result,
} }
@ -472,12 +479,12 @@ func (p *Parser) combineObject(result MetricNode) ([]telegraf.Metric, error) {
} }
var outputName string var outputName string
if p.currentSettings.DisablePrependKeys { if p.objectConfig.DisablePrependKeys {
outputName = strings.ReplaceAll(key.String(), " ", "_") outputName = strings.ReplaceAll(key.String(), " ", "_")
} else { } else {
outputName = setName outputName = setName
} }
for k, n := range p.currentSettings.Renames { for k, n := range p.objectConfig.Renames {
if k == setName { if k == setName {
outputName = n outputName = n
break break
@ -490,7 +497,7 @@ func (p *Parser) combineObject(result MetricNode) ([]telegraf.Metric, error) {
arrayNode.SetName = setName arrayNode.SetName = setName
arrayNode.Result = val arrayNode.Result = val
for k, t := range p.currentSettings.Fields { for k, t := range p.objectConfig.Fields {
if setName == k { if setName == k {
arrayNode.DesiredType = t arrayNode.DesiredType = t
break break
@ -498,7 +505,7 @@ func (p *Parser) combineObject(result MetricNode) ([]telegraf.Metric, error) {
} }
tag := false tag := false
for _, t := range p.currentSettings.Tags { for _, t := range p.objectConfig.Tags {
if setName == t { if setName == t {
tag = true tag = true
break break
@ -531,11 +538,11 @@ func (p *Parser) combineObject(result MetricNode) ([]telegraf.Metric, error) {
} }
func (p *Parser) isIncluded(key string, val gjson.Result) bool { func (p *Parser) isIncluded(key string, val gjson.Result) bool {
if len(p.currentSettings.IncludedKeys) == 0 { if len(p.objectConfig.IncludedKeys) == 0 {
return true return true
} }
// automatically adds tags to included_keys so it does NOT have to be repeated in the config // automatically adds tags to included_keys so it does NOT have to be repeated in the config
allKeys := append(p.currentSettings.IncludedKeys, p.currentSettings.Tags...) allKeys := append(p.objectConfig.IncludedKeys, p.objectConfig.Tags...)
for _, i := range allKeys { for _, i := range allKeys {
if i == key { if i == key {
return true return true
@ -551,7 +558,7 @@ func (p *Parser) isIncluded(key string, val gjson.Result) bool {
} }
func (p *Parser) isExcluded(key string) bool { func (p *Parser) isExcluded(key string) bool {
for _, i := range p.currentSettings.ExcludedKeys { for _, i := range p.objectConfig.ExcludedKeys {
if i == key { if i == key {
return true return true
} }
@ -576,25 +583,25 @@ func (p *Parser) convertType(input gjson.Result, desiredType string, name string
case "uint": case "uint":
r, err := strconv.ParseUint(inputType, 10, 64) r, err := strconv.ParseUint(inputType, 10, 64)
if err != nil { if err != nil {
return nil, fmt.Errorf("Unable to convert field '%s' to type uint: %v", name, err) return nil, fmt.Errorf("unable to convert field '%s' to type uint: %v", name, err)
} }
return r, nil return r, nil
case "int": case "int":
r, err := strconv.ParseInt(inputType, 10, 64) r, err := strconv.ParseInt(inputType, 10, 64)
if err != nil { if err != nil {
return nil, fmt.Errorf("Unable to convert field '%s' to type int: %v", name, err) return nil, fmt.Errorf("unable to convert field '%s' to type int: %v", name, err)
} }
return r, nil return r, nil
case "float": case "float":
r, err := strconv.ParseFloat(inputType, 64) r, err := strconv.ParseFloat(inputType, 64)
if err != nil { if err != nil {
return nil, fmt.Errorf("Unable to convert field '%s' to type float: %v", name, err) return nil, fmt.Errorf("unable to convert field '%s' to type float: %v", name, err)
} }
return r, nil return r, nil
case "bool": case "bool":
r, err := strconv.ParseBool(inputType) r, err := strconv.ParseBool(inputType)
if err != nil { if err != nil {
return nil, fmt.Errorf("Unable to convert field '%s' to type bool: %v", name, err) return nil, fmt.Errorf("unable to convert field '%s' to type bool: %v", name, err)
} }
return r, nil return r, nil
} }
@ -631,7 +638,7 @@ func (p *Parser) convertType(input gjson.Result, desiredType string, name string
} else if inputType == 1 { } else if inputType == 1 {
return true, nil return true, nil
} else { } else {
return nil, fmt.Errorf("Unable to convert field '%s' to type bool", name) return nil, fmt.Errorf("unable to convert field '%s' to type bool", name)
} }
} }
} }