From 283af2b1da5fa8cab70eb728bbb3626e1c1089fe Mon Sep 17 00:00:00 2001 From: Chase Sterling Date: Fri, 28 Jul 2023 11:41:04 -0400 Subject: [PATCH] feat(serializers.template): Add new template based serializer (#13656) --- docs/LICENSE_OF_DEPENDENCIES.md | 1 + go.mod | 5 +- go.sum | 16 +- plugins/serializers/all/template.go | 7 + plugins/serializers/template/README.md | 45 +++++ plugins/serializers/template/template.go | 98 ++++++++++ plugins/serializers/template/template_test.go | 182 ++++++++++++++++++ 7 files changed, 349 insertions(+), 5 deletions(-) create mode 100644 plugins/serializers/all/template.go create mode 100644 plugins/serializers/template/README.md create mode 100644 plugins/serializers/template/template.go create mode 100644 plugins/serializers/template/template_test.go diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index b7acc050b..f5b211cdf 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -312,6 +312,7 @@ following works: - github.com/sirupsen/logrus [MIT License](https://github.com/sirupsen/logrus/blob/master/LICENSE) - github.com/sleepinggenius2/gosmi [MIT License](https://github.com/sleepinggenius2/gosmi/blob/master/LICENSE) - github.com/snowflakedb/gosnowflake [Apache License 2.0](https://github.com/snowflakedb/gosnowflake/blob/master/LICENSE) +- github.com/spf13/cast [MIT License](https://github.com/spf13/cast/blob/master/LICENSE) - github.com/spf13/pflag [BSD 3-Clause "New" or "Revised" License](https://github.com/spf13/pflag/blob/master/LICENSE) - github.com/srebhan/cborquery [MIT License](https://github.com/srebhan/cborquery/blob/main/LICENSE) - github.com/stoewer/go-strcase [MIT License](https://github.com/stoewer/go-strcase/blob/master/LICENSE) diff --git a/go.mod b/go.mod index 8d752bc5c..d41be3a9b 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/ClickHouse/clickhouse-go v1.5.4 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Masterminds/sprig v2.22.0+incompatible + github.com/Masterminds/sprig/v3 v3.2.3 github.com/Mellanox/rdmamap v1.1.0 github.com/Shopify/sarama v1.38.1 github.com/aerospike/aerospike-client-go/v5 v5.11.0 @@ -238,6 +239,7 @@ require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect + github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect github.com/alecthomas/participle v0.4.1 // indirect github.com/andybalholm/brotli v1.0.5 // indirect @@ -333,7 +335,7 @@ require ( github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/packer-plugin-sdk v0.3.2 // indirect github.com/hashicorp/serf v0.10.1 // indirect - github.com/huandu/xstrings v1.3.2 // indirect + github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -415,6 +417,7 @@ require ( github.com/signalfx/com_signalfx_metrics_protobuf v0.0.3 // indirect github.com/signalfx/gohistogram v0.0.0-20160107210732-1ccfd2ff5083 // indirect github.com/signalfx/sapm-proto v0.12.0 // indirect + github.com/spf13/cast v1.5.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/stretchr/objx v0.5.0 // indirect diff --git a/go.sum b/go.sum index 01bf06f94..5e8163f2e 100644 --- a/go.sum +++ b/go.sum @@ -141,10 +141,13 @@ github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJ github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Mellanox/rdmamap v1.1.0 h1:A/W1wAXw+6vm58f3VklrIylgV+eDJlPVIMaIKuxgUT4= github.com/Mellanox/rdmamap v1.1.0/go.mod h1:fN+/V9lf10ABnDCwTaXRjeeWijLt2iVLETnK+sx/LY8= github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= @@ -519,8 +522,8 @@ github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk= github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -811,9 +814,10 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= -github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -1102,6 +1106,7 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -1116,6 +1121,7 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY= github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ= @@ -1390,6 +1396,8 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= diff --git a/plugins/serializers/all/template.go b/plugins/serializers/all/template.go new file mode 100644 index 000000000..783b90d13 --- /dev/null +++ b/plugins/serializers/all/template.go @@ -0,0 +1,7 @@ +//go:build !custom || serializers || serializers.template + +package all + +import ( + _ "github.com/influxdata/telegraf/plugins/serializers/template" // register plugin +) diff --git a/plugins/serializers/template/README.md b/plugins/serializers/template/README.md new file mode 100644 index 000000000..a2485fb50 --- /dev/null +++ b/plugins/serializers/template/README.md @@ -0,0 +1,45 @@ +# Template Serializer + +The `template` output data format outputs metrics using an user defined go template. +[Sprig](http://masterminds.github.io/sprig/) helper functions are also available. + +## Configuration + +```toml +[[outputs.file]] + ## Files to write to, "stdout" is a specially handled file. + files = ["stdout", "/tmp/metrics.out"] + + ## Data format to output. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + data_format = "template" + + ## Go template which defines output format + template = '{{ .Tag "host" }} {{ .Field "available" }}' + + ## When used with output plugins that allow for batch serialisation + ## the template for the entire batch can be defined + # use_batch_format = true # The 'file' plugin allows batch mode with this option + # batch_template = ''' +{{range $metric := . -}} +{{$metric.Tag "host"}}: {{range $metric.Fields | keys | initial -}} +{{.}}={{get $metric.Fields .}}, {{end}} +{{- $metric.Fields|keys|last}}={{$metric.Fields|values|last}} +{{end -}} +''' +``` + +### Batch mode + +When an output plugin emits multiple metrics in a batch fashion, by default the +template will just be repeated for each metric. If you would like to specifically +define how a batch should be formatted, you can use a `batch_template` instead. +In this mode, the context of the template (the 'dot') will be a slice of metrics. + +```toml +batch_template = '''My batch metric names: {{range $index, $metric := . -}} +{{if $index}}, {{ end }}{{ $metric.Name }} +{{- end }}''' +``` diff --git a/plugins/serializers/template/template.go b/plugins/serializers/template/template.go new file mode 100644 index 000000000..a734c4383 --- /dev/null +++ b/plugins/serializers/template/template.go @@ -0,0 +1,98 @@ +package template + +import ( + "bytes" + "fmt" + "text/template" + + "github.com/Masterminds/sprig/v3" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/serializers" +) + +type Serializer struct { + Template string `toml:"template"` + BatchTemplate string `toml:"batch_template"` + Log telegraf.Logger `toml:"-"` + + tmplMetric *template.Template + tmplBatch *template.Template +} + +func (s *Serializer) Init() error { + // Setting defaults + var err error + + s.tmplMetric, err = template.New("template").Funcs(sprig.TxtFuncMap()).Parse(s.Template) + if err != nil { + return fmt.Errorf("creating template failed: %w", err) + } + if s.BatchTemplate == "" { + s.BatchTemplate = fmt.Sprintf("{{range .}}%s{{end}}", s.Template) + } + s.tmplBatch, err = template.New("batch template").Funcs(sprig.TxtFuncMap()).Parse(s.BatchTemplate) + if err != nil { + return fmt.Errorf("creating batch template failed: %w", err) + } + return nil +} + +func (s *Serializer) Serialize(metric telegraf.Metric) ([]byte, error) { + m, ok := metric.(telegraf.TemplateMetric) + if !ok { + s.Log.Errorf("metric of type %T is not a template metric", metric) + return nil, nil + } + var b bytes.Buffer + // The template was defined for one metric, just execute it + if s.Template != "" { + if err := s.tmplMetric.Execute(&b, &m); err != nil { + s.Log.Errorf("failed to execute template: %v", err) + return nil, nil + } + return b.Bytes(), nil + } + + // The template was defined for a batch of metrics, so wrap the metric into a slice + if s.BatchTemplate != "" { + metrics := []telegraf.TemplateMetric{m} + if err := s.tmplBatch.Execute(&b, &metrics); err != nil { + s.Log.Errorf("failed to execute batch template: %v", err) + return nil, nil + } + return b.Bytes(), nil + } + + // No template was defined + return nil, nil +} + +func (s *Serializer) SerializeBatch(metrics []telegraf.Metric) ([]byte, error) { + newMetrics := make([]telegraf.TemplateMetric, 0, len(metrics)) + + for _, metric := range metrics { + m, ok := metric.(telegraf.TemplateMetric) + if !ok { + s.Log.Errorf("metric of type %T is not a template metric", metric) + return nil, nil + } + newMetrics = append(newMetrics, m) + } + + var b bytes.Buffer + if err := s.tmplBatch.Execute(&b, &newMetrics); err != nil { + s.Log.Errorf("failed to execute batch template: %v", err) + return nil, nil + } + + return b.Bytes(), nil +} + +func init() { + serializers.Add("template", + func() serializers.Serializer { + return &Serializer{} + }, + ) +} diff --git a/plugins/serializers/template/template_test.go b/plugins/serializers/template/template_test.go new file mode 100644 index 000000000..5fea65d3c --- /dev/null +++ b/plugins/serializers/template/template_test.go @@ -0,0 +1,182 @@ +package template + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/metric" +) + +func TestSerializer(t *testing.T) { + var tests = []struct { + name string + input telegraf.Metric + template string + output []byte + errReason string + }{ + { + name: "name", + input: metric.New( + "cpu", + map[string]string{}, + map[string]interface{}{}, + time.Unix(100, 0), + ), + template: "{{ .Name }}", + output: []byte("cpu"), + }, + { + name: "time", + input: metric.New( + "cpu", + map[string]string{}, + map[string]interface{}{}, + time.Unix(100, 0), + ), + template: "{{ .Time.Unix }}", + output: []byte("100"), + }, + { + name: "specific field", + input: metric.New( + "cpu", + map[string]string{}, + map[string]interface{}{ + "x": 42.0, + "y": 43.0, + }, + time.Unix(100, 0), + ), + template: `{{ .Field "x" }}`, + output: []byte("42"), + }, + { + name: "specific tag", + input: metric.New( + "cpu", + map[string]string{ + "host": "localhost", + "cpu": "CPU0", + }, + map[string]interface{}{}, + time.Unix(100, 0), + ), + template: `{{ .Tag "cpu" }}`, + output: []byte("CPU0"), + }, + { + name: "all fields", + input: metric.New( + "cpu", + map[string]string{}, + map[string]interface{}{ + "x": 42.0, + "y": 43.0, + }, + time.Unix(100, 0), + ), + template: `{{ range $k, $v := .Fields }}{{$k}}={{$v}},{{end}}`, + output: []byte("x=42,y=43,"), + }, + { + name: "all tags", + input: metric.New( + "cpu", + map[string]string{ + "host": "localhost", + "cpu": "CPU0", + }, + map[string]interface{}{}, + time.Unix(100, 0), + ), + template: `{{ range $k, $v := .Tags }}{{$k}}={{$v}},{{end}}`, + output: []byte("cpu=CPU0,host=localhost,"), + }, + { + name: "string", + input: metric.New( + "cpu", + map[string]string{ + "host": "localhost", + "cpu": "CPU0", + }, + map[string]interface{}{ + "x": 42.0, + "y": 43.0, + }, + time.Unix(100, 0), + ), + template: "{{ .String }}", + output: []byte("cpu map[cpu:CPU0 host:localhost] map[x:42 y:43] 100000000000"), + }, + { + name: "complex", + input: metric.New( + "cpu", + map[string]string{ + "tag1": "tag", + }, + map[string]interface{}{ + "value": 42.0, + }, + time.Unix(0, 0), + ), + template: `{{ .Name }} {{ range $k, $v := .Fields}}{{$k}}={{$v}}{{end}} {{ .Tag "tag1" }} {{.Time.UnixNano}} literal`, + output: []byte("cpu value=42 tag 0 literal"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + serializer := &Serializer{ + Template: tt.template, + } + require.NoError(t, serializer.Init()) + output, err := serializer.Serialize(tt.input) + if tt.errReason != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tt.errReason) + } + require.Equal(t, string(tt.output), string(output)) + // Ensure we get the same output in batch mode + batchOutput, err := serializer.SerializeBatch([]telegraf.Metric{tt.input}) + if tt.errReason != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tt.errReason) + } + require.Equal(t, string(tt.output), string(batchOutput)) + }) + } +} + +func TestSerializeBatch(t *testing.T) { + m := metric.New( + "cpu", + map[string]string{}, + map[string]interface{}{ + "value": 42.0, + }, + time.Unix(0, 0), + ) + metrics := []telegraf.Metric{m, m} + s := &Serializer{BatchTemplate: `{{ range $index, $metric := . }}{{$index}}: {{$metric.Name}} {{$metric.Field "value"}} +{{end}}`} + require.NoError(t, s.Init()) + buf, err := s.SerializeBatch(metrics) + require.NoError(t, err) + require.Equal( + t, + string(buf), + `0: cpu 42 +1: cpu 42 +`, + ) + // A batch template should still work when serializing a single metric + singleBuf, err := s.Serialize(m) + require.NoError(t, err) + require.Equal(t, string(singleBuf), "0: cpu 42\n") +}