feat(serializers.template): Add new template based serializer (#13656)

This commit is contained in:
Chase Sterling 2023-07-28 11:41:04 -04:00 committed by GitHub
parent 3f8e916455
commit 283af2b1da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 349 additions and 5 deletions

View File

@ -312,6 +312,7 @@ following works:
- github.com/sirupsen/logrus [MIT License](https://github.com/sirupsen/logrus/blob/master/LICENSE)
- github.com/sleepinggenius2/gosmi [MIT License](https://github.com/sleepinggenius2/gosmi/blob/master/LICENSE)
- github.com/snowflakedb/gosnowflake [Apache License 2.0](https://github.com/snowflakedb/gosnowflake/blob/master/LICENSE)
- github.com/spf13/cast [MIT License](https://github.com/spf13/cast/blob/master/LICENSE)
- github.com/spf13/pflag [BSD 3-Clause "New" or "Revised" License](https://github.com/spf13/pflag/blob/master/LICENSE)
- github.com/srebhan/cborquery [MIT License](https://github.com/srebhan/cborquery/blob/main/LICENSE)
- github.com/stoewer/go-strcase [MIT License](https://github.com/stoewer/go-strcase/blob/master/LICENSE)

5
go.mod
View File

@ -21,6 +21,7 @@ require (
github.com/ClickHouse/clickhouse-go v1.5.4
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/Masterminds/sprig v2.22.0+incompatible
github.com/Masterminds/sprig/v3 v3.2.3
github.com/Mellanox/rdmamap v1.1.0
github.com/Shopify/sarama v1.38.1
github.com/aerospike/aerospike-client-go/v5 v5.11.0
@ -238,6 +239,7 @@ require (
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/semver/v3 v3.2.0 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/alecthomas/participle v0.4.1 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
@ -333,7 +335,7 @@ require (
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/hashicorp/packer-plugin-sdk v0.3.2 // indirect
github.com/hashicorp/serf v0.10.1 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/huandu/xstrings v1.3.3 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
@ -415,6 +417,7 @@ require (
github.com/signalfx/com_signalfx_metrics_protobuf v0.0.3 // indirect
github.com/signalfx/gohistogram v0.0.0-20160107210732-1ccfd2ff5083 // indirect
github.com/signalfx/sapm-proto v0.12.0 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stoewer/go-strcase v1.2.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect

16
go.sum
View File

@ -141,10 +141,13 @@ github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJ
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60=
github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
github.com/Mellanox/rdmamap v1.1.0 h1:A/W1wAXw+6vm58f3VklrIylgV+eDJlPVIMaIKuxgUT4=
github.com/Mellanox/rdmamap v1.1.0/go.mod h1:fN+/V9lf10ABnDCwTaXRjeeWijLt2iVLETnK+sx/LY8=
github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
@ -519,8 +522,8 @@ github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq
github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@ -811,9 +814,10 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4=
github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
@ -1102,6 +1106,7 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@ -1116,6 +1121,7 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY=
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ=
@ -1390,6 +1396,8 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=

View File

@ -0,0 +1,7 @@
//go:build !custom || serializers || serializers.template
package all
import (
_ "github.com/influxdata/telegraf/plugins/serializers/template" // register plugin
)

View File

@ -0,0 +1,45 @@
# Template Serializer
The `template` output data format outputs metrics using an user defined go template.
[Sprig](http://masterminds.github.io/sprig/) helper functions are also available.
## Configuration
```toml
[[outputs.file]]
## Files to write to, "stdout" is a specially handled file.
files = ["stdout", "/tmp/metrics.out"]
## Data format to output.
## Each data format has its own unique set of configuration options, read
## more about them here:
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
data_format = "template"
## Go template which defines output format
template = '{{ .Tag "host" }} {{ .Field "available" }}'
## When used with output plugins that allow for batch serialisation
## the template for the entire batch can be defined
# use_batch_format = true # The 'file' plugin allows batch mode with this option
# batch_template = '''
{{range $metric := . -}}
{{$metric.Tag "host"}}: {{range $metric.Fields | keys | initial -}}
{{.}}={{get $metric.Fields .}}, {{end}}
{{- $metric.Fields|keys|last}}={{$metric.Fields|values|last}}
{{end -}}
'''
```
### Batch mode
When an output plugin emits multiple metrics in a batch fashion, by default the
template will just be repeated for each metric. If you would like to specifically
define how a batch should be formatted, you can use a `batch_template` instead.
In this mode, the context of the template (the 'dot') will be a slice of metrics.
```toml
batch_template = '''My batch metric names: {{range $index, $metric := . -}}
{{if $index}}, {{ end }}{{ $metric.Name }}
{{- end }}'''
```

View File

@ -0,0 +1,98 @@
package template
import (
"bytes"
"fmt"
"text/template"
"github.com/Masterminds/sprig/v3"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/plugins/serializers"
)
type Serializer struct {
Template string `toml:"template"`
BatchTemplate string `toml:"batch_template"`
Log telegraf.Logger `toml:"-"`
tmplMetric *template.Template
tmplBatch *template.Template
}
func (s *Serializer) Init() error {
// Setting defaults
var err error
s.tmplMetric, err = template.New("template").Funcs(sprig.TxtFuncMap()).Parse(s.Template)
if err != nil {
return fmt.Errorf("creating template failed: %w", err)
}
if s.BatchTemplate == "" {
s.BatchTemplate = fmt.Sprintf("{{range .}}%s{{end}}", s.Template)
}
s.tmplBatch, err = template.New("batch template").Funcs(sprig.TxtFuncMap()).Parse(s.BatchTemplate)
if err != nil {
return fmt.Errorf("creating batch template failed: %w", err)
}
return nil
}
func (s *Serializer) Serialize(metric telegraf.Metric) ([]byte, error) {
m, ok := metric.(telegraf.TemplateMetric)
if !ok {
s.Log.Errorf("metric of type %T is not a template metric", metric)
return nil, nil
}
var b bytes.Buffer
// The template was defined for one metric, just execute it
if s.Template != "" {
if err := s.tmplMetric.Execute(&b, &m); err != nil {
s.Log.Errorf("failed to execute template: %v", err)
return nil, nil
}
return b.Bytes(), nil
}
// The template was defined for a batch of metrics, so wrap the metric into a slice
if s.BatchTemplate != "" {
metrics := []telegraf.TemplateMetric{m}
if err := s.tmplBatch.Execute(&b, &metrics); err != nil {
s.Log.Errorf("failed to execute batch template: %v", err)
return nil, nil
}
return b.Bytes(), nil
}
// No template was defined
return nil, nil
}
func (s *Serializer) SerializeBatch(metrics []telegraf.Metric) ([]byte, error) {
newMetrics := make([]telegraf.TemplateMetric, 0, len(metrics))
for _, metric := range metrics {
m, ok := metric.(telegraf.TemplateMetric)
if !ok {
s.Log.Errorf("metric of type %T is not a template metric", metric)
return nil, nil
}
newMetrics = append(newMetrics, m)
}
var b bytes.Buffer
if err := s.tmplBatch.Execute(&b, &newMetrics); err != nil {
s.Log.Errorf("failed to execute batch template: %v", err)
return nil, nil
}
return b.Bytes(), nil
}
func init() {
serializers.Add("template",
func() serializers.Serializer {
return &Serializer{}
},
)
}

View File

@ -0,0 +1,182 @@
package template
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
)
func TestSerializer(t *testing.T) {
var tests = []struct {
name string
input telegraf.Metric
template string
output []byte
errReason string
}{
{
name: "name",
input: metric.New(
"cpu",
map[string]string{},
map[string]interface{}{},
time.Unix(100, 0),
),
template: "{{ .Name }}",
output: []byte("cpu"),
},
{
name: "time",
input: metric.New(
"cpu",
map[string]string{},
map[string]interface{}{},
time.Unix(100, 0),
),
template: "{{ .Time.Unix }}",
output: []byte("100"),
},
{
name: "specific field",
input: metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"x": 42.0,
"y": 43.0,
},
time.Unix(100, 0),
),
template: `{{ .Field "x" }}`,
output: []byte("42"),
},
{
name: "specific tag",
input: metric.New(
"cpu",
map[string]string{
"host": "localhost",
"cpu": "CPU0",
},
map[string]interface{}{},
time.Unix(100, 0),
),
template: `{{ .Tag "cpu" }}`,
output: []byte("CPU0"),
},
{
name: "all fields",
input: metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"x": 42.0,
"y": 43.0,
},
time.Unix(100, 0),
),
template: `{{ range $k, $v := .Fields }}{{$k}}={{$v}},{{end}}`,
output: []byte("x=42,y=43,"),
},
{
name: "all tags",
input: metric.New(
"cpu",
map[string]string{
"host": "localhost",
"cpu": "CPU0",
},
map[string]interface{}{},
time.Unix(100, 0),
),
template: `{{ range $k, $v := .Tags }}{{$k}}={{$v}},{{end}}`,
output: []byte("cpu=CPU0,host=localhost,"),
},
{
name: "string",
input: metric.New(
"cpu",
map[string]string{
"host": "localhost",
"cpu": "CPU0",
},
map[string]interface{}{
"x": 42.0,
"y": 43.0,
},
time.Unix(100, 0),
),
template: "{{ .String }}",
output: []byte("cpu map[cpu:CPU0 host:localhost] map[x:42 y:43] 100000000000"),
},
{
name: "complex",
input: metric.New(
"cpu",
map[string]string{
"tag1": "tag",
},
map[string]interface{}{
"value": 42.0,
},
time.Unix(0, 0),
),
template: `{{ .Name }} {{ range $k, $v := .Fields}}{{$k}}={{$v}}{{end}} {{ .Tag "tag1" }} {{.Time.UnixNano}} literal`,
output: []byte("cpu value=42 tag 0 literal"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
serializer := &Serializer{
Template: tt.template,
}
require.NoError(t, serializer.Init())
output, err := serializer.Serialize(tt.input)
if tt.errReason != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tt.errReason)
}
require.Equal(t, string(tt.output), string(output))
// Ensure we get the same output in batch mode
batchOutput, err := serializer.SerializeBatch([]telegraf.Metric{tt.input})
if tt.errReason != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tt.errReason)
}
require.Equal(t, string(tt.output), string(batchOutput))
})
}
}
func TestSerializeBatch(t *testing.T) {
m := metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"value": 42.0,
},
time.Unix(0, 0),
)
metrics := []telegraf.Metric{m, m}
s := &Serializer{BatchTemplate: `{{ range $index, $metric := . }}{{$index}}: {{$metric.Name}} {{$metric.Field "value"}}
{{end}}`}
require.NoError(t, s.Init())
buf, err := s.SerializeBatch(metrics)
require.NoError(t, err)
require.Equal(
t,
string(buf),
`0: cpu 42
1: cpu 42
`,
)
// A batch template should still work when serializing a single metric
singleBuf, err := s.Serialize(m)
require.NoError(t, err)
require.Equal(t, string(singleBuf), "0: cpu 42\n")
}