CL-Softwares
/
telegraf
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added force_document_id option to ES output enable resend data and avoiding duplicated ES documents, fix #7891 (#8019)

This commit is contained in:
Toni Moreno 2020-09-08 22:34:27 +02:00 committed by GitHub
parent 8c28661ca7
commit 9a06ac1d6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 31 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
etc/telegraf.conf
View File

@ -512,7 +512,9 @@
# template_name = "telegraf"
# ## Set to true if you want telegraf to overwrite an existing template
# overwrite_template = false
# ## If set to true a unique ID hash will be sent as sha256(concat(timestamp,measurement,series-hash)) string
# ## it will enable data resend and update metric points avoiding duplicated metrics with diferent id's
# force_document_id = false
# # Send metrics to command as input over stdin
# [[outputs.exec]]

4
plugins/outputs/elasticsearch/README.md
View File

@ -196,6 +196,9 @@ This plugin will format the events in the following way:
template_name = "telegraf"
## Set to true if you want telegraf to overwrite an existing template
overwrite_template = false
## If set to true a unique ID hash will be sent as sha256(concat(timestamp,measurement,series-hash)) string
## it will enable data resend and update metric points avoiding duplicated metrics with diferent id's
force_document_id = false
```
#### Permissions
@ -232,6 +235,7 @@ Additionally, you can specify dynamic index names by using tags with the notatio
* `manage_template`: Set to true if you want telegraf to manage its index template. If enabled it will create a recommended index template for telegraf indexes.
* `template_name`: The template name used for telegraf indexes.
* `overwrite_template`: Set to true if you want telegraf to overwrite an existing template.
* `force_document_id`: Set to true will compute a unique hash from as sha256(concat(timestamp,measurement,series-hash)),enables resend or update data withoud ES duplicated documents.
### Known issues

24
plugins/outputs/elasticsearch/elasticsearch.go
View File

@ -11,6 +11,8 @@ import (
"text/template"
"time"
"crypto/sha256"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/common/tls"
@ -31,6 +33,7 @@ type Elasticsearch struct {
ManageTemplate bool
TemplateName string
OverwriteTemplate bool
ForceDocumentId bool
MajorReleaseNumber int
tls.ClientConfig
@ -86,6 +89,9 @@ var sampleConfig = `
template_name = "telegraf"
## Set to true if you want telegraf to overwrite an existing template
overwrite_template = false
## If set to true a unique ID hash will be sent as sha256(concat(timestamp,measurement,series-hash)) string
## it will enable data resend and update metric points avoiding duplicated metrics with diferent id's
force_document_id = false
`
const telegrafTemplate = `
@ -242,6 +248,19 @@ func (a *Elasticsearch) Connect() error {
return nil
}
// GetPointID generates a unique ID for a Metric Point
func GetPointID(m telegraf.Metric) string {
var buffer bytes.Buffer
//Timestamp(ns),measurement name and Series Hash for compute the final SHA256 based hash ID
buffer.WriteString(strconv.FormatInt(m.Time().Local().UnixNano(), 10))
buffer.WriteString(m.Name())
buffer.WriteString(strconv.FormatUint(m.HashID(), 10))
return fmt.Sprintf("%x", sha256.Sum256(buffer.Bytes()))
}
func (a *Elasticsearch) Write(metrics []telegraf.Metric) error {
if len(metrics) == 0 {
return nil
@ -265,6 +284,11 @@ func (a *Elasticsearch) Write(metrics []telegraf.Metric) error {
br := elastic.NewBulkIndexRequest().Index(indexName).Doc(m)
if a.ForceDocumentId {
id := GetPointID(metric)
br.Id(id)
}
if a.MajorReleaseNumber <= 6 {
br.Type("metrics")
}