feat: Add RedisTimeSeries plugin (#11054)
This commit is contained in:
Guy Korland
GitHub
parent
f29f6cf6a1
commit
3c5d71b7cd
|
|
@ -21,96 +21,96 @@
|
|||
## Environment variables can be used as tags, and throughout the config file
|
||||
# user = "$USER"
|
||||
|
||||
# Configuration for telegraf agent
|
||||
[agent]
|
||||
## Default data collection interval for all inputs
|
||||
interval = "10s"
|
||||
## Rounds collection interval to 'interval'
|
||||
## ie, if interval="10s" then always collect on :00, :10, :20, etc.
|
||||
round_interval = true
|
||||
|
||||
## Telegraf will send metrics to outputs in batches of at most
|
||||
## metric_batch_size metrics.
|
||||
## This controls the size of writes that Telegraf sends to output plugins.
|
||||
metric_batch_size = 1000
|
||||
|
||||
## Maximum number of unwritten metrics per output. Increasing this value
|
||||
## allows for longer periods of output downtime without dropping metrics at the
|
||||
## cost of higher maximum memory usage.
|
||||
metric_buffer_limit = 10000
|
||||
|
||||
## Collection jitter is used to jitter the collection by a random amount.
|
||||
## Each plugin will sleep for a random time within jitter before collecting.
|
||||
## This can be used to avoid many plugins querying things like sysfs at the
|
||||
## same time, which can have a measurable effect on the system.
|
||||
collection_jitter = "0s"
|
||||
|
||||
## Collection offset is used to shift the collection by the given amount.
|
||||
## This can be be used to avoid many plugins querying constraint devices
|
||||
## at the same time by manually scheduling them in time.
|
||||
# collection_offset = "0s"
|
||||
|
||||
## Default flushing interval for all outputs. Maximum flush_interval will be
|
||||
## flush_interval + flush_jitter
|
||||
flush_interval = "10s"
|
||||
## Jitter the flush interval by a random amount. This is primarily to avoid
|
||||
## large write spikes for users running a large number of telegraf instances.
|
||||
## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
|
||||
flush_jitter = "0s"
|
||||
|
||||
## Collected metrics are rounded to the precision specified. Precision is
|
||||
## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s).
|
||||
## Valid time units are "ns", "us" (or "µs"), "ms", "s".
|
||||
##
|
||||
## By default or when set to "0s", precision will be set to the same
|
||||
## timestamp order as the collection interval, with the maximum being 1s:
|
||||
## ie, when interval = "10s", precision will be "1s"
|
||||
## when interval = "250ms", precision will be "1ms"
|
||||
##
|
||||
## Precision will NOT be used for service inputs. It is up to each individual
|
||||
## service input to set the timestamp at the appropriate precision.
|
||||
precision = "0s"
|
||||
|
||||
## Log at debug level.
|
||||
# debug = false
|
||||
## Log only error level messages.
|
||||
# quiet = false
|
||||
|
||||
## Log target controls the destination for logs and can be one of "file",
|
||||
## "stderr" or, on Windows, "eventlog". When set to "file", the output file
|
||||
## is determined by the "logfile" setting.
|
||||
# logtarget = "file"
|
||||
|
||||
## Name of the file to be logged to when using the "file" logtarget. If set to
|
||||
## the empty string then logs are written to stderr.
|
||||
# logfile = ""
|
||||
|
||||
## The logfile will be rotated after the time interval specified. When set
|
||||
## to 0 no time based rotation is performed. Logs are rotated only when
|
||||
## written to, if there is no log activity rotation may be delayed.
|
||||
# logfile_rotation_interval = "0h"
|
||||
|
||||
## The logfile will be rotated when it becomes larger than the specified
|
||||
## size. When set to 0 no size based rotation is performed.
|
||||
# logfile_rotation_max_size = "0MB"
|
||||
|
||||
## Maximum number of rotated archives to keep, any older logs are deleted.
|
||||
## If set to -1, no archives are removed.
|
||||
# logfile_rotation_max_archives = 5
|
||||
|
||||
## Pick a timezone to use when logging or type 'local' for local time.
|
||||
## Example: America/Chicago
|
||||
# log_with_timezone = ""
|
||||
|
||||
## Override default hostname, if empty use os.Hostname()
|
||||
hostname = ""
|
||||
## If set to true, do no set the "host" tag in the telegraf agent.
|
||||
omit_hostname = false
|
||||
|
||||
## Method of translating SNMP objects. Can be "netsnmp" which
|
||||
## translates by calling external programs snmptranslate and snmptable,
|
||||
## or "gosmi" which translates using the built-in gosmi library.
|
||||
# snmp_translator = "netsnmp"
|
||||
# Configuration for telegraf agent
|
||||
[agent]
|
||||
## Default data collection interval for all inputs
|
||||
interval = "10s"
|
||||
## Rounds collection interval to 'interval'
|
||||
## ie, if interval="10s" then always collect on :00, :10, :20, etc.
|
||||
round_interval = true
|
||||
|
||||
## Telegraf will send metrics to outputs in batches of at most
|
||||
## metric_batch_size metrics.
|
||||
## This controls the size of writes that Telegraf sends to output plugins.
|
||||
metric_batch_size = 1000
|
||||
|
||||
## Maximum number of unwritten metrics per output. Increasing this value
|
||||
## allows for longer periods of output downtime without dropping metrics at the
|
||||
## cost of higher maximum memory usage.
|
||||
metric_buffer_limit = 10000
|
||||
|
||||
## Collection jitter is used to jitter the collection by a random amount.
|
||||
## Each plugin will sleep for a random time within jitter before collecting.
|
||||
## This can be used to avoid many plugins querying things like sysfs at the
|
||||
## same time, which can have a measurable effect on the system.
|
||||
collection_jitter = "0s"
|
||||
|
||||
## Collection offset is used to shift the collection by the given amount.
|
||||
## This can be be used to avoid many plugins querying constraint devices
|
||||
## at the same time by manually scheduling them in time.
|
||||
# collection_offset = "0s"
|
||||
|
||||
## Default flushing interval for all outputs. Maximum flush_interval will be
|
||||
## flush_interval + flush_jitter
|
||||
flush_interval = "10s"
|
||||
## Jitter the flush interval by a random amount. This is primarily to avoid
|
||||
## large write spikes for users running a large number of telegraf instances.
|
||||
## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
|
||||
flush_jitter = "0s"
|
||||
|
||||
## Collected metrics are rounded to the precision specified. Precision is
|
||||
## specified as an interval with an integer + unit (e.g. 0s, 10ms, 2us, 4s).
|
||||
## Valid time units are "ns", "us" (or "µs"), "ms", "s".
|
||||
##
|
||||
## By default or when set to "0s", precision will be set to the same
|
||||
## timestamp order as the collection interval, with the maximum being 1s:
|
||||
## ie, when interval = "10s", precision will be "1s"
|
||||
## when interval = "250ms", precision will be "1ms"
|
||||
##
|
||||
## Precision will NOT be used for service inputs. It is up to each individual
|
||||
## service input to set the timestamp at the appropriate precision.
|
||||
precision = "0s"
|
||||
|
||||
## Log at debug level.
|
||||
# debug = false
|
||||
## Log only error level messages.
|
||||
# quiet = false
|
||||
|
||||
## Log target controls the destination for logs and can be one of "file",
|
||||
## "stderr" or, on Windows, "eventlog". When set to "file", the output file
|
||||
## is determined by the "logfile" setting.
|
||||
# logtarget = "file"
|
||||
|
||||
## Name of the file to be logged to when using the "file" logtarget. If set to
|
||||
## the empty string then logs are written to stderr.
|
||||
# logfile = ""
|
||||
|
||||
## The logfile will be rotated after the time interval specified. When set
|
||||
## to 0 no time based rotation is performed. Logs are rotated only when
|
||||
## written to, if there is no log activity rotation may be delayed.
|
||||
# logfile_rotation_interval = "0h"
|
||||
|
||||
## The logfile will be rotated when it becomes larger than the specified
|
||||
## size. When set to 0 no size based rotation is performed.
|
||||
# logfile_rotation_max_size = "0MB"
|
||||
|
||||
## Maximum number of rotated archives to keep, any older logs are deleted.
|
||||
## If set to -1, no archives are removed.
|
||||
# logfile_rotation_max_archives = 5
|
||||
|
||||
## Pick a timezone to use when logging or type 'local' for local time.
|
||||
## Example: America/Chicago
|
||||
# log_with_timezone = ""
|
||||
|
||||
## Override default hostname, if empty use os.Hostname()
|
||||
hostname = ""
|
||||
## If set to true, do no set the "host" tag in the telegraf agent.
|
||||
omit_hostname = false
|
||||
|
||||
## Method of translating SNMP objects. Can be "netsnmp" which
|
||||
## translates by calling external programs snmptranslate and snmptable,
|
||||
## or "gosmi" which translates using the built-in gosmi library.
|
||||
# snmp_translator = "netsnmp"
|
||||
|
||||
###############################################################################
|
||||
# OUTPUT PLUGINS #
|
||||
|
|
@ -1677,6 +1677,23 @@
|
|||
# ## Export metric collection time.
|
||||
# # export_timestamp = false
|
||||
|
||||
# # Configuration for the RedisTimeSeries server to send metrics to
|
||||
# [[outputs.redistimeseries]]
|
||||
# ## The address of the RedisTimeSeries server.
|
||||
# address = "127.0.0.1:6379"
|
||||
# ## password to login Redis
|
||||
# password = ""
|
||||
#
|
||||
# ## username (optional)
|
||||
# username = ""
|
||||
# # redis database number (optional, must be an integer)
|
||||
# database = 0
|
||||
#
|
||||
# ## optional TLS configurations
|
||||
# tls_ca = "/etc/telegraf/ca.pem
|
||||
# tls_cert = "/etc/telegraf/cert.pem"
|
||||
# tls_key = "/etc/telegraf/key.pem"
|
||||
# insecure_skip_verify = false
|
||||
|
||||
# # Configuration for Riemann to send metrics to
|
||||
# [[outputs.riemann]]
|
||||
|
|
@ -6216,9 +6233,9 @@
|
|||
|
||||
|
||||
# # A plugin to collect stats from Opensmtpd - a validating, recursive, and caching DNS resolver
|
||||
# [[inputs.opensmtpd]]
|
||||
# ## If running as a restricted user you can prepend sudo for additional access:
|
||||
# #use_sudo = false
|
||||
# [[inputs.opensmtpd]]
|
||||
# ## If running as a restricted user you can prepend sudo for additional access:
|
||||
# #use_sudo = false
|
||||
#
|
||||
# ## The default location of the smtpctl binary can be overridden with:
|
||||
# binary = "/usr/sbin/smtpctl"
|
||||
|
|
@ -8403,35 +8420,35 @@
|
|||
|
||||
# # Read Intel RDT metrics
|
||||
# [[inputs.intel_rdt]]
|
||||
# ## Optionally set sampling interval to Nx100ms.
|
||||
# ## This value is propagated to pqos tool. Interval format is defined by pqos itself.
|
||||
# ## If not provided or provided 0, will be set to 10 = 10x100ms = 1s.
|
||||
# # sampling_interval = "10"
|
||||
# ## Optionally set sampling interval to Nx100ms.
|
||||
# ## This value is propagated to pqos tool. Interval format is defined by pqos itself.
|
||||
# ## If not provided or provided 0, will be set to 10 = 10x100ms = 1s.
|
||||
# # sampling_interval = "10"
|
||||
#
|
||||
# ## Optionally specify the path to pqos executable.
|
||||
# ## If not provided, auto discovery will be performed.
|
||||
# # pqos_path = "/usr/local/bin/pqos"
|
||||
# ## Optionally specify the path to pqos executable.
|
||||
# ## If not provided, auto discovery will be performed.
|
||||
# # pqos_path = "/usr/local/bin/pqos"
|
||||
#
|
||||
# ## Optionally specify if IPC and LLC_Misses metrics shouldn't be propagated.
|
||||
# ## If not provided, default value is false.
|
||||
# # shortened_metrics = false
|
||||
# ## Optionally specify if IPC and LLC_Misses metrics shouldn't be propagated.
|
||||
# ## If not provided, default value is false.
|
||||
# # shortened_metrics = false
|
||||
#
|
||||
# ## Specify the list of groups of CPU core(s) to be provided as pqos input.
|
||||
# ## Mandatory if processes aren't set and forbidden if processes are specified.
|
||||
# ## e.g. ["0-3", "4,5,6"] or ["1-3,4"]
|
||||
# # cores = ["0-3"]
|
||||
# ## Specify the list of groups of CPU core(s) to be provided as pqos input.
|
||||
# ## Mandatory if processes aren't set and forbidden if processes are specified.
|
||||
# ## e.g. ["0-3", "4,5,6"] or ["1-3,4"]
|
||||
# # cores = ["0-3"]
|
||||
#
|
||||
# ## Specify the list of processes for which Metrics will be collected.
|
||||
# ## Mandatory if cores aren't set and forbidden if cores are specified.
|
||||
# ## e.g. ["qemu", "pmd"]
|
||||
# # processes = ["process"]
|
||||
# ## Specify the list of processes for which Metrics will be collected.
|
||||
# ## Mandatory if cores aren't set and forbidden if cores are specified.
|
||||
# ## e.g. ["qemu", "pmd"]
|
||||
# # processes = ["process"]
|
||||
#
|
||||
# ## Specify if the pqos process should be called with sudo.
|
||||
# ## Mandatory if the telegraf process does not run as root.
|
||||
# # use_sudo = false
|
||||
# ## Specify if the pqos process should be called with sudo.
|
||||
# ## Mandatory if the telegraf process does not run as root.
|
||||
# # use_sudo = false
|
||||
|
||||
|
||||
# # Subscribe and receive OpenConfig Telemetry data using JTI
|
||||
# # Read JTI OpenConfig Telemetry from listed sensors
|
||||
# [[inputs.jti_openconfig_telemetry]]
|
||||
# ## List of device addresses to collect telemetry from
|
||||
# servers = ["localhost:1883"]
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -60,7 +60,6 @@ require (
|
|||
github.com/go-ldap/ldap/v3 v3.4.1
|
||||
github.com/go-logfmt/logfmt v0.5.1
|
||||
github.com/go-ping/ping v0.0.0-20210201095549-52eed920f98c
|
||||
github.com/go-redis/redis v6.15.9+incompatible
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/go-stomp/stomp v2.1.4+incompatible
|
||||
|
|
@ -174,6 +173,8 @@ require (
|
|||
modernc.org/sqlite v1.17.3
|
||||
)
|
||||
|
||||
require github.com/go-redis/redis/v7 v7.4.1
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.102.1 // indirect
|
||||
cloud.google.com/go/compute v1.6.1 // indirect
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -918,6 +918,8 @@ github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3yg
|
|||
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
|
||||
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
|
||||
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
|
||||
github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI=
|
||||
github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
|
||||
github.com/go-redis/redis/v8 v8.0.0-beta.6/go.mod h1:g79Vpae8JMzg5qjk8BiwU9tK+HmU3iDVyS4UAJLFycI=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ The Redis input plugin gathers metrics from one or many Redis servers.
|
|||
# # Can be "string", "integer", or "float"
|
||||
# type = "string"
|
||||
|
||||
## specify server password
|
||||
## specify server connection parameters
|
||||
# username = "myuser"
|
||||
# password = "s#cr@t%"
|
||||
|
||||
## specify username for ACL auth (Redis 6.0+)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import (
|
|||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/go-redis/redis"
|
||||
"github.com/go-redis/redis/v7"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/plugins/common/tls"
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ import (
|
|||
_ "github.com/influxdata/telegraf/plugins/outputs/opentelemetry"
|
||||
_ "github.com/influxdata/telegraf/plugins/outputs/opentsdb"
|
||||
_ "github.com/influxdata/telegraf/plugins/outputs/prometheus_client"
|
||||
_ "github.com/influxdata/telegraf/plugins/outputs/redistimeseries"
|
||||
_ "github.com/influxdata/telegraf/plugins/outputs/riemann"
|
||||
_ "github.com/influxdata/telegraf/plugins/outputs/riemann_legacy"
|
||||
_ "github.com/influxdata/telegraf/plugins/outputs/sensu"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
# RedisTimeSeries Producer Output Plugin
|
||||
|
||||
The RedisTimeSeries output plugin writes metrics to the RedisTimeSeries server.
|
||||
|
||||
## Configuration
|
||||
|
||||
```toml
|
||||
[[outputs.redistimeseries]]
|
||||
## The address of the RedisTimeSeries server.
|
||||
address = "127.0.0.1:6379"
|
||||
## password to login Redis
|
||||
password = ""
|
||||
|
||||
## username (optional)
|
||||
# username = ""
|
||||
# redis database number (optional, must be an integer)
|
||||
# database = 0
|
||||
|
||||
## optional TLS configurations
|
||||
# tls_ca = "/etc/telegraf/ca.pem"
|
||||
# tls_cert = "/etc/telegraf/cert.pem"
|
||||
# tls_key = "/etc/telegraf/key.pem"
|
||||
# insecure_skip_verify = false
|
||||
```
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
package redistimeseries
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/go-redis/redis/v7"
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/plugins/common/tls"
|
||||
"github.com/influxdata/telegraf/plugins/outputs"
|
||||
)
|
||||
|
||||
const sampleConfig = `
|
||||
## The address of the RedisTimeSeries server.
|
||||
address = "127.0.0.1:6379"
|
||||
|
||||
## Redis ACL credentials
|
||||
# username = ""
|
||||
# password = ""
|
||||
# database = 0
|
||||
|
||||
## Optional TLS Config
|
||||
# tls_ca = "/etc/telegraf/ca.pem"
|
||||
# tls_cert = "/etc/telegraf/cert.pem"
|
||||
# tls_key = "/etc/telegraf/key.pem"
|
||||
# insecure_skip_verify = false
|
||||
`
|
||||
|
||||
type RedisTimeSeries struct {
|
||||
Address string `toml:"address"`
|
||||
Username string `toml:"username"`
|
||||
Password string `toml:"password"`
|
||||
Database int `toml:"database"`
|
||||
Log telegraf.Logger `toml:"-"`
|
||||
tls.ClientConfig
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
func (r *RedisTimeSeries) Connect() error {
|
||||
if r.Address == "" {
|
||||
return errors.New("redis address must be specified")
|
||||
}
|
||||
r.client = redis.NewClient(&redis.Options{
|
||||
Addr: r.Address,
|
||||
Password: r.Password,
|
||||
Username: r.Username,
|
||||
DB: r.Database,
|
||||
})
|
||||
return r.client.Ping().Err()
|
||||
}
|
||||
|
||||
func (r *RedisTimeSeries) Close() error {
|
||||
return r.client.Close()
|
||||
}
|
||||
|
||||
func (r *RedisTimeSeries) Description() string {
|
||||
return "Plugin for sending metrics to RedisTimeSeries"
|
||||
}
|
||||
|
||||
func (r *RedisTimeSeries) SampleConfig() string {
|
||||
return sampleConfig
|
||||
}
|
||||
func (r *RedisTimeSeries) Write(metrics []telegraf.Metric) error {
|
||||
for _, m := range metrics {
|
||||
now := m.Time().UnixNano() / 1000000 // in milliseconds
|
||||
name := m.Name()
|
||||
|
||||
var tags []interface{}
|
||||
for k, v := range m.Tags() {
|
||||
tags = append(tags, k, v)
|
||||
}
|
||||
|
||||
for fieldName, value := range m.Fields() {
|
||||
key := name + "_" + fieldName
|
||||
|
||||
addSlice := []interface{}{"TS.ADD", key, now, value}
|
||||
addSlice = append(addSlice, tags...)
|
||||
|
||||
if err := r.client.Do(addSlice...).Err(); err != nil {
|
||||
return fmt.Errorf("adding sample failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
outputs.Add("redistimeseries", func() telegraf.Output {
|
||||
return &RedisTimeSeries{}
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package redistimeseries
|
||||
|
||||
import (
|
||||
"github.com/influxdata/telegraf/testutil"
|
||||
"github.com/stretchr/testify/require"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestConnectAndWrite(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
address := testutil.GetLocalHost() + ":6379"
|
||||
redis := &RedisTimeSeries{
|
||||
Address: address,
|
||||
}
|
||||
|
||||
// Verify that we can connect to the RedisTimeSeries server
|
||||
err := redis.Connect()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify that we can successfully write data to the RedisTimeSeries server
|
||||
err = redis.Write(testutil.MockMetrics())
|
||||
require.NoError(t, err)
|
||||
}
|
||||
Loading…
Reference in New Issue