CL-Softwares
/
telegraf
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(outputs.stackdriver): Options to use official path and types (#13454)

This commit is contained in:
Joshua Powers 2023-06-26 07:30:11 -06:00 committed by GitHub
parent 56aac4f0e1
commit 45f994268c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 340 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
plugins/outputs/stackdriver/README.md
View File

@ -9,8 +9,11 @@ costs.
Requires `project` to specify where Stackdriver metrics will be delivered to.
Metrics are grouped by the `namespace` variable and metric key - eg:
`custom.googleapis.com/telegraf/system/load5`
By default, Metrics are grouped by the `namespace` variable and metric key -
eg: `custom.googleapis.com/telegraf/system/load5`. However, this is not the
best practice. Setting `metric_name_format = "official"` will produce a more
easily queried format of: `metric_type_prefix/[namespace_]name_key/kind`. If
the global namespace is not set, it is omitted as well.
[Resource type](https://cloud.google.com/monitoring/api/resources) is configured
by the `resource_type` variable (default `global`).
@ -36,12 +39,27 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
project = "erudite-bloom-151019"
## The namespace for the metric descriptor
## This is optional and users are encouraged to set the namespace as a
## resource label instead. If omitted it is not included in the metric name.
namespace = "telegraf"
## Metric Type Prefix
## The DNS name used with the metric type as a prefix.
# metric_type_prefix = "custom.googleapis.com"
## Metric Name Format
## Specifies the layout of the metric name, choose from:
## * path: 'metric_type_prefix_namespace_name_key'
## * official: 'metric_type_prefix/namespace_name_key/kind'
# metric_name_format = "path"
## Metric Data Type
## By default, telegraf will use whatever type the metric comes in as.
## However, for some use cases, forcing int64, may be preferred for values:
## * source: use whatever was passed in
## * double: preferred datatype to allow queries by PromQL.
# metric_data_type = "source"
## Custom resource type
# resource_type = "generic_node"

15
plugins/outputs/stackdriver/sample.conf
View File

@ -4,12 +4,27 @@
project = "erudite-bloom-151019"
## The namespace for the metric descriptor
## This is optional and users are encouraged to set the namespace as a
## resource label instead. If omitted it is not included in the metric name.
namespace = "telegraf"
## Metric Type Prefix
## The DNS name used with the metric type as a prefix.
# metric_type_prefix = "custom.googleapis.com"
## Metric Name Format
## Specifies the layout of the metric name, choose from:
## * path: 'metric_type_prefix_namespace_name_key'
## * official: 'metric_type_prefix/namespace_name_key/kind'
# metric_name_format = "path"
## Metric Data Type
## By default, telegraf will use whatever type the metric comes in as.
## However, for some use cases, forcing int64, may be preferred for values:
## * source: use whatever was passed in
## * double: preferred datatype to allow queries by PromQL.
# metric_data_type = "source"
## Custom resource type
# resource_type = "generic_node"

88
plugins/outputs/stackdriver/stackdriver.go
View File

@ -32,6 +32,8 @@ type Stackdriver struct {
ResourceType string `toml:"resource_type"`
ResourceLabels map[string]string `toml:"resource_labels"`
MetricTypePrefix string `toml:"metric_type_prefix"`
MetricNameFormat string `toml:"metric_name_format"`
MetricDataType string `toml:"metric_data_type"`
Log telegraf.Logger `toml:"-"`
client *monitoring.MetricClient
@ -62,6 +64,22 @@ func (s *Stackdriver) Init() error {
s.MetricTypePrefix = "custom.googleapis.com"
}
switch s.MetricNameFormat {
case "":
s.MetricNameFormat = "path"
case "path", "official":
default:
return fmt.Errorf("unrecognized metric name format: %s", s.MetricNameFormat)
}
switch s.MetricDataType {
case "":
s.MetricDataType = "source"
case "source", "double":
default:
return fmt.Errorf("unrecognized metric data type: %s", s.MetricDataType)
}
return nil
}
@ -76,7 +94,7 @@ func (s *Stackdriver) Connect() error {
}
if s.Namespace == "" {
return fmt.Errorf("namespace is a required field for stackdriver output")
s.Log.Warn("plugin-level namespace is empty")
}
if s.ResourceType == "" {
@ -175,7 +193,7 @@ func (s *Stackdriver) sendBatch(batch []telegraf.Metric) error {
buckets := make(timeSeriesBuckets)
for _, m := range batch {
for _, f := range m.FieldList() {
value, err := getStackdriverTypedValue(f.Value)
value, err := s.getStackdriverTypedValue(f.Value)
if err != nil {
s.Log.Errorf("Get type failed: %q", err)
continue
@ -208,7 +226,7 @@ func (s *Stackdriver) sendBatch(batch []telegraf.Metric) error {
// Prepare time series.
timeSeries := &monitoringpb.TimeSeries{
Metric: &metricpb.Metric{
Type: path.Join(s.MetricTypePrefix, s.Namespace, m.Name(), f.Key),
Type: s.generateMetricName(m, f.Key),
Labels: s.getStackdriverLabels(m.TagList()),
},
MetricKind: metricKind,
@ -222,6 +240,28 @@ func (s *Stackdriver) sendBatch(batch []telegraf.Metric) error {
}
buckets.Add(m, f, timeSeries)
// If the metric is untyped, it will end with unknown. We will also
// send another metric with the unknown:counter suffix. Google will
// do some heuristics to know which one to use for queries. This
// only occurs when using the official name format.
if s.MetricNameFormat == "official" && strings.HasSuffix(timeSeries.Metric.Type, "unknown") {
counterTimeSeries := &monitoringpb.TimeSeries{
Metric: &metricpb.Metric{
Type: s.generateMetricName(m, f.Key) + ":counter",
Labels: s.getStackdriverLabels(m.TagList()),
},
MetricKind: metricpb.MetricDescriptor_CUMULATIVE,
Resource: &monitoredrespb.MonitoredResource{
Type: s.ResourceType,
Labels: s.ResourceLabels,
},
Points: []*monitoringpb.Point{
dataPoint,
},
}
buckets.Add(m, f, counterTimeSeries)
}
}
}
@ -273,6 +313,33 @@ func (s *Stackdriver) sendBatch(batch []telegraf.Metric) error {
return nil
}
func (s *Stackdriver) generateMetricName(m telegraf.Metric, key string) string {
if s.MetricNameFormat == "path" {
return path.Join(s.MetricTypePrefix, s.Namespace, m.Name(), key)
}
name := m.Name() + "_" + key
if s.Namespace != "" {
name = s.Namespace + "_" + m.Name() + "_" + key
}
var kind string
switch m.Type() {
case telegraf.Gauge:
kind = "gauge"
case telegraf.Untyped:
kind = "unknown"
case telegraf.Counter:
kind = "counter"
case telegraf.Histogram:
kind = "histogram"
default:
kind = ""
}
return path.Join(s.MetricTypePrefix, name, kind)
}
func getStackdriverIntervalEndpoints(
kind metricpb.MetricDescriptor_MetricKind,
value *monitoringpb.TypedValue,
@ -328,7 +395,20 @@ func getStackdriverMetricKind(vt telegraf.ValueType) (metricpb.MetricDescriptor_
}
}
func getStackdriverTypedValue(value interface{}) (*monitoringpb.TypedValue, error) {
func (s *Stackdriver) getStackdriverTypedValue(value interface{}) (*monitoringpb.TypedValue, error) {
if s.MetricDataType == "double" {
v, err := internal.ToFloat64(value)
if err != nil {
return nil, err
}
return &monitoringpb.TypedValue{
Value: &monitoringpb.TypedValue_DoubleValue{
DoubleValue: v,
},
}, nil
}
switch v := value.(type) {
case uint64:
if v <= uint64(MaxInt) {

222
plugins/outputs/stackdriver/stackdriver_test.go
View File

@ -7,6 +7,7 @@ import (
"log"
"net"
"os"
"reflect"
"strings"
"testing"
"time"
@ -24,6 +25,7 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
"github.com/influxdata/telegraf/testutil"
)
@ -540,7 +542,7 @@ func TestGetStackdriverIntervalEndpoints(t *testing.T) {
for idx, m := range metrics {
for _, f := range m.FieldList() {
value, err := getStackdriverTypedValue(f.Value)
value, err := s.getStackdriverTypedValue(f.Value)
require.NoError(t, err)
require.NotNilf(t, value, "Got nil value for metric %q field %q", m, f)
@ -570,3 +572,221 @@ func TestGetStackdriverIntervalEndpoints(t *testing.T) {
}
}
}
func TestStackdriverTypedValuesSource(t *testing.T) {
s := &Stackdriver{
Namespace: "namespace",
MetricTypePrefix: "foo",
MetricDataType: "source",
}
tests := []struct {
name string
key string
expected string
value any
}{
{
name: "float",
key: "key",
expected: "*monitoringpb.TypedValue_DoubleValue",
value: float64(44.0),
},
{
name: "int64",
key: "key",
expected: "*monitoringpb.TypedValue_Int64Value",
value: int64(46),
},
{
name: "uint",
key: "key",
expected: "*monitoringpb.TypedValue_Int64Value",
value: uint64(46),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
typedValue, err := s.getStackdriverTypedValue(tt.value)
require.NoError(t, err)
require.Equal(t, tt.expected, reflect.TypeOf(typedValue.Value).String())
})
}
}
func TestStackdriverTypedValuesInt64(t *testing.T) {
s := &Stackdriver{
Namespace: "namespace",
MetricTypePrefix: "foo",
MetricDataType: "double",
}
tests := []struct {
name string
key string
expected string
value any
}{
{
name: "int",
key: "key",
expected: "*monitoringpb.TypedValue_DoubleValue",
value: 42,
},
{
name: "float",
key: "key",
expected: "*monitoringpb.TypedValue_DoubleValue",
value: float64(44.0),
},
{
name: "int64",
key: "key",
expected: "*monitoringpb.TypedValue_DoubleValue",
value: int64(46),
},
{
name: "uint",
key: "key",
expected: "*monitoringpb.TypedValue_DoubleValue",
value: uint64(46),
},
{
name: "numeric string",
key: "key",
expected: "*monitoringpb.TypedValue_DoubleValue",
value: "3.2",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
typedValue, err := s.getStackdriverTypedValue(tt.value)
require.NoError(t, err)
require.Equal(t, tt.expected, reflect.TypeOf(typedValue.Value).String())
})
}
}
func TestStackdriverMetricNamePath(t *testing.T) {
s := &Stackdriver{
Namespace: "namespace",
MetricTypePrefix: "foo",
MetricNameFormat: "path",
}
m := testutil.MustMetric("uptime",
map[string]string{
"foo": "bar",
},
map[string]interface{}{
"value": 42,
},
time.Now(),
telegraf.Gauge,
)
require.Equal(t, "foo/namespace/uptime/key", s.generateMetricName(m, "key"))
}
func TestStackdriverMetricNameOfficial(t *testing.T) {
s := &Stackdriver{
Namespace: "namespace",
MetricTypePrefix: "prometheus.googleapis.com",
MetricNameFormat: "official",
}
tests := []struct {
name string
key string
expected string
metric telegraf.Metric
}{
{
name: "gauge",
key: "key",
expected: "prometheus.googleapis.com/namespace_uptime_key/gauge",
metric: metric.New(
"uptime",
map[string]string{},
map[string]interface{}{
"value": 42,
},
time.Now(),
telegraf.Gauge,
),
},
{
name: "untyped",
key: "key",
expected: "prometheus.googleapis.com/namespace_uptime_key/unknown",
metric: metric.New(
"uptime",
map[string]string{},
map[string]interface{}{
"value": 42,
},
time.Now(),
telegraf.Untyped,
),
},
{
name: "histogram",
key: "key",
expected: "prometheus.googleapis.com/namespace_uptime_key/histogram",
metric: metric.New(
"uptime",
map[string]string{},
map[string]interface{}{
"value": 42,
},
time.Now(),
telegraf.Histogram,
),
},
{
name: "counter",
key: "key",
expected: "prometheus.googleapis.com/namespace_uptime_key/counter",
metric: metric.New(
"uptime",
map[string]string{},
map[string]interface{}{
"value": 42,
},
time.Now(),
telegraf.Counter,
),
},
{
name: "summary",
key: "key",
expected: "prometheus.googleapis.com/namespace_uptime_key",
metric: metric.New(
"uptime",
map[string]string{},
map[string]interface{}{
"value": 42,
},
time.Now(),
telegraf.Summary,
),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
require.Equal(t, tt.expected, s.generateMetricName(tt.metric, tt.key))
})
}
}
func TestStackdriverValueInvalid(t *testing.T) {
s := &Stackdriver{
MetricDataType: "foobar",
}
require.Error(t, s.Init())
}
func TestStackdriverMetricNameInvalid(t *testing.T) {
s := &Stackdriver{
MetricNameFormat: "foobar",
}
require.Error(t, s.Init())
}