diff --git a/plugins/processors/starlark/README.md b/plugins/processors/starlark/README.md index 7cc02ec26..dafdd85a1 100644 --- a/plugins/processors/starlark/README.md +++ b/plugins/processors/starlark/README.md @@ -9,7 +9,7 @@ Existing Python code is unlikely to work unmodified. The execution environment is sandboxed, and it is not possible to do I/O operations such as reading from files or sockets. -The Starlark [specification][] has details about the syntax and available +The **[Starlark specification][]** has details about the syntax and available functions. Telegraf minimum version: Telegraf 1.15.0 @@ -44,7 +44,7 @@ def apply(metric): ``` For a list of available types and functions that can be used in the code, see -the Starlark [specification][]. +the [Starlark specification][]. In addition to these, the following InfluxDB-specific types and functions are exposed to the script. @@ -149,14 +149,17 @@ Attempting to modify the global scope will fail with an error. ### Examples -- [ratio](/plugins/processors/starlark/testdata/ratio.star) -- [rename](/plugins/processors/starlark/testdata/rename.star) -- [scale](/plugins/processors/starlark/testdata/scale.star) -- [number logic](/plugins/processors/starlark/testdata/number_logic.star) -- [pivot](/plugins/processors/starlark/testdata/pivot.star) +- [ratio](/plugins/processors/starlark/testdata/ratio.star) - Compute the ratio of two integer fields +- [rename](/plugins/processors/starlark/testdata/rename.star) - Rename tags or fields using a name mapping. +- [scale](/plugins/processors/starlark/testdata/scale.star) - Multiply any field by a number +- [number logic](/plugins/processors/starlark/testdata/number_logic.star) - transform a numerical value to another numerical value +- [pivot](/plugins/processors/starlark/testdata/pivot.star) - Pivots a key's value to be the key for another key. +- [value filter](plugins/processors/starlark/testdata/value_filter.star) - remove a metric based on a field value. -Open a [PR](https://github.com/influxdata/telegraf/compare) to add any other useful Starlark examples. +[All examples](/plugins/processors/starlark/testdata) are in the testdata folder. -[specification]: https://github.com/google/starlark-go/blob/master/doc/spec.md +Open a Pull Request to add any other useful Starlark examples. + +[Starlark specification]: https://github.com/google/starlark-go/blob/master/doc/spec.md [string]: https://github.com/google/starlark-go/blob/master/doc/spec.md#strings [dict]: https://github.com/google/starlark-go/blob/master/doc/spec.md#dictionaries diff --git a/plugins/processors/starlark/starlark_test.go b/plugins/processors/starlark/starlark_test.go index 1cdd10db0..f80676a73 100644 --- a/plugins/processors/starlark/starlark_test.go +++ b/plugins/processors/starlark/starlark_test.go @@ -1,10 +1,16 @@ package starlark import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" "testing" "time" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/parsers" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" ) @@ -2787,3 +2793,72 @@ def apply(metric): }) } } +func TestAllScriptTestData(t *testing.T) { + // can be run from multiple folders + paths := []string{"testdata", "plugins/processors/starlark/testdata"} + for _, testdataPath := range paths { + filepath.Walk(testdataPath, func(path string, info os.FileInfo, err error) error { + if info == nil || info.IsDir() { + return nil + } + fn := path + t.Run(fn, func(t *testing.T) { + b, err := ioutil.ReadFile(fn) + require.NoError(t, err) + lines := strings.Split(string(b), "\n") + inputMetrics := parseMetricsFrom(t, lines, "Example Input:") + outputMetrics := parseMetricsFrom(t, lines, "Example Output:") + plugin := &Starlark{ + Script: fn, + Log: testutil.Logger{}, + } + require.NoError(t, plugin.Init()) + + acc := &testutil.Accumulator{} + + err = plugin.Start(acc) + require.NoError(t, err) + + for _, m := range inputMetrics { + err = plugin.Add(m, acc) + require.NoError(t, err) + } + + err = plugin.Stop() + require.NoError(t, err) + + testutil.RequireMetricsEqual(t, outputMetrics, acc.GetTelegrafMetrics(), testutil.SortMetrics(), testutil.IgnoreTime()) + }) + return nil + }) + } +} + +var parser, _ = parsers.NewInfluxParser() // literally never returns errors. + +// parses metric lines out of line protocol following a header, with a trailing blank line +func parseMetricsFrom(t *testing.T, lines []string, header string) (metrics []telegraf.Metric) { + require.NotZero(t, len(lines), "Expected some lines to parse from .star file, found none") + startIdx := -1 + endIdx := len(lines) + for i := range lines { + if strings.TrimLeft(lines[i], "# ") == header { + startIdx = i + 1 + break + } + } + require.NotEqual(t, -1, startIdx, fmt.Sprintf("Header %q must exist in file", header)) + for i := startIdx; i < len(lines); i++ { + line := strings.TrimLeft(lines[i], "# ") + if line == "" || line == "'''" { + endIdx = i + break + } + } + for i := startIdx; i < endIdx; i++ { + m, err := parser.ParseLine(strings.TrimLeft(lines[i], "# ")) + require.NoError(t, err, fmt.Sprintf("Expected to be able to parse %q metric, but found error", header)) + metrics = append(metrics, m) + } + return metrics +} diff --git a/plugins/processors/starlark/testdata/number_logic.star b/plugins/processors/starlark/testdata/number_logic.star index 719b61a9f..fced8c76d 100644 --- a/plugins/processors/starlark/testdata/number_logic.star +++ b/plugins/processors/starlark/testdata/number_logic.star @@ -2,9 +2,10 @@ # Example: Set any 'status' field between 1 and 6 to a value of 0 # # Example Input: -# lb, http_method=GET status=5 1465839830100400201 +# lb,http_method=GET status=5i 1465839830100400201 +# # Example Output: -# lb, http_method=GET status=0 1465839830100400201 +# lb,http_method=GET status=0i 1465839830100400201 def apply(metric): diff --git a/plugins/processors/starlark/testdata/pivot.star b/plugins/processors/starlark/testdata/pivot.star index d1a380cef..f32ebf45d 100644 --- a/plugins/processors/starlark/testdata/pivot.star +++ b/plugins/processors/starlark/testdata/pivot.star @@ -3,15 +3,15 @@ Pivots a key's value to be the key for another key. In this example it pivots the value of key `sensor` to be the key of the value in key `value` -Input: -temperature sensor=001A0,value=111.48 +Example Input: +temperature sensor="001A0",value=111.48 -Output: +Example Output: temperature 001A0=111.48 ''' - - def apply(metric): - metric.fields[str(metric.fields['sensor'])] = metric.fields['value'] - metric.fields.pop('value',None) - metric.fields.pop('sensor',None) - return metric + +def apply(metric): + metric.fields[str(metric.fields['sensor'])] = metric.fields['value'] + metric.fields.pop('value',None) + metric.fields.pop('sensor',None) + return metric diff --git a/plugins/processors/starlark/testdata/ratio.star b/plugins/processors/starlark/testdata/ratio.star index ee4f5b20a..60dcedaf5 100644 --- a/plugins/processors/starlark/testdata/ratio.star +++ b/plugins/processors/starlark/testdata/ratio.star @@ -3,9 +3,10 @@ # Example: A new field 'usage' from an existing fields 'used' and 'total' # # Example Input: -# memory, host=hostname used=11038756864.4948,total=17179869184.1221 1597255082000000000 +# memory,host=hostname used=11038756864.4948,total=17179869184.1221 1597255082000000000 +# # Example Output: -# memory, host=hostname used=11038756864.4948,total=17179869184.1221,usage=64.254021647 1597255082000000000 +# memory,host=hostname used=11038756864.4948,total=17179869184.1221,usage=64.25402164701573 1597255082000000000 def apply(metric): used = float(metric.fields['used']) diff --git a/plugins/processors/starlark/testdata/rename.star b/plugins/processors/starlark/testdata/rename.star index 5c5be1120..cf5d118dd 100644 --- a/plugins/processors/starlark/testdata/rename.star +++ b/plugins/processors/starlark/testdata/rename.star @@ -1,9 +1,10 @@ # Rename any tags using the mapping in the renames dict. # # Example Input: -# measurement, host=hostname lower=0,upper=100 1597255410000000000 +# measurement,host=hostname lower=0,upper=100 1597255410000000000 +# # Example Output: -# measurement, host=hostname min=0,max=100 1597255410000000000 +# measurement,host=hostname min=0,max=100 1597255410000000000 renames = { 'lower': 'min', @@ -15,4 +16,8 @@ def apply(metric): if k in renames: metric.tags[renames[k]] = v metric.tags.pop(k) + for k, v in metric.fields.items(): + if k in renames: + metric.fields[renames[k]] = v + metric.fields.pop(k) return metric diff --git a/plugins/processors/starlark/testdata/scale.star b/plugins/processors/starlark/testdata/scale.star index a71c5a9aa..efba6042b 100644 --- a/plugins/processors/starlark/testdata/scale.star +++ b/plugins/processors/starlark/testdata/scale.star @@ -2,6 +2,7 @@ # # Example Input: # modbus,host=hostname Current=1.22,Energy=0,Frequency=60i,Power=0,Voltage=123.9000015258789 1554079521000000000 +# # Example Output: # modbus,host=hostname Current=12.2,Energy=0,Frequency=60i,Power=0,Voltage=1239.000015258789 1554079521000000000 diff --git a/plugins/processors/starlark/testdata/value_filter.star b/plugins/processors/starlark/testdata/value_filter.star new file mode 100644 index 000000000..eeb2432f6 --- /dev/null +++ b/plugins/processors/starlark/testdata/value_filter.star @@ -0,0 +1,18 @@ +# Filter metrics by value +''' +In this example we look at the `value` field of the metric. +If the value is zeor, we delete all the fields, effectively dropping the metric. + +Example Input: +temperature sensor="001A0",value=111.48 +temperature sensor="001B0",value=0.0 + +Example Output: +temperature sensor="001A0",value=111.48 +''' + +def apply(metric): + if metric.fields["value"] == 0.0: + # removing all fields deletes a metric + metric.fields.clear() + return metric