Add support of the time module in Starlark Processor (#9004)
This commit is contained in:
Nicolas Filotto
GitHub
parent
f4a51a4c33
commit
f267f342ae
6
go.mod
6
go.mod
|
|
@ -141,11 +141,11 @@ require (
|
|||
github.com/wvanbergen/kazoo-go v0.0.0-20180202103751-f72d8611297a // indirect
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
|
||||
github.com/yuin/gopher-lua v0.0.0-20180630135845-46796da1b0b4 // indirect
|
||||
go.starlark.net v0.0.0-20200901195727-6e684ef5eeee
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
|
||||
go.starlark.net v0.0.0-20210312235212-74c10e2c17dc
|
||||
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
|
||||
golang.org/x/sys v0.0.0-20201112073958-5cba982894dd
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68
|
||||
golang.org/x/text v0.3.4
|
||||
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200205215550-e35592f146e4
|
||||
google.golang.org/api v0.20.0
|
||||
|
|
|
|||
14
go.sum
14
go.sum
|
|
@ -335,6 +335,7 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
|
|||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
|
|
@ -353,6 +354,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
|||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
|
|
@ -742,8 +744,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
|||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.starlark.net v0.0.0-20200901195727-6e684ef5eeee h1:N4eRtIIYHZE5Mw/Km/orb+naLdwAe+lv2HCxRR5rEBw=
|
||||
go.starlark.net v0.0.0-20200901195727-6e684ef5eeee/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU=
|
||||
go.starlark.net v0.0.0-20210312235212-74c10e2c17dc h1:pVkptfeOTFfx+zXZo7HEHN3d5LmhatBFvHdm/f2QnpY=
|
||||
go.starlark.net v0.0.0-20210312235212-74c10e2c17dc/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
|
|
@ -838,8 +840,9 @@ golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81R
|
|||
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA=
|
||||
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -891,11 +894,12 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY=
|
||||
golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
|
|||
|
|
@ -227,7 +227,10 @@ def apply(metric):
|
|||
- [ratio](/plugins/processors/starlark/testdata/ratio.star) - Compute the ratio of two integer fields
|
||||
- [rename](/plugins/processors/starlark/testdata/rename.star) - Rename tags or fields using a name mapping.
|
||||
- [scale](/plugins/processors/starlark/testdata/scale.star) - Multiply any field by a number
|
||||
- [value filter](/plugins/processors/starlark/testdata/value_filter.star) - remove a metric based on a field value.
|
||||
- [time date](/plugins/processors/starlark/testdata/time_date.star) - Parse a date and extract the year, month and day from it.
|
||||
- [time duration](/plugins/processors/starlark/testdata/time_duration.star) - Parse a duration and convert it into a total amount of seconds.
|
||||
- [time timestamp](/plugins/processors/starlark/testdata/time_timestamp.star) - Filter metrics based on the timestamp.
|
||||
- [value filter](/plugins/processors/starlark/testdata/value_filter.star) - Remove a metric based on a field value.
|
||||
- [logging](/plugins/processors/starlark/testdata/logging.star) - Log messages with the logger of Telegraf
|
||||
- [multiple metrics](/plugins/processors/starlark/testdata/multiple_metrics.star) - Return multiple metrics by using [a list](https://docs.bazel.build/versions/master/skylark/lib/list.html) of metrics.
|
||||
- [multiple metrics from json array](/plugins/processors/starlark/testdata/multiple_metrics_with_json.star) - Builds a new metric from each element of a json array then returns all the created metrics.
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/plugins/processors"
|
||||
"go.starlark.net/lib/time"
|
||||
"go.starlark.net/resolve"
|
||||
"go.starlark.net/starlark"
|
||||
"go.starlark.net/starlarkjson"
|
||||
|
|
@ -252,6 +253,10 @@ func loadFunc(module string, logger telegraf.Logger) (starlark.StringDict, error
|
|||
return starlark.StringDict{
|
||||
"log": LogModule(logger),
|
||||
}, nil
|
||||
case "time.star":
|
||||
return starlark.StringDict{
|
||||
"time": time.Module,
|
||||
}, nil
|
||||
default:
|
||||
return nil, errors.New("module " + module + " is not available")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
# Example of parsing a date out of a field and modifying the metric to inject the year, month and day.
|
||||
#
|
||||
# Example Input:
|
||||
# time value="2009-06-12T12:06:10.000000099" 1465839830100400201
|
||||
#
|
||||
# Example Output:
|
||||
# time year=2009i,month=6i,day=12i 1465839830100400201
|
||||
|
||||
load('time.star', 'time')
|
||||
# loads time.parse_duration(), time.is_valid_timezone(), time.now(), time.time(),
|
||||
# time.parse_time() and time.from_timestamp()
|
||||
|
||||
def apply(metric):
|
||||
date = time.parse_time(metric.fields.get('value'), format="2006-01-02T15:04:05.999999999", location="UTC")
|
||||
metric.fields.pop('value')
|
||||
metric.fields["year"] = date.year
|
||||
metric.fields["month"] = date.month
|
||||
metric.fields["day"] = date.day
|
||||
return metric
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
# Example of parsing a duration out of a field and modifying the metric to inject the equivalent in seconds.
|
||||
#
|
||||
# Example Input:
|
||||
# time value="3m35s" 1465839830100400201
|
||||
#
|
||||
# Example Output:
|
||||
# time seconds=215 1465839830100400201
|
||||
|
||||
load('time.star', 'time')
|
||||
# loads time.parse_duration(), time.is_valid_timezone(), time.now(), time.time(),
|
||||
# time.parse_time() and time.from_timestamp()
|
||||
|
||||
def apply(metric):
|
||||
duration = time.parse_duration(metric.fields.get('value'))
|
||||
metric.fields.pop('value')
|
||||
metric.fields["seconds"] = duration.seconds
|
||||
return metric
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
# Example of filtering metrics based on the timestamp. Beware the built-in function from_timestamp
|
||||
# only supports timestamps in seconds.
|
||||
#
|
||||
# Example Input:
|
||||
# time result="KO" 1616020365100400201
|
||||
# time result="OK" 1616150517100400201
|
||||
#
|
||||
# Example Output:
|
||||
# time result="OK" 1616150517100400201
|
||||
|
||||
load('time.star', 'time')
|
||||
# loads time.parse_duration(), time.is_valid_timezone(), time.now(), time.time(),
|
||||
# time.parse_time() and time.from_timestamp()
|
||||
|
||||
def apply(metric):
|
||||
# 1616198400 sec = Saturday, March 20, 2021 0:00:00 GMT
|
||||
refDate = time.from_timestamp(1616198400)
|
||||
# 1616020365 sec = Wednesday, March 17, 2021 22:32:45 GMT
|
||||
# 1616150517 sec = Friday, March 19, 2021 10:41:57 GMT
|
||||
metric_date = time.from_timestamp(int(metric.time / 1e9))
|
||||
# Only keep metrics with a timestamp that is not more than 24 hours before the reference date
|
||||
if refDate - time.parse_duration("24h") < metric_date:
|
||||
return metric
|
||||
Loading…
Reference in New Issue