feat: add custom time/date format field for elasticsearch_query (#9838)

plugins/inputs/elasticsearch_query/README.md

@@ -54,6 +54,13 @@ Currently it is known to break on 7.x or greater versions.
     ## The date/time field in the Elasticsearch index (mandatory).
     date_field = "@timestamp"
 
+    ## If the field used for the date/time field in Elasticsearch is also using
+    ## a custom date/time format it may be required to provide the format to
+    ## correctly parse the field.
+    ##
+    ## If using one of the built in elasticsearch formats this is not required.
+    # date_field_custom_format = ""
+
     ## Time window to query (eg. "1m" to query documents from last minute).
     ## Normally should be set to same as collection interval
     query_period = "1m"
@@ -150,6 +157,7 @@ Please note that the `[[inputs.elasticsearch_query]]` is still required for all
 
 ### Optional parameters
 
+- `date_field_custom_format`: Not needed if using one of the built in date/time formats of Elasticsearch, but may be required if using a custom date/time format. The format syntax uses the [Joda date format](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/search-aggregations-bucket-daterange-aggregation.html#date-format-pattern).
 - `filter_query`: Lucene query to filter the results (default: "\*")
 - `metric_fields`: The list of fields to perform metric aggregation (these must be indexed as numeric fields)
 - `metric_funcion`: The single-value metric aggregation function to be performed on the `metric_fields` defined. Currently supported aggregations are "avg", "min", "max", "sum". (see [https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics.html](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics.html)

plugins/inputs/elasticsearch_query/aggregation_query.go

@@ -33,7 +33,7 @@ func (e *ElasticsearchQuery) runAggregationQuery(ctx context.Context, aggregatio
 
 	query := elastic5.NewBoolQuery()
 	query = query.Filter(elastic5.NewQueryStringQuery(filterQuery))
-	query = query.Filter(elastic5.NewRangeQuery(aggregation.DateField).From(from).To(now))
+	query = query.Filter(elastic5.NewRangeQuery(aggregation.DateField).From(from).To(now).Format(aggregation.DateFieldFormat))
 
 	src, err := query.Source()
 	if err != nil {

plugins/inputs/elasticsearch_query/elasticsearch_query.go

@@ -55,6 +55,13 @@ const sampleConfig = `
     ## The date/time field in the Elasticsearch index (mandatory).
     date_field = "@timestamp"
 
+    ## If the field used for the date/time field in Elasticsearch is also using
+    ## a custom date/time format it may be required to provide the format to
+    ## correctly parse the field.
+    ##
+    ## If using one of the built in elasticsearch formats this is not required.
+    # date_field_custom_format = ""
+
     ## Time window to query (eg. "1m" to query documents from last minute).
     ## Normally should be set to same as collection interval
     query_period = "1m"
@@ -104,6 +111,7 @@ type esAggregation struct {
 	Index                string          `toml:"index"`
 	MeasurementName      string          `toml:"measurement_name"`
 	DateField            string          `toml:"date_field"`
+	DateFieldFormat      string          `toml:"date_field_custom_format"`
 	QueryPeriod          config.Duration `toml:"query_period"`
 	FilterQuery          string          `toml:"filter_query"`
 	MetricFields         []string        `toml:"metric_fields"`

plugins/inputs/elasticsearch_query/elasticsearch_query_test.go

@@ -484,6 +484,23 @@ var testEsAggregationData = []esAggregationQueryTest{
 		false,
 		false,
 	},
+	{
+		"query 14 - non-existing custom date/time format",
+		esAggregation{
+			Index:           testindex,
+			MeasurementName: "measurement14",
+			DateField:       "@timestamp",
+			DateFieldFormat: "yyyy",
+			QueryPeriod:     queryPeriod,
+			Tags:            []string{},
+			mapMetricFields: map[string]string{},
+		},
+		nil,
+		nil,
+		false,
+		false,
+		true,
+	},
 }
 
 func setupIntegrationTest() error {