diff --git a/plugins/outputs/azure_data_explorer/README.md b/plugins/outputs/azure_data_explorer/README.md index 4ae5bf713..db2aba469 100644 --- a/plugins/outputs/azure_data_explorer/README.md +++ b/plugins/outputs/azure_data_explorer/README.md @@ -31,6 +31,10 @@ Azure Data Explorer is a distributed, columnar store, purpose built for any type ## Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable"). # table_name = "" + + ## Creates tables and relevant mapping if set to true(default). + ## Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. + # create_tables = true ``` ## Metrics Grouping @@ -85,7 +89,10 @@ These methods are: [principal]: https://docs.microsoft.com/en-us/azure/active-directory/develop/active-directory-application-objects -Whichever method, the designated Principal needs to be assigned the `Database User` role on the Database level in the Azure Data Explorer. This role will allow the plugin to create the required tables and ingest data into it. +Whichever method, the designated Principal needs to be assigned the `Database User` role on the Database level in the Azure Data Explorer. This role will +allow the plugin to create the required tables and ingest data into it. +If `create_tables=false` then the designated principal only needs the `Database Ingestor` role at least. + ### Configurations of the chosen Authentication Method diff --git a/plugins/outputs/azure_data_explorer/azure_data_explorer.go b/plugins/outputs/azure_data_explorer/azure_data_explorer.go index b4c2054d3..1f958d525 100644 --- a/plugins/outputs/azure_data_explorer/azure_data_explorer.go +++ b/plugins/outputs/azure_data_explorer/azure_data_explorer.go @@ -27,6 +27,7 @@ type AzureDataExplorer struct { Timeout config.Duration `toml:"timeout"` MetricsGrouping string `toml:"metrics_grouping_type"` TableName string `toml:"table_name"` + CreateTables bool `toml:"create_tables"` client localClient ingesters map[string]localIngestor serializer serializers.Serializer @@ -57,7 +58,7 @@ func (adx *AzureDataExplorer) Description() string { func (adx *AzureDataExplorer) SampleConfig() string { return ` - ## Azure Data Exlorer cluster endpoint + ## Azure Data Explorer cluster endpoint ## ex: endpoint_url = "https://clustername.australiasoutheast.kusto.windows.net" endpoint_url = "" @@ -77,6 +78,9 @@ func (adx *AzureDataExplorer) SampleConfig() string { ## Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable"). # table_name = "" + ## Creates tables and relevant mapping if set to true(default). + ## Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. + # create_tables = true ` } @@ -198,6 +202,10 @@ func (adx *AzureDataExplorer) getIngestor(ctx context.Context, tableName string) } func (adx *AzureDataExplorer) createAzureDataExplorerTable(ctx context.Context, tableName string) error { + if !adx.CreateTables { + adx.Log.Info("skipped table creation") + return nil + } createStmt := kusto.NewStmt("", kusto.UnsafeStmt(unsafe.Stmt{Add: true, SuppressWarning: true})).UnsafeAdd(fmt.Sprintf(createTableCommand, tableName)) if _, err := adx.client.Mgmt(ctx, adx.Database, createStmt); err != nil { return err @@ -241,7 +249,8 @@ func (adx *AzureDataExplorer) Init() error { func init() { outputs.Add("azure_data_explorer", func() telegraf.Output { return &AzureDataExplorer{ - Timeout: config.Duration(20 * time.Second), + Timeout: config.Duration(20 * time.Second), + CreateTables: true, } }) } diff --git a/plugins/outputs/azure_data_explorer/azure_data_explorer_test.go b/plugins/outputs/azure_data_explorer/azure_data_explorer_test.go index b8d30d66c..ce53acf43 100644 --- a/plugins/outputs/azure_data_explorer/azure_data_explorer_test.go +++ b/plugins/outputs/azure_data_explorer/azure_data_explorer_test.go @@ -31,10 +31,12 @@ func TestWrite(t *testing.T) { tableName string expected map[string]interface{} expectedWriteError string + createTables bool }{ { - name: "Valid metric", - inputMetric: testutil.MockMetrics(), + name: "Valid metric", + inputMetric: testutil.MockMetrics(), + createTables: true, client: &fakeClient{ queries: make([]string, 0), internalMgmt: func(f *fakeClient, ctx context.Context, db string, query kusto.Stmt, options ...kusto.MgmtOption) (*kusto.RowIterator, error) { @@ -56,8 +58,34 @@ func TestWrite(t *testing.T) { }, }, { - name: "Error in Mgmt", - inputMetric: testutil.MockMetrics(), + name: "Don't create tables'", + inputMetric: testutil.MockMetrics(), + createTables: false, + client: &fakeClient{ + queries: make([]string, 0), + internalMgmt: func(f *fakeClient, ctx context.Context, db string, query kusto.Stmt, options ...kusto.MgmtOption) (*kusto.RowIterator, error) { + require.Fail(t, "Mgmt shouldn't be called when create_tables is false") + f.queries = append(f.queries, query.String()) + return &kusto.RowIterator{}, nil + }, + }, + createIngestor: createFakeIngestor, + metricsGrouping: tablePerMetric, + expected: map[string]interface{}{ + "metricName": "test1", + "fields": map[string]interface{}{ + "value": 1.0, + }, + "tags": map[string]interface{}{ + "tag1": "value1", + }, + "timestamp": float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).UnixNano() / int64(time.Second)), + }, + }, + { + name: "Error in Mgmt", + inputMetric: testutil.MockMetrics(), + createTables: true, client: &fakeClient{ queries: make([]string, 0), internalMgmt: func(f *fakeClient, ctx context.Context, db string, query kusto.Stmt, options ...kusto.MgmtOption) (*kusto.RowIterator, error) { @@ -79,8 +107,9 @@ func TestWrite(t *testing.T) { expectedWriteError: "creating table for \"test1\" failed: Something went wrong", }, { - name: "SingleTable metric grouping type", - inputMetric: testutil.MockMetrics(), + name: "SingleTable metric grouping type", + inputMetric: testutil.MockMetrics(), + createTables: true, client: &fakeClient{ queries: make([]string, 0), internalMgmt: func(f *fakeClient, ctx context.Context, db string, query kusto.Stmt, options ...kusto.MgmtOption) (*kusto.RowIterator, error) { @@ -114,6 +143,7 @@ func TestWrite(t *testing.T) { Log: testutil.Logger{}, MetricsGrouping: tC.metricsGrouping, TableName: tC.tableName, + CreateTables: tC.createTables, client: tC.client, ingesters: map[string]localIngestor{}, createIngestor: tC.createIngestor, @@ -149,11 +179,15 @@ func TestWrite(t *testing.T) { expectedTime := tC.expected["timestamp"].(float64) require.Equal(t, expectedTime, createdFakeIngestor.actualOutputMetric["timestamp"]) - createTableString := fmt.Sprintf(createTableCommandExpected, expectedNameOfTable) - require.Equal(t, createTableString, tC.client.queries[0]) + if tC.createTables { + createTableString := fmt.Sprintf(createTableCommandExpected, expectedNameOfTable) + require.Equal(t, createTableString, tC.client.queries[0]) - createTableMappingString := fmt.Sprintf(createTableMappingCommandExpected, expectedNameOfTable, expectedNameOfTable) - require.Equal(t, createTableMappingString, tC.client.queries[1]) + createTableMappingString := fmt.Sprintf(createTableMappingCommandExpected, expectedNameOfTable, expectedNameOfTable) + require.Equal(t, createTableMappingString, tC.client.queries[1]) + } else { + require.Empty(t, tC.client.queries) + } } }) } @@ -185,10 +219,10 @@ type fakeIngestor struct { actualOutputMetric map[string]interface{} } -func createFakeIngestor(client localClient, database string, tableName string) (localIngestor, error) { +func createFakeIngestor(localClient, string, string) (localIngestor, error) { return &fakeIngestor{}, nil } -func (f *fakeIngestor) FromReader(ctx context.Context, reader io.Reader, options ...ingest.FileOption) (*ingest.Result, error) { +func (f *fakeIngestor) FromReader(_ context.Context, reader io.Reader, _ ...ingest.FileOption) (*ingest.Result, error) { scanner := bufio.NewScanner(reader) scanner.Scan() firstLine := scanner.Text()