diff --git a/plugins/outputs/sql/README.md b/plugins/outputs/sql/README.md index 3a4cefe44..dfbb3cedc 100644 --- a/plugins/outputs/sql/README.md +++ b/plugins/outputs/sql/README.md @@ -91,7 +91,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ## See the plugin readme for details. data_source_name = "" - ## Timestamp column name + ## Timestamp column name, set to empty to ignore the timestamp # timestamp_column = "timestamp" ## Table creation template diff --git a/plugins/outputs/sql/sample.conf b/plugins/outputs/sql/sample.conf index 9535b861a..59f727afd 100644 --- a/plugins/outputs/sql/sample.conf +++ b/plugins/outputs/sql/sample.conf @@ -10,7 +10,7 @@ ## See the plugin readme for details. data_source_name = "" - ## Timestamp column name + ## Timestamp column name, set to empty to ignore the timestamp # timestamp_column = "timestamp" ## Table creation template diff --git a/plugins/outputs/sql/sql.go b/plugins/outputs/sql/sql.go index ec7e90806..ee9a6b557 100644 --- a/plugins/outputs/sql/sql.go +++ b/plugins/outputs/sql/sql.go @@ -75,10 +75,6 @@ func (p *SQL) Init() error { p.TableExistsTemplate = "SELECT 1 FROM {TABLE} LIMIT 1" } - if p.TimestampColumn == "" { - p.TimestampColumn = "timestamp" - } - if p.TableTemplate == "" { if p.Driver == "clickhouse" { p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS}) ORDER BY ({TAG_COLUMN_NAMES}, {TIMESTAMP_COLUMN_NAME})" @@ -363,6 +359,9 @@ func init() { return &SQL{ Convert: defaultConvert, + // Allow overriding the timestamp column to empty by the user + TimestampColumn: "timestamp", + // Defaults for the connection settings (ConnectionMaxIdleTime, // ConnectionMaxLifetime, ConnectionMaxIdle, and ConnectionMaxOpen) // mirror the golang defaults. As of go 1.18 all of them default to 0 diff --git a/plugins/outputs/sql/sql_test.go b/plugins/outputs/sql/sql_test.go index 62b9e55d9..8c3b38e05 100644 --- a/plugins/outputs/sql/sql_test.go +++ b/plugins/outputs/sql/sql_test.go @@ -169,6 +169,7 @@ func TestMysqlIntegration(t *testing.T) { DataSourceName: address, Convert: defaultConvert, InitSQL: "SET sql_mode='ANSI_QUOTES';", + TimestampColumn: "timestamp", ConnectionMaxIdle: 2, Log: testutil.Logger{}, } @@ -252,6 +253,7 @@ func TestPostgresIntegration(t *testing.T) { Driver: "pgx", DataSourceName: address, Convert: defaultConvert, + TimestampColumn: "timestamp", ConnectionMaxIdle: 2, Log: testutil.Logger{}, } @@ -343,6 +345,7 @@ func TestClickHouseIntegration(t *testing.T) { Driver: "clickhouse", DataSourceName: address, Convert: defaultConvert, + TimestampColumn: "timestamp", ConnectionMaxIdle: 2, Log: testutil.Logger{}, } @@ -434,3 +437,87 @@ func TestClickHouseDsnConvert(t *testing.T) { require.Equal(t, tt.expected, plugin.DataSourceName) } } + +func TestMysqlEmptyTimestampColumnIntegration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + initdb, err := filepath.Abs("testdata/mariadb_no_timestamp/initdb/script.sql") + require.NoError(t, err) + + // initdb/script.sql creates this database + const dbname = "foo" + + // The mariadb image lets you set the root password through an env + // var. We'll use root to insert and query test data. + const username = "root" + + password := testutil.GetRandomString(32) + outDir := t.TempDir() + + servicePort := "3306" + container := testutil.Container{ + Image: "mariadb", + Env: map[string]string{ + "MARIADB_ROOT_PASSWORD": password, + }, + Files: map[string]string{ + "/docker-entrypoint-initdb.d/script.sql": initdb, + "/out": outDir, + }, + ExposedPorts: []string{servicePort}, + WaitingFor: wait.ForAll( + wait.ForListeningPort(nat.Port(servicePort)), + wait.ForLog("mariadbd: ready for connections.").WithOccurrence(2), + ), + } + require.NoError(t, container.Start(), "failed to start container") + defer container.Terminate() + + // use the plugin to write to the database + address := fmt.Sprintf("%v:%v@tcp(%v:%v)/%v", + username, password, container.Address, container.Ports[servicePort], dbname, + ) + p := &SQL{ + Driver: "mysql", + DataSourceName: address, + Convert: defaultConvert, + InitSQL: "SET sql_mode='ANSI_QUOTES';", + ConnectionMaxIdle: 2, + Log: testutil.Logger{}, + } + require.NoError(t, p.Init()) + + require.NoError(t, p.Connect()) + require.NoError(t, p.Write(testMetrics)) + + files := []string{ + "./testdata/mariadb_no_timestamp/expected_metric_one.sql", + "./testdata/mariadb_no_timestamp/expected_metric_two.sql", + "./testdata/mariadb_no_timestamp/expected_metric_three.sql", + } + for _, fn := range files { + expected, err := os.ReadFile(fn) + require.NoError(t, err) + + require.Eventually(t, func() bool { + rc, out, err := container.Exec([]string{ + "bash", + "-c", + "mariadb-dump --user=" + username + + " --password=" + password + + " --compact" + + " --skip-opt " + + dbname, + }) + require.NoError(t, err) + require.Equal(t, 0, rc) + + b, err := io.ReadAll(out) + require.NoError(t, err) + + return bytes.Contains(b, expected) + }, 10*time.Second, 500*time.Millisecond, fn) + } +} diff --git a/plugins/outputs/sql/sqlite_test.go b/plugins/outputs/sql/sqlite_test.go index 4bcba28b6..1c6d416d0 100644 --- a/plugins/outputs/sql/sqlite_test.go +++ b/plugins/outputs/sql/sqlite_test.go @@ -25,6 +25,7 @@ func TestSqlite(t *testing.T) { Driver: "sqlite", DataSourceName: address, Convert: defaultConvert, + TimestampColumn: "timestamp", ConnectionMaxIdle: 2, Log: testutil.Logger{}, } diff --git a/plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_one.sql b/plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_one.sql new file mode 100644 index 000000000..365abd1b9 --- /dev/null +++ b/plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_one.sql @@ -0,0 +1,13 @@ +/*!40101 SET character_set_client = utf8mb4 */; +CREATE TABLE `metric_one` ( + `tag_one` text DEFAULT NULL, + `tag_two` text DEFAULT NULL, + `int64_one` int(11) DEFAULT NULL, + `int64_two` int(11) DEFAULT NULL, + `bool_one` tinyint(1) DEFAULT NULL, + `bool_two` tinyint(1) DEFAULT NULL, + `uint64_one` int(10) unsigned DEFAULT NULL, + `float64_one` double DEFAULT NULL +); +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `metric_one` VALUES ('tag1','tag2',1234,2345,1,0,1000000000,3.1415); diff --git a/plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_three.sql b/plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_three.sql new file mode 100644 index 000000000..04eb5cd5e --- /dev/null +++ b/plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_three.sql @@ -0,0 +1,7 @@ +/*!40101 SET character_set_client = utf8mb4 */; +CREATE TABLE `metric three` ( + `tag four` text DEFAULT NULL, + `string two` text DEFAULT NULL +); +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `metric three` VALUES ('tag4','string2'); diff --git a/plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_two.sql b/plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_two.sql new file mode 100644 index 000000000..f5a38d0f8 --- /dev/null +++ b/plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_two.sql @@ -0,0 +1,7 @@ +/*!40101 SET character_set_client = utf8mb4 */; +CREATE TABLE `metric_two` ( + `tag_three` text DEFAULT NULL, + `string_one` text DEFAULT NULL +); +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `metric_two` VALUES ('tag3','string1'); diff --git a/plugins/outputs/sql/testdata/mariadb_no_timestamp/initdb/script.sql b/plugins/outputs/sql/testdata/mariadb_no_timestamp/initdb/script.sql new file mode 100644 index 000000000..7e155e105 --- /dev/null +++ b/plugins/outputs/sql/testdata/mariadb_no_timestamp/initdb/script.sql @@ -0,0 +1,4 @@ +create database foo; +use foo; +create table bar (baz int); +insert into bar (baz) values (1);