fix(outputs.sql): Allow to disable timestamp column (#16625)

plugins/outputs/sql/README.md

@@ -91,7 +91,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
   ## See the plugin readme for details.
   data_source_name = ""
 
-  ## Timestamp column name
+  ## Timestamp column name, set to empty to ignore the timestamp
   # timestamp_column = "timestamp"
 
   ## Table creation template

plugins/outputs/sql/sample.conf

@@ -10,7 +10,7 @@
   ## See the plugin readme for details.
   data_source_name = ""
 
-  ## Timestamp column name
+  ## Timestamp column name, set to empty to ignore the timestamp
   # timestamp_column = "timestamp"
 
   ## Table creation template

plugins/outputs/sql/sql.go

@@ -75,10 +75,6 @@ func (p *SQL) Init() error {
 		p.TableExistsTemplate = "SELECT 1 FROM {TABLE} LIMIT 1"
 	}
 
-	if p.TimestampColumn == "" {
-		p.TimestampColumn = "timestamp"
-	}
-
 	if p.TableTemplate == "" {
 		if p.Driver == "clickhouse" {
 			p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS}) ORDER BY ({TAG_COLUMN_NAMES}, {TIMESTAMP_COLUMN_NAME})"
@@ -363,6 +359,9 @@ func init() {
 		return &SQL{
 			Convert: defaultConvert,
 
+			// Allow overriding the timestamp column to empty by the user
+			TimestampColumn: "timestamp",
+
 			// Defaults for the connection settings (ConnectionMaxIdleTime,
 			// ConnectionMaxLifetime, ConnectionMaxIdle, and ConnectionMaxOpen)
 			// mirror the golang defaults. As of go 1.18 all of them default to 0

plugins/outputs/sql/sql_test.go

@@ -169,6 +169,7 @@ func TestMysqlIntegration(t *testing.T) {
 		DataSourceName:    address,
 		Convert:           defaultConvert,
 		InitSQL:           "SET sql_mode='ANSI_QUOTES';",
+		TimestampColumn:   "timestamp",
 		ConnectionMaxIdle: 2,
 		Log:               testutil.Logger{},
 	}
@@ -252,6 +253,7 @@ func TestPostgresIntegration(t *testing.T) {
 		Driver:            "pgx",
 		DataSourceName:    address,
 		Convert:           defaultConvert,
+		TimestampColumn:   "timestamp",
 		ConnectionMaxIdle: 2,
 		Log:               testutil.Logger{},
 	}
@@ -343,6 +345,7 @@ func TestClickHouseIntegration(t *testing.T) {
 		Driver:            "clickhouse",
 		DataSourceName:    address,
 		Convert:           defaultConvert,
+		TimestampColumn:   "timestamp",
 		ConnectionMaxIdle: 2,
 		Log:               testutil.Logger{},
 	}
@@ -434,3 +437,87 @@ func TestClickHouseDsnConvert(t *testing.T) {
 		require.Equal(t, tt.expected, plugin.DataSourceName)
 	}
 }
+
+func TestMysqlEmptyTimestampColumnIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	initdb, err := filepath.Abs("testdata/mariadb_no_timestamp/initdb/script.sql")
+	require.NoError(t, err)
+
+	// initdb/script.sql creates this database
+	const dbname = "foo"
+
+	// The mariadb image lets you set the root password through an env
+	// var. We'll use root to insert and query test data.
+	const username = "root"
+
+	password := testutil.GetRandomString(32)
+	outDir := t.TempDir()
+
+	servicePort := "3306"
+	container := testutil.Container{
+		Image: "mariadb",
+		Env: map[string]string{
+			"MARIADB_ROOT_PASSWORD": password,
+		},
+		Files: map[string]string{
+			"/docker-entrypoint-initdb.d/script.sql": initdb,
+			"/out":                                   outDir,
+		},
+		ExposedPorts: []string{servicePort},
+		WaitingFor: wait.ForAll(
+			wait.ForListeningPort(nat.Port(servicePort)),
+			wait.ForLog("mariadbd: ready for connections.").WithOccurrence(2),
+		),
+	}
+	require.NoError(t, container.Start(), "failed to start container")
+	defer container.Terminate()
+
+	// use the plugin to write to the database
+	address := fmt.Sprintf("%v:%v@tcp(%v:%v)/%v",
+		username, password, container.Address, container.Ports[servicePort], dbname,
+	)
+	p := &SQL{
+		Driver:            "mysql",
+		DataSourceName:    address,
+		Convert:           defaultConvert,
+		InitSQL:           "SET sql_mode='ANSI_QUOTES';",
+		ConnectionMaxIdle: 2,
+		Log:               testutil.Logger{},
+	}
+	require.NoError(t, p.Init())
+
+	require.NoError(t, p.Connect())
+	require.NoError(t, p.Write(testMetrics))
+
+	files := []string{
+		"./testdata/mariadb_no_timestamp/expected_metric_one.sql",
+		"./testdata/mariadb_no_timestamp/expected_metric_two.sql",
+		"./testdata/mariadb_no_timestamp/expected_metric_three.sql",
+	}
+	for _, fn := range files {
+		expected, err := os.ReadFile(fn)
+		require.NoError(t, err)
+
+		require.Eventually(t, func() bool {
+			rc, out, err := container.Exec([]string{
+				"bash",
+				"-c",
+				"mariadb-dump --user=" + username +
+					" --password=" + password +
+					" --compact" +
+					" --skip-opt " +
+					dbname,
+			})
+			require.NoError(t, err)
+			require.Equal(t, 0, rc)
+
+			b, err := io.ReadAll(out)
+			require.NoError(t, err)
+
+			return bytes.Contains(b, expected)
+		}, 10*time.Second, 500*time.Millisecond, fn)
+	}
+}

plugins/outputs/sql/sqlite_test.go

@@ -25,6 +25,7 @@ func TestSqlite(t *testing.T) {
 		Driver:            "sqlite",
 		DataSourceName:    address,
 		Convert:           defaultConvert,
+		TimestampColumn:   "timestamp",
 		ConnectionMaxIdle: 2,
 		Log:               testutil.Logger{},
 	}

plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_one.sql

@@ -0,0 +1,13 @@
+/*!40101 SET character_set_client = utf8mb4 */;
+CREATE TABLE `metric_one` (
+  `tag_one` text DEFAULT NULL,
+  `tag_two` text DEFAULT NULL,
+  `int64_one` int(11) DEFAULT NULL,
+  `int64_two` int(11) DEFAULT NULL,
+  `bool_one` tinyint(1) DEFAULT NULL,
+  `bool_two` tinyint(1) DEFAULT NULL,
+  `uint64_one` int(10) unsigned DEFAULT NULL,
+  `float64_one` double DEFAULT NULL
+);
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `metric_one` VALUES ('tag1','tag2',1234,2345,1,0,1000000000,3.1415);

plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_three.sql

@@ -0,0 +1,7 @@
+/*!40101 SET character_set_client = utf8mb4 */;
+CREATE TABLE `metric three` (
+  `tag four` text DEFAULT NULL,
+  `string two` text DEFAULT NULL
+);
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `metric three` VALUES ('tag4','string2');

plugins/outputs/sql/testdata/mariadb_no_timestamp/expected_metric_two.sql

@@ -0,0 +1,7 @@
+/*!40101 SET character_set_client = utf8mb4 */;
+CREATE TABLE `metric_two` (
+  `tag_three` text DEFAULT NULL,
+  `string_one` text DEFAULT NULL
+);
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `metric_two` VALUES ('tag3','string1');

plugins/outputs/sql/testdata/mariadb_no_timestamp/initdb/script.sql

@@ -0,0 +1,4 @@
+create database foo;
+use foo;
+create table bar (baz int);
+insert into bar (baz) values (1);