From 008e43b7c83232cbdb98040e01ad95caa58b41e0 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Thu, 3 Mar 2022 08:21:39 -0700 Subject: [PATCH] fix: sql unsigned settings (#10673) --- plugins/outputs/sql/README.md | 9 +++- plugins/outputs/sql/sql.go | 46 +++++++++++++------ plugins/outputs/sql/sql_test.go | 12 +++++ .../sql/testdata/clickhouse/expected.txt | 6 ++- .../outputs/sql/testdata/mariadb/expected.sql | 6 ++- .../sql/testdata/postgres/expected.sql | 8 ++-- 6 files changed, 64 insertions(+), 23 deletions(-) diff --git a/plugins/outputs/sql/README.md b/plugins/outputs/sql/README.md index 8863a7b04..75294d35e 100644 --- a/plugins/outputs/sql/README.md +++ b/plugins/outputs/sql/README.md @@ -112,6 +112,13 @@ through the convert settings. # defaultvalue = "TEXT" # unsigned = "UNSIGNED" # bool = "BOOL" + + ## This setting controls the behavior of the unsigned value. By default the + ## setting will take the integer value and append the unsigned value to it. The other + ## option is "literal", which will use the actual value the user provides to + ## the unsigned option. This is useful for a database like ClickHouse where + ## the unsigned value should use a value like "uint64". + # conversion_style = "unsigned_suffix" ``` ## Driver-specific information @@ -158,7 +165,7 @@ Use this metric type to SQL type conversion: timestamp = "DateTime" defaultvalue = "String" unsigned = "UInt64" - bool = "Uint8" + bool = "UInt8" ``` See [ClickHouse data types](https://clickhouse.com/docs/en/sql-reference/data-types/) for more info. diff --git a/plugins/outputs/sql/sql.go b/plugins/outputs/sql/sql.go index 8598a220c..6f4f9b36c 100644 --- a/plugins/outputs/sql/sql.go +++ b/plugins/outputs/sql/sql.go @@ -17,13 +17,14 @@ import ( ) type ConvertStruct struct { - Integer string - Real string - Text string - Timestamp string - Defaultvalue string - Unsigned string - Bool string + Integer string + Real string + Text string + Timestamp string + Defaultvalue string + Unsigned string + Bool string + ConversionStyle string } type SQL struct { @@ -100,7 +101,13 @@ func (p *SQL) deriveDatatype(value interface{}) string { case int64: datatype = p.Convert.Integer case uint64: - datatype = fmt.Sprintf("%s %s", p.Convert.Integer, p.Convert.Unsigned) + if p.Convert.ConversionStyle == "unsigned_suffix" { + datatype = fmt.Sprintf("%s %s", p.Convert.Integer, p.Convert.Unsigned) + } else if p.Convert.ConversionStyle == "literal" { + datatype = p.Convert.Unsigned + } else { + p.Log.Errorf("unknown converstaion style: %s", p.Convert.ConversionStyle) + } case float64: datatype = p.Convert.Real case string: @@ -158,6 +165,14 @@ var sampleConfig = ` # timestamp = "TIMESTAMP" # defaultvalue = "TEXT" # unsigned = "UNSIGNED" + # bool = "BOOL" + + ## This setting controls the behavior of the unsigned value. By default the + ## setting will take the integer value and append the unsigned value to it. The other + ## option is "literal", which will use the actual value the user provides to + ## the unsigned option. This is useful for a database like ClickHouse where + ## the unsigned value should use a value like "uint64". + # conversion_style = "unsigned_suffix" ` func (p *SQL) SampleConfig() string { return sampleConfig } @@ -300,13 +315,14 @@ func newSQL() *SQL { TableExistsTemplate: "SELECT 1 FROM {TABLE} LIMIT 1", TimestampColumn: "timestamp", Convert: ConvertStruct{ - Integer: "INT", - Real: "DOUBLE", - Text: "TEXT", - Timestamp: "TIMESTAMP", - Defaultvalue: "TEXT", - Unsigned: "UNSIGNED", - Bool: "BOOL", + Integer: "INT", + Real: "DOUBLE", + Text: "TEXT", + Timestamp: "TIMESTAMP", + Defaultvalue: "TEXT", + Unsigned: "UNSIGNED", + Bool: "BOOL", + ConversionStyle: "unsigned_suffix", }, } } diff --git a/plugins/outputs/sql/sql_test.go b/plugins/outputs/sql/sql_test.go index 256f02957..da8fc3ef5 100644 --- a/plugins/outputs/sql/sql_test.go +++ b/plugins/outputs/sql/sql_test.go @@ -107,6 +107,14 @@ var ( Key: "bool_two", Value: false, }, + { + Key: "uint64_one", + Value: uint64(1000000000), + }, + { + Key: "float64_one", + Value: float64(3.1415), + }, }, ts, ), @@ -295,6 +303,9 @@ func TestPostgresIntegration(t *testing.T) { p.Log = testutil.Logger{} p.Driver = "pgx" p.DataSourceName = address + p.Convert.Real = "double precision" + p.Convert.Unsigned = "bigint" + p.Convert.ConversionStyle = "literal" require.NoError(t, p.Connect()) require.NoError(t, p.Write( @@ -396,6 +407,7 @@ func TestClickHouseIntegration(t *testing.T) { p.Convert.Defaultvalue = "String" p.Convert.Unsigned = "UInt64" p.Convert.Bool = "UInt8" + p.Convert.ConversionStyle = "literal" require.NoError(t, p.Connect()) diff --git a/plugins/outputs/sql/testdata/clickhouse/expected.txt b/plugins/outputs/sql/testdata/clickhouse/expected.txt index 3f1efddc1..f530c2228 100644 --- a/plugins/outputs/sql/testdata/clickhouse/expected.txt +++ b/plugins/outputs/sql/testdata/clickhouse/expected.txt @@ -1,4 +1,4 @@ -2021-05-17 22:04:45 tag1 tag2 1234 2345 1 0 +2021-05-17 22:04:45 tag1 tag2 1234 2345 1 0 1000000000 3.1415 CREATE TABLE foo.metric_one ( `timestamp` DateTime, @@ -7,7 +7,9 @@ CREATE TABLE foo.metric_one `int64_one` Int64, `int64_two` Int64, `bool_one` UInt8, - `bool_two` UInt8 + `bool_two` UInt8, + `uint64_one` UInt64, + `float64_one` Float64 ) ENGINE = MergeTree ORDER BY timestamp diff --git a/plugins/outputs/sql/testdata/mariadb/expected.sql b/plugins/outputs/sql/testdata/mariadb/expected.sql index 43e0fa5e5..2bcf73f2f 100644 --- a/plugins/outputs/sql/testdata/mariadb/expected.sql +++ b/plugins/outputs/sql/testdata/mariadb/expected.sql @@ -23,10 +23,12 @@ CREATE TABLE `metric_one` ( `int64_one` int(11) DEFAULT NULL, `int64_two` int(11) DEFAULT NULL, `bool_one` tinyint(1) DEFAULT NULL, - `bool_two` tinyint(1) DEFAULT NULL + `bool_two` tinyint(1) DEFAULT NULL, + `uint64_one` int(10) unsigned DEFAULT NULL, + `float64_one` double DEFAULT NULL ); /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `metric_one` VALUES ('2021-05-17 22:04:45','tag1','tag2',1234,2345,1,0); +INSERT INTO `metric_one` VALUES ('2021-05-17 22:04:45','tag1','tag2',1234,2345,1,0,1000000000,3.1415); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `metric_two` ( diff --git a/plugins/outputs/sql/testdata/postgres/expected.sql b/plugins/outputs/sql/testdata/postgres/expected.sql index c1ee733ac..f4cde683b 100644 --- a/plugins/outputs/sql/testdata/postgres/expected.sql +++ b/plugins/outputs/sql/testdata/postgres/expected.sql @@ -23,7 +23,9 @@ CREATE TABLE public.metric_one ( int64_one integer, int64_two integer, bool_one boolean, - bool_two boolean + bool_two boolean, + uint64_one bigint, + float64_one double precision ); ALTER TABLE public.metric_one OWNER TO postgres; CREATE TABLE public.metric_two ( @@ -35,8 +37,8 @@ ALTER TABLE public.metric_two OWNER TO postgres; COPY public."metric three" ("timestamp", "tag four", "string two") FROM stdin; 2021-05-17 22:04:45 tag4 string2 \. -COPY public.metric_one ("timestamp", tag_one, tag_two, int64_one, int64_two, bool_one, bool_two) FROM stdin; -2021-05-17 22:04:45 tag1 tag2 1234 2345 t f +COPY public.metric_one ("timestamp", tag_one, tag_two, int64_one, int64_two, bool_one, bool_two, uint64_one, float64_one) FROM stdin; +2021-05-17 22:04:45 tag1 tag2 1234 2345 t f 1000000000 3.1415 \. COPY public.metric_two ("timestamp", tag_three, string_one) FROM stdin; 2021-05-17 22:04:45 tag3 string1