feat: Add ClickHouse driver to sql inputs/outputs plugins (#9671)

This commit is contained in:
Anatoly Laskaris 2022-01-28 23:35:03 +03:00 committed by GitHub
parent 74357c548c
commit 531d7bb741
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 328 additions and 20 deletions

View File

@ -16,6 +16,7 @@ following works:
- github.com/Azure/go-amqp [MIT License](https://github.com/Azure/go-amqp/blob/master/LICENSE) - github.com/Azure/go-amqp [MIT License](https://github.com/Azure/go-amqp/blob/master/LICENSE)
- github.com/Azure/go-autorest [Apache License 2.0](https://github.com/Azure/go-autorest/blob/master/LICENSE) - github.com/Azure/go-autorest [Apache License 2.0](https://github.com/Azure/go-autorest/blob/master/LICENSE)
- github.com/Azure/go-ntlmssp [MIT License](https://github.com/Azure/go-ntlmssp/blob/master/LICENSE) - github.com/Azure/go-ntlmssp [MIT License](https://github.com/Azure/go-ntlmssp/blob/master/LICENSE)
- github.com/ClickHouse/clickhouse-go [MIT License](https://github.com/ClickHouse/clickhouse-go/blob/master/LICENSE)
- github.com/Mellanox/rdmamap [Apache License 2.0](https://github.com/Mellanox/rdmamap/blob/master/LICENSE) - github.com/Mellanox/rdmamap [Apache License 2.0](https://github.com/Mellanox/rdmamap/blob/master/LICENSE)
- github.com/Microsoft/go-winio [MIT License](https://github.com/Microsoft/go-winio/blob/master/LICENSE) - github.com/Microsoft/go-winio [MIT License](https://github.com/Microsoft/go-winio/blob/master/LICENSE)
- github.com/Shopify/sarama [MIT License](https://github.com/Shopify/sarama/blob/master/LICENSE) - github.com/Shopify/sarama [MIT License](https://github.com/Shopify/sarama/blob/master/LICENSE)

View File

@ -3,15 +3,16 @@
This is a list of available drivers for the SQL input plugin. The data-source-name (DSN) is driver specific and This is a list of available drivers for the SQL input plugin. The data-source-name (DSN) is driver specific and
might change between versions. Please check the driver documentation for available options and the format. might change between versions. Please check the driver documentation for available options and the format.
database | driver | aliases | example DSN | comment | database | driver | aliases | example DSN | comment |
---------------------| ------------------------------------------------------| --------------- | -------------------------------------------------------------------------------------- | ------- | -------------------- | --------------------------------------------------------- | --------------- | -------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
CockroachDB | [cockroach](https://github.com/jackc/pgx) | postgres or pgx | see _postgres_ driver | uses PostgresQL driver | CockroachDB | [cockroach](https://github.com/jackc/pgx) | postgres or pgx | see _postgres_ driver | uses PostgresQL driver |
MariaDB | [maria](https://github.com/go-sql-driver/mysql) | mysql | see _mysql_ driver | uses MySQL driver | MariaDB | [maria](https://github.com/go-sql-driver/mysql) | mysql | see _mysql_ driver | uses MySQL driver |
Microsoft SQL Server | [sqlserver](https://github.com/denisenkom/go-mssqldb) | mssql | `username:password@host/instance?param1=value&param2=value` | uses newer _sqlserver_ driver | Microsoft SQL Server | [sqlserver](https://github.com/denisenkom/go-mssqldb) | mssql | `username:password@host/instance?param1=value&param2=value` | uses newer _sqlserver_ driver |
MySQL | [mysql](https://github.com/go-sql-driver/mysql) | | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]` | see [driver docs](https://github.com/go-sql-driver/mysql) for more information | MySQL | [mysql](https://github.com/go-sql-driver/mysql) | | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]` | see [driver docs](https://github.com/go-sql-driver/mysql) for more information |
PostgreSQL | [postgres](https://github.com/jackc/pgx) | pgx | `[user[:password]@][netloc][:port][,...][/dbname][?param1=value1&...]` | see [postgres docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) for more information | PostgreSQL | [postgres](https://github.com/jackc/pgx) | pgx | `[user[:password]@][netloc][:port][,...][/dbname][?param1=value1&...]` | see [postgres docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) for more information |
SQLite | [sqlite](https://gitlab.com/cznic/sqlite) | | `filename` | see [driver docu](https://pkg.go.dev/modernc.org/sqlite) for more information | SQLite | [sqlite](https://gitlab.com/cznic/sqlite) | | `filename` | see [driver docu](https://pkg.go.dev/modernc.org/sqlite) for more information |
TiDB | [tidb](https://github.com/go-sql-driver/mysql) | mysql | see _mysql_ driver | uses MySQL driver | TiDB | [tidb](https://github.com/go-sql-driver/mysql) | mysql | see _mysql_ driver | uses MySQL driver |
| ClickHouse | [clickhouse](https://github.com/ClickHouse/clickhouse-go) | | `tcp://host:port[?param1=value&...&paramN=value]"` | see [clickhouse-go docs](https://github.com/ClickHouse/clickhouse-go#dsn) for more information |
## Comments ## Comments

2
go.mod
View File

@ -14,6 +14,7 @@ require (
github.com/Azure/go-autorest/autorest/adal v0.9.16 github.com/Azure/go-autorest/autorest/adal v0.9.16
github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8
github.com/BurntSushi/toml v0.4.1 github.com/BurntSushi/toml v0.4.1
github.com/ClickHouse/clickhouse-go v1.5.1
github.com/Mellanox/rdmamap v0.0.0-20191106181932-7c3c4763a6ee github.com/Mellanox/rdmamap v0.0.0-20191106181932-7c3c4763a6ee
github.com/Shopify/sarama v1.29.1 github.com/Shopify/sarama v1.29.1
github.com/aerospike/aerospike-client-go v1.27.0 github.com/aerospike/aerospike-client-go v1.27.0
@ -202,6 +203,7 @@ require (
github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.1.1 // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
github.com/containerd/cgroups v1.0.1 // indirect github.com/containerd/cgroups v1.0.1 // indirect
github.com/containerd/containerd v1.5.9 // indirect github.com/containerd/containerd v1.5.9 // indirect
github.com/couchbase/gomemcached v0.1.3 // indirect github.com/couchbase/gomemcached v0.1.3 // indirect

6
go.sum
View File

@ -159,6 +159,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/ClickHouse/clickhouse-go v1.5.1 h1:I8zVFZTz80crCs0FFEBJooIxsPcV0xfthzK1YrkpJTc=
github.com/ClickHouse/clickhouse-go v1.5.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
@ -432,6 +434,8 @@ github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2io
github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/bkielbasa/cyclop v1.2.0/go.mod h1:qOI0yy6A7dYC4Zgsa72Ppm9kONl0RoIlPbzot9mhmeI= github.com/bkielbasa/cyclop v1.2.0/go.mod h1:qOI0yy6A7dYC4Zgsa72Ppm9kONl0RoIlPbzot9mhmeI=
@ -494,6 +498,8 @@ github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6D
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=

View File

@ -2,6 +2,7 @@ package sql
import ( import (
// Blank imports to register the drivers // Blank imports to register the drivers
_ "github.com/ClickHouse/clickhouse-go"
_ "github.com/denisenkom/go-mssqldb" _ "github.com/denisenkom/go-mssqldb"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
_ "github.com/jackc/pgx/v4/stdlib" _ "github.com/jackc/pgx/v4/stdlib"

View File

@ -270,3 +270,114 @@ func TestPostgreSQL(t *testing.T) {
}) })
} }
} }
func TestClickHouse(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
logger := testutil.Logger{}
addr := "127.0.0.1"
port := "9000"
user := "default"
if *spinup {
logger.Infof("Spinning up container...")
// Determine the test-data mountpoint
testdata, err := filepath.Abs("testdata/clickhouse")
require.NoError(t, err, "determining absolute path of test-data failed")
// Spin-up the container
ctx := context.Background()
req := testcontainers.GenericContainerRequest{
ContainerRequest: testcontainers.ContainerRequest{
Image: "yandex/clickhouse-server",
BindMounts: map[string]string{
testdata: "/docker-entrypoint-initdb.d",
},
ExposedPorts: []string{"9000/tcp", "8123/tcp"},
WaitingFor: wait.NewHTTPStrategy("/").WithPort("8123/tcp"),
},
Started: true,
}
container, err := testcontainers.GenericContainer(ctx, req)
require.NoError(t, err, "starting container failed")
defer func() {
require.NoError(t, container.Terminate(ctx), "terminating container failed")
}()
// Get the connection details from the container
addr, err = container.Host(ctx)
require.NoError(t, err, "getting container host address failed")
p, err := container.MappedPort(ctx, "9000/tcp")
require.NoError(t, err, "getting container host port failed")
port = p.Port()
}
// Define the testset
var testset = []struct {
name string
queries []Query
expected []telegraf.Metric
}{
{
name: "metric_one",
queries: []Query{
{
Query: "SELECT * FROM default.metric_one",
TagColumnsInclude: []string{"tag_*"},
FieldColumnsExclude: []string{"tag_*", "timestamp"},
TimeColumn: "timestamp",
TimeFormat: "unix",
},
},
expected: []telegraf.Metric{
testutil.MustMetric(
"sql",
map[string]string{
"tag_one": "tag1",
"tag_two": "tag2",
},
map[string]interface{}{
"int64_one": int64(1234),
"int64_two": int64(2345),
},
time.Unix(1621289085, 0),
),
},
},
}
for _, tt := range testset {
t.Run(tt.name, func(t *testing.T) {
// Setup the plugin-under-test
plugin := &SQL{
Driver: "clickhouse",
Dsn: fmt.Sprintf("tcp://%v:%v?username=%v", addr, port, user),
Queries: tt.queries,
Log: logger,
}
var acc testutil.Accumulator
// Startup the plugin
err := plugin.Init()
require.NoError(t, err)
err = plugin.Start(&acc)
require.NoError(t, err)
// Gather
err = plugin.Gather(&acc)
require.NoError(t, err)
require.Len(t, acc.Errors, 0)
// Stopping the plugin
plugin.Stop()
// Do the comparison
testutil.RequireMetricsEqual(t, tt.expected, acc.GetTelegrafMetrics())
})
}
}

View File

@ -0,0 +1,15 @@
CREATE TABLE IF NOT EXISTS default.metric_one (
tag_one String,
tag_two String,
int64_one Int64,
int64_two Int64,
timestamp Int64
) ENGINE MergeTree() ORDER BY timestamp;
INSERT INTO default.metric_one (
tag_one,
tag_two,
int64_one,
int64_two,
timestamp
) VALUES ('tag1', 'tag2', 1234, 2345, 1621289085);

View File

@ -70,7 +70,7 @@ through the convert settings.
[[outputs.sql]] [[outputs.sql]]
## Database driver ## Database driver
## Valid options: mssql (Microsoft SQL Server), mysql (MySQL), pgx (Postgres), ## Valid options: mssql (Microsoft SQL Server), mysql (MySQL), pgx (Postgres),
## sqlite (SQLite3), snowflake (snowflake.com) ## sqlite (SQLite3), snowflake (snowflake.com) clickhouse (ClickHouse)
# driver = "" # driver = ""
## Data source name ## Data source name
@ -147,6 +147,22 @@ FreeBSD, and other Linux and Darwin platforms.
The DSN is a filename or url with scheme "file:". See the [driver The DSN is a filename or url with scheme "file:". See the [driver
docs](https://modernc.org/sqlite) for details. docs](https://modernc.org/sqlite) for details.
### clickhouse
Use this metric type to SQL type conversion:
```toml
[outputs.sql.convert]
integer = "Int64"
text = "String"
timestamp = "DateTime"
defaultvalue = "String"
unsigned = "UInt64"
bool = "Uint8"
```
See [ClickHouse data types](https://clickhouse.com/docs/en/sql-reference/data-types/) for more info.
### denisenkom/go-mssqldb ### denisenkom/go-mssqldb
Telegraf doesn't have unit tests for go-mssqldb so it should be Telegraf doesn't have unit tests for go-mssqldb so it should be

View File

@ -6,10 +6,11 @@ import (
"strings" "strings"
//Register sql drivers //Register sql drivers
_ "github.com/denisenkom/go-mssqldb" // mssql (sql server) _ "github.com/ClickHouse/clickhouse-go" // clickhouse
_ "github.com/go-sql-driver/mysql" // mysql _ "github.com/denisenkom/go-mssqldb" // mssql (sql server)
_ "github.com/jackc/pgx/v4/stdlib" // pgx (postgres) _ "github.com/go-sql-driver/mysql" // mysql
_ "github.com/snowflakedb/gosnowflake" // snowflake _ "github.com/jackc/pgx/v4/stdlib" // pgx (postgres)
_ "github.com/snowflakedb/gosnowflake" // snowflake
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/outputs"
@ -116,7 +117,7 @@ func (p *SQL) deriveDatatype(value interface{}) string {
var sampleConfig = ` var sampleConfig = `
## Database driver ## Database driver
## Valid options: mssql (Microsoft SQL Server), mysql (MySQL), pgx (Postgres), ## Valid options: mssql (Microsoft SQL Server), mysql (MySQL), pgx (Postgres),
## sqlite (SQLite3), snowflake (snowflake.com) ## sqlite (SQLite3), snowflake (snowflake.com) clickhouse (ClickHouse)
# driver = "" # driver = ""
## Data source name ## Data source name
@ -223,6 +224,8 @@ func (p *SQL) tableExists(tableName string) bool {
} }
func (p *SQL) Write(metrics []telegraf.Metric) error { func (p *SQL) Write(metrics []telegraf.Metric) error {
var err error
for _, metric := range metrics { for _, metric := range metrics {
tablename := metric.Name() tablename := metric.Name()
@ -255,12 +258,33 @@ func (p *SQL) Write(metrics []telegraf.Metric) error {
} }
sql := p.generateInsert(tablename, columns) sql := p.generateInsert(tablename, columns)
_, err := p.db.Exec(sql, values...)
if err != nil { switch p.Driver {
// check if insert error was caused by column mismatch case "clickhouse":
p.Log.Errorf("Error during insert: %v, %v", err, sql) // ClickHouse needs to batch inserts with prepared statements
return err tx, err := p.db.Begin()
if err != nil {
return fmt.Errorf("begin failed: %v", err)
}
stmt, err := tx.Prepare(sql)
if err != nil {
return fmt.Errorf("prepare failed: %v", err)
}
defer stmt.Close() //nolint:revive // We cannot do anything about a failing close.
_, err = stmt.Exec(values...)
if err != nil {
return fmt.Errorf("execution failed: %v", err)
}
err = tx.Commit()
if err != nil {
return fmt.Errorf("commit failed: %v", err)
}
default:
_, err = p.db.Exec(sql, values...)
if err != nil {
return fmt.Errorf("execution failed: %v", err)
}
} }
} }
return nil return nil

View File

@ -334,3 +334,99 @@ func TestPostgresIntegration(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, string(expected), string(actual)) require.Equal(t, string(expected), string(actual))
} }
func TestClickHouseIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
initdb, err := filepath.Abs("testdata/clickhouse/initdb")
// confd, err := filepath.Abs("testdata/clickhouse/config.d")
require.NoError(t, err)
// initdb/init.sql creates this database
const dbname = "foo"
// default username for clickhouse is default
const username = "default"
outDir, err := os.MkdirTemp("", "tg-clickhouse-*")
require.NoError(t, err)
defer os.RemoveAll(outDir)
ctx := context.Background()
req := testcontainers.GenericContainerRequest{
ContainerRequest: testcontainers.ContainerRequest{
Image: "yandex/clickhouse-server",
BindMounts: map[string]string{
initdb: "/docker-entrypoint-initdb.d",
outDir: "/out",
},
ExposedPorts: []string{"9000/tcp", "8123/tcp"},
WaitingFor: wait.NewHTTPStrategy("/").WithPort("8123/tcp"),
},
Started: true,
}
cont, err := testcontainers.GenericContainer(ctx, req)
require.NoError(t, err, "starting container failed")
defer func() {
require.NoError(t, cont.Terminate(ctx), "terminating container failed")
}()
// Get the connection details from the container
host, err := cont.Host(ctx)
require.NoError(t, err, "getting container host address failed")
require.NotEmpty(t, host)
natPort, err := cont.MappedPort(ctx, "9000/tcp")
require.NoError(t, err, "getting container host port failed")
port := natPort.Port()
require.NotEmpty(t, port)
//use the plugin to write to the database
// host, port, username, password, dbname
address := fmt.Sprintf("tcp://%v:%v?username=%v&database=%v", host, port, username, dbname)
p := newSQL()
p.Log = testutil.Logger{}
p.Driver = "clickhouse"
p.DataSourceName = address
p.TableTemplate = "CREATE TABLE {TABLE}({COLUMNS}) ENGINE MergeTree() ORDER by timestamp"
p.Convert.Integer = "Int64"
p.Convert.Text = "String"
p.Convert.Timestamp = "DateTime"
p.Convert.Defaultvalue = "String"
p.Convert.Unsigned = "UInt64"
p.Convert.Bool = "UInt8"
require.NoError(t, p.Connect())
require.NoError(t, p.Write(testMetrics))
// dump the database
var rc int
for _, testMetric := range testMetrics {
rc, err = cont.Exec(ctx, []string{
"bash",
"-c",
"clickhouse-client" +
" --user=" + username +
" --database=" + dbname +
" --format=TabSeparatedRaw" +
" --multiquery --query=" +
"\"SELECT * FROM \\\"" + testMetric.Name() + "\\\";" +
"SHOW CREATE TABLE \\\"" + testMetric.Name() + "\\\"\"" +
" >> /out/dump 2>&1",
})
require.NoError(t, err)
require.Equal(t, 0, rc)
}
dumpfile := filepath.Join(outDir, "dump")
require.FileExists(t, dumpfile)
//compare the dump to what we expected
expected, err := os.ReadFile("testdata/clickhouse/expected.txt")
require.NoError(t, err)
actual, err := os.ReadFile(dumpfile)
require.NoError(t, err)
require.Equal(t, string(expected), string(actual))
}

View File

@ -0,0 +1,34 @@
2021-05-17 22:04:45 tag1 tag2 1234 2345 1 0
CREATE TABLE foo.metric_one
(
`timestamp` DateTime,
`tag_one` String,
`tag_two` String,
`int64_one` Int64,
`int64_two` Int64,
`bool_one` UInt8,
`bool_two` UInt8
)
ENGINE = MergeTree
ORDER BY timestamp
SETTINGS index_granularity = 8192
2021-05-17 22:04:45 tag3 string1
CREATE TABLE foo.metric_two
(
`timestamp` DateTime,
`tag_three` String,
`string_one` String
)
ENGINE = MergeTree
ORDER BY timestamp
SETTINGS index_granularity = 8192
2021-05-17 22:04:45 tag4 string2
CREATE TABLE foo.`metric three`
(
`timestamp` DateTime,
`tag four` String,
`string two` String
)
ENGINE = MergeTree
ORDER BY timestamp
SETTINGS index_granularity = 8192

View File

@ -0,0 +1 @@
CREATE DATABASE foo;