chore: Add generator for sample-config includes (#13348)

.gitignore

@@ -6,7 +6,10 @@
 /telegraf.gz
 /tools/package_lxd_test/package_lxd_test
 /tools/license_checker/license_checker*
-/tools/readme_config_includer/generator*
+/tools/readme_config_includer/generator
+/tools/readme_config_includer/generator.exe
+/tools/config_includer/generator
+/tools/config_includer/generator.exe
 /tools/readme_linter/readme_linter*
 /tools/custom_builder/custom_builder*
 /vendor

Makefile

@@ -112,10 +112,12 @@ build_tools:
 	$(HOSTGO) build -o ./tools/custom_builder/custom_builder$(EXEEXT) ./tools/custom_builder
 	$(HOSTGO) build -o ./tools/license_checker/license_checker$(EXEEXT) ./tools/license_checker
 	$(HOSTGO) build -o ./tools/readme_config_includer/generator$(EXEEXT) ./tools/readme_config_includer/generator.go
+	$(HOSTGO) build -o ./tools/config_includer/generator$(EXEEXT) ./tools/config_includer/generator.go
 	$(HOSTGO) build -o ./tools/readme_linter/readme_linter$(EXEEXT) ./tools/readme_linter
 
 embed_readme_%:
-	go generate -run="readme_config_includer/generator$$" ./plugins/$*/...
+	go generate -run="tools/config_includer/generator" ./plugins/$*/...
+	go generate -run="tools/readme_config_includer/generator" ./plugins/$*/...
 
 .PHONY: config
 config:

plugins/common/tls/client.conf

@@ -0,0 +1,13 @@
+  ## Set to true/false to enforce TLS being enabled/disabled. If not set,
+  ## enable TLS only if any of the other options are specified.
+  # tls_enable =
+  ## Trusted root certificates for server
+  # tls_ca = "/path/to/cafile"
+  ## Used for TLS client certificate authentication
+  # tls_cert = "/path/to/certfile"
+  ## Used for TLS client certificate authentication
+  # tls_key = "/path/to/keyfile"
+  ## Send the specified TLS server name via SNI
+  # tls_server_name = "kubernetes.example.com"
+  ## Use TLS but skip chain & host verification
+  # insecure_skip_verify = false

plugins/inputs/http/README.md

@@ -66,11 +66,17 @@ to use them.
   # http_proxy_url = ""
 
   ## Optional TLS Config
-  # tls_ca = "/etc/telegraf/ca.pem"
+  ## Set to true/false to enforce TLS being enabled/disabled. If not set,
-  # tls_cert = "/etc/telegraf/cert.pem"
+  ## enable TLS only if any of the other options are specified.
-  # tls_key = "/etc/telegraf/key.pem"
+  # tls_enable =
-  ## Minimal TLS version to accept by the client
+  ## Trusted root certificates for server
-  # tls_min_version = "TLS12"
+  # tls_ca = "/path/to/cafile"
+  ## Used for TLS client certificate authentication
+  # tls_cert = "/path/to/certfile"
+  ## Used for TLS client certificate authentication
+  # tls_key = "/path/to/keyfile"
+  ## Send the specified TLS server name via SNI
+  # tls_server_name = "kubernetes.example.com"
   ## Use TLS but skip chain & host verification
   # insecure_skip_verify = false
 

plugins/inputs/http/http.go

@@ -1,3 +1,4 @@
+//go:generate ../../../tools/config_includer/generator
 //go:generate ../../../tools/readme_config_includer/generator
 package http
 

plugins/inputs/http/sample.conf

@@ -37,11 +37,17 @@
   # http_proxy_url = ""
 
   ## Optional TLS Config
-  # tls_ca = "/etc/telegraf/ca.pem"
+  ## Set to true/false to enforce TLS being enabled/disabled. If not set,
-  # tls_cert = "/etc/telegraf/cert.pem"
+  ## enable TLS only if any of the other options are specified.
-  # tls_key = "/etc/telegraf/key.pem"
+  # tls_enable =
-  ## Minimal TLS version to accept by the client
+  ## Trusted root certificates for server
-  # tls_min_version = "TLS12"
+  # tls_ca = "/path/to/cafile"
+  ## Used for TLS client certificate authentication
+  # tls_cert = "/path/to/certfile"
+  ## Used for TLS client certificate authentication
+  # tls_key = "/path/to/keyfile"
+  ## Send the specified TLS server name via SNI
+  # tls_server_name = "kubernetes.example.com"
   ## Use TLS but skip chain & host verification
   # insecure_skip_verify = false
 

plugins/inputs/http/sample.conf.in

@@ -0,0 +1,63 @@
+# Read formatted metrics from one or more HTTP endpoints
+[[inputs.http]]
+  ## One or more URLs from which to read formatted metrics
+  urls = [
+    "http://localhost/metrics"
+  ]
+
+  ## HTTP method
+  # method = "GET"
+
+  ## Optional HTTP headers
+  # headers = {"X-Special-Header" = "Special-Value"}
+
+  ## HTTP entity-body to send with POST/PUT requests.
+  # body = ""
+
+  ## HTTP Content-Encoding for write request body, can be set to "gzip" to
+  ## compress body or "identity" to apply no encoding.
+  # content_encoding = "identity"
+
+  ## Optional file with Bearer token
+  ## file content is added as an Authorization header
+  # bearer_token = "/path/to/file"
+
+  ## Optional HTTP Basic Auth Credentials
+  # username = "username"
+  # password = "pa$$word"
+
+  ## OAuth2 Client Credentials. The options 'client_id', 'client_secret', and 'token_url' are required to use OAuth2.
+  # client_id = "clientid"
+  # client_secret = "secret"
+  # token_url = "https://indentityprovider/oauth2/v1/token"
+  # scopes = ["urn:opc:idm:__myscopes__"]
+
+  ## HTTP Proxy support
+  # use_system_proxy = false
+  # http_proxy_url = ""
+
+  ## Optional TLS Config
+{{template "/plugins/common/tls/client.conf"}}
+
+  ## Optional Cookie authentication
+  # cookie_auth_url = "https://localhost/authMe"
+  # cookie_auth_method = "POST"
+  # cookie_auth_username = "username"
+  # cookie_auth_password = "pa$$word"
+  # cookie_auth_headers = { Content-Type = "application/json", X-MY-HEADER = "hello" }
+  # cookie_auth_body = '{"username": "user", "password": "pa$$word", "authenticate": "me"}'
+  ## cookie_auth_renewal not set or set to "0" will auth once and never renew the cookie
+  # cookie_auth_renewal = "5m"
+
+  ## Amount of time allowed to complete the HTTP request
+  # timeout = "5s"
+
+  ## List of success status codes
+  # success_status_codes = [200]
+
+  ## Data format to consume.
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  # data_format = "influx"
+

plugins/inputs/modbus/sample_general_begin.conf

@@ -50,4 +50,3 @@
   ##  |---register -- define fields per register type in the original style (only supports one slave ID)
   ##  |---request  -- define fields on a requests base
   configuration_type = "register"
-

plugins/inputs/modbus/sample_register.conf

@@ -49,4 +49,3 @@
     { name = "tank_ph",      byte_order = "AB",   data_type = "INT16",   scale=1.0,     address = [1]},
     { name = "pump1_speed",  byte_order = "ABCD", data_type = "INT32",   scale=1.0,     address = [3,4]},
   ]
-

plugins/inputs/modbus/sample_request.conf

@@ -115,4 +115,3 @@
     [inputs.modbus.request.tags]
       machine = "impresser"
       location = "main building"
-

tools/config_includer/generator.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"text/template/parse"
+)
+
+func extractIncludes(tmpl *template.Template) []string {
+	var includes []string
+	for _, node := range tmpl.Root.Nodes {
+		if n, ok := node.(*parse.TemplateNode); ok {
+			includes = append(includes, n.Name)
+		}
+	}
+	return includes
+}
+
+func absolutePath(root, fn string) (string, error) {
+	pwd, err := filepath.Abs(fn)
+	if err != nil {
+		return "", fmt.Errorf("cannot determine absolute location of %q: %w", fn, err)
+	}
+	pwd, err = filepath.Rel(root, filepath.Dir(pwd))
+	if err != nil {
+		return "", fmt.Errorf("Cannot determine location of %q relative to %q: %w", pwd, root, err)
+	}
+	return string(filepath.Separator) + pwd, nil
+}
+
+func main() {
+	// Estimate Telegraf root to be able to handle absolute paths
+	cwd, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("Cannot get working directory: %v", err)
+	}
+	cwd, err = filepath.Abs(cwd)
+	if err != nil {
+		log.Fatalf("Cannot resolve working directory: %v", err)
+	}
+
+	var root string
+	idx := strings.LastIndex(cwd, filepath.FromSlash("/plugins/"))
+	if idx <= 0 {
+		log.Fatalln("Cannot determine include root!")
+	}
+	root = cwd[:idx]
+
+	var parent, inputFilename, outputFilename string
+	switch len(os.Args) {
+	case 1:
+		parent = strings.TrimPrefix(filepath.ToSlash(cwd[idx:]), "/plugins/")
+		parent = strings.ReplaceAll(parent, "/", ".")
+		inputFilename = "sample.conf.in"
+		outputFilename = "sample.conf"
+	case 2:
+		parent = os.Args[1]
+		inputFilename = "sample.conf.in"
+		outputFilename = "sample.conf"
+	case 3:
+		parent = os.Args[1]
+		inputFilename = os.Args[2]
+		if !strings.HasSuffix(inputFilename, ".in") {
+			log.Fatalf("Template filename %q does not have '.in' suffix!", inputFilename)
+		}
+		outputFilename = strings.TrimSuffix(inputFilename, ".in")
+	case 4:
+		parent = os.Args[1]
+		inputFilename = os.Args[2]
+		outputFilename = os.Args[3]
+	default:
+		log.Fatalln("Invalid number of arguments")
+	}
+
+	roottmpl := template.New(inputFilename)
+	known := make(map[string]bool)
+	inroot, err := absolutePath(root, inputFilename)
+	if err != nil {
+		log.Fatal(err)
+	}
+	unresolved := map[string]string{inputFilename: filepath.Join(inroot, inputFilename)}
+	for {
+		if len(unresolved) == 0 {
+			break
+		}
+
+		newUnresolved := make(map[string]string)
+		for name, fn := range unresolved {
+			if filepath.IsAbs(fn) {
+				fn = filepath.Join(root, fn)
+			}
+
+			if known[name] {
+				// Include already resolved, skipping
+				continue
+			}
+
+			tmpl, err := template.ParseFiles(fn)
+			if err != nil {
+				log.Fatalf("Reading template %q failed: %v", fn, err)
+			}
+			known[name] = true
+			if _, err := roottmpl.AddParseTree(name, tmpl.Tree); err != nil {
+				log.Fatalf("Adding include %q failed: %v", fn, err)
+			}
+
+			// For relative paths we need to make it relative to the include
+			pwd, err := filepath.Abs(fn)
+			if err != nil {
+				log.Fatalf("Cannot determine absolute location of %q: %v", fn, err)
+			}
+			pwd, err = filepath.Rel(root, filepath.Dir(pwd))
+			if err != nil {
+				log.Fatalf("Cannot determine location of %q relative to %q: %v", pwd, root, err)
+			}
+			pwd = string(filepath.Separator) + pwd
+			for _, iname := range extractIncludes(tmpl) {
+				ifn := iname
+				if !filepath.IsAbs(ifn) {
+					ifn = filepath.Join(pwd, ifn)
+				}
+				newUnresolved[iname] = ifn
+			}
+		}
+		unresolved = newUnresolved
+	}
+
+	defines := map[string]string{"parent": parent}
+	var buf bytes.Buffer
+	if err := roottmpl.Execute(&buf, defines); err != nil {
+		log.Fatalf("Executing template failed: %v", err)
+	}
+
+	if err := os.WriteFile(outputFilename, buf.Bytes(), 0640); err != nil {
+		log.Fatalf("Writing output %q failed: %v", outputFilename, err)
+	}
+}

tools/readme_config_includer/generator.go

@@ -99,7 +99,13 @@ func insertIncludes(buf *bytes.Buffer, b *includeBlock) error {
 	}
 
 	// Insert all includes in the order they occurred
-	for _, include := range b.Includes {
+	for i, include := range b.Includes {
+		if i > 0 {
+			// Add a separating newline between included blocks
+			if _, err := buf.Write([]byte("\n")); err != nil {
+				return errors.New("adding newline failed")
+			}
+		}
 		if err := insertInclude(buf, include); err != nil {
 			return err
 		}