feat(inputs.whois): Add plugin (#16509)

Signed-off-by: Paulo Dias <paulodias.gm@gmail.com>
Co-authored-by: Sven Rebhan <36194019+srebhan@users.noreply.github.com>
This commit is contained in:
Paulo Dias 2025-03-27 14:18:23 +00:00 committed by GitHub
parent c17467b8cb
commit b5fe07de9a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 553 additions and 0 deletions

View File

@ -258,6 +258,9 @@ following works:
- github.com/kylelemons/godebug [Apache License 2.0](https://github.com/kylelemons/godebug/blob/master/LICENSE)
- github.com/leodido/go-syslog [MIT License](https://github.com/influxdata/go-syslog/blob/develop/LICENSE)
- github.com/leodido/ragel-machinery [MIT License](https://github.com/leodido/ragel-machinery/blob/develop/LICENSE)
- github.com/likexian/gokit [Apache License 2.0](https://github.com/likexian/gokit/blob/master/LICENSE)
- github.com/likexian/whois [Apache License 2.0](https://github.com/likexian/whois/blob/master/LICENSE)
- github.com/likexian/whois-parser [Apache License 2.0](https://github.com/likexian/whois-parser/blob/master/LICENSE)
- github.com/linkedin/goavro [Apache License 2.0](https://github.com/linkedin/goavro/blob/master/LICENSE)
- github.com/logzio/azure-monitor-metrics-receiver [MIT License](https://github.com/logzio/azure-monitor-metrics-receiver/blob/master/LICENSE)
- github.com/magiconair/properties [BSD 2-Clause "Simplified" License](https://github.com/magiconair/properties/blob/main/LICENSE.md)

3
go.mod
View File

@ -136,6 +136,8 @@ require (
github.com/klauspost/pgzip v1.2.6
github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b
github.com/leodido/go-syslog/v4 v4.2.0
github.com/likexian/whois v1.15.6
github.com/likexian/whois-parser v1.24.20
github.com/linkedin/goavro/v2 v2.13.1
github.com/logzio/azure-monitor-metrics-receiver v1.1.0
github.com/lxc/incus/v6 v6.9.0
@ -408,6 +410,7 @@ require (
github.com/kr/fs v0.1.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect
github.com/likexian/gokit v0.25.15 // indirect
github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a // indirect
github.com/magiconair/properties v1.8.9 // indirect
github.com/mailru/easyjson v0.9.0 // indirect

6
go.sum
View File

@ -1778,6 +1778,12 @@ github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/likexian/gokit v0.25.15 h1:QjospM1eXhdMMHwZRpMKKAHY/Wig9wgcREmLtf9NslY=
github.com/likexian/gokit v0.25.15/go.mod h1:S2QisdsxLEHWeD/XI0QMVeggp+jbxYqUxMvSBil7MRg=
github.com/likexian/whois v1.15.6 h1:hizngFHJTNQDlhwhU+FEGyPGxy8bRnf25gHDNrSB4Ag=
github.com/likexian/whois v1.15.6/go.mod h1:vx3kt3sZ4mx4XFgpaNp3GXQCZQIzAoyrUAkRtJwoM2I=
github.com/likexian/whois-parser v1.24.20 h1:oxEkRi0GxgqWQRLDMJpXU1EhgWmLmkqEFZ2ChXTeQLE=
github.com/likexian/whois-parser v1.24.20/go.mod h1:rAtaofg2luol09H+ogDzGIfcG8ig1NtM5R16uQADDz4=
github.com/linkedin/goavro/v2 v2.13.1 h1:4qZ5M0QzQFDRqccsroJlgOJznqAS/TpdvXg55h429+I=
github.com/linkedin/goavro/v2 v2.13.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
github.com/logzio/azure-monitor-metrics-receiver v1.1.0 h1:L2LU/jWTOFibZeSKUeEDBdPY6iFL1gkSE3A/9mnk/Ms=

View File

@ -0,0 +1,5 @@
//go:build !custom || inputs || inputs.whois
package all
import _ "github.com/influxdata/telegraf/plugins/inputs/whois" // register plugin

View File

@ -0,0 +1,67 @@
# WHOIS Input Plugin
This plugin queries [WHOIS information][whois] for configured
domains and provides metrics such as expiration timestamps, registrar
details and domain status from e.g. [IANA][iana] or [ICANN][icann]
servers.
⭐ Telegraf v1.35.0
🏷️ network, web
💻 all
[whois]: https://datatracker.ietf.org/doc/html/rfc3912
[icann]: https://lookup.icann.org/
[iana]: https://www.iana.org/whois
## Global configuration options <!-- @/docs/includes/plugin_config.md -->
In addition to the plugin-specific configuration settings, plugins support
additional global and plugin configuration settings. These settings are used to
modify metrics, tags, and field or create aliases and configure ordering, etc.
See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
## Configuration
```toml @sample.conf
# Reads whois data and expose as metrics
[[inputs.whois]]
## List of domains to query
domains = ["example.com", "influxdata.com"]
## Use Custom WHOIS server
# server = "whois.iana.org"
## Timeout for WHOIS queries
# timeout = "30s"
## Enable WHOIS referral chain query
# referral_chain_query = false
```
## Metrics
- whois
- tags:
- domain
- status (string)
- fields:
- creation_timestamp (int, seconds)
- dnssec_enabled (bool)
- error (string)
- expiration_timestamp (int, seconds)
- expiry (int, seconds) - Remaining time until the domain expires, in seconds.
This value can be **negative** if the domain is already expired.
`SELECT (expiry / 60 / 60 / 24) as "expiry_in_days"`
- registrar (string)
- registrant (string)
- updated_timestamp (int, seconds)
## Example Output
```text
whois,domain=example.com,status=unknown creation_timestamp=694224000i,dnssec_enabled=false,expiration_timestamp=0i,expiry=0i,name_servers="",registrant="",registrar="",updated_timestamp=0i 1741128738000000000
whois,domain=influxdata.com,status=clientTransferProhibited creation_timestamp=1403603283i,dnssec_enabled=false,expiration_timestamp=1750758483i,expiry=9629744i,name_servers="ns-1200.awsdns-22.org,ns-127.awsdns-15.com,ns-2037.awsdns-62.co.uk,ns-820.awsdns-38.net",registrant="",registrar="NameCheap, Inc.",updated_timestamp=1716620263i 1741128738000000000
whois,domain=influxdata-test.com,status=not\ found error="whoisparser: domain is not found" 1741128739000000000
```

View File

@ -0,0 +1,13 @@
# Reads whois data and expose as metrics
[[inputs.whois]]
## List of domains to query
domains = ["example.com", "influxdata.com"]
## Use Custom WHOIS server
# server = "whois.iana.org"
## Timeout for WHOIS queries
# timeout = "30s"
## Enable WHOIS referral chain query
# referral_chain_query = false

View File

@ -0,0 +1 @@
invalid domain format: "invalid..com"

View File

@ -0,0 +1,3 @@
[[inputs.whois]]
domains = ["invalid..com"]
timeout = "5s"

View File

@ -0,0 +1 @@
whois,domain=invalid-domain.xyz,status=not\ found error="whoisparser: domain is not found"

View File

@ -0,0 +1,5 @@
# whois.nic.xyz
The queried object does not exist: DOMAIN NOT FOUND
>>> Last update of WHOIS database: 2025-02-27T00:18:04.0Z <<<

View File

@ -0,0 +1,3 @@
[[inputs.whois]]
domains = ["invalid-domain.xyz"]
timeout = "5s"

View File

@ -0,0 +1,2 @@
whois,domain=example.com,status=unknown registrar="RESERVED-Internet Assigned Numbers Authority",registrant="not set",name_servers="ns1.example.com,ns2.example.com",dnssec_enabled=false,creation_timestamp=808372800i,expiration_timestamp=1912910400i,updated_timestamp=1696118400i
whois,domain=test.com,status=unknown registrar="TEST-Registrar",registrant="not set",name_servers="ns1.test.com,ns2.test.com",dnssec_enabled=false,creation_timestamp=957931200i,expiration_timestamp=2062382400i,updated_timestamp=1706745600i

View File

@ -0,0 +1,6 @@
Domain Name: EXAMPLE.COM
Registrar: RESERVED-Internet Assigned Numbers Authority
Name Server: ns1.example.com, ns2.example.com
Updated Date: 2023-10-01T00:00:00Z
Creation Date: 1995-08-14T04:00:00Z
Expiration Date: 2030-08-14T04:00:00Z

View File

@ -0,0 +1,6 @@
Domain Name: TEST.COM
Registrar: TEST-Registrar
Name Server: ns1.test.com, ns2.test.com
Updated Date: 2024-02-01T00:00:00Z
Creation Date: 2000-05-10T04:00:00Z
Expiration Date: 2035-05-10T04:00:00Z

View File

@ -0,0 +1,3 @@
[[inputs.whois]]
domains = ["example.com", "test.com"]
timeout = "5s"

View File

@ -0,0 +1 @@
whois,domain=example.com,status=unknown registrant="not set",registrar="RESERVED-Internet Assigned Numbers Authority",name_servers="ns1.example.com,ns2.example.com",dnssec_enabled=false,creation_timestamp=808358400i,expiration_timestamp=1912896000i,updated_timestamp=1704067200i,expiry=172283583i

View File

@ -0,0 +1,7 @@
Domain Name: example.com
Registrar: RESERVED-Internet Assigned Numbers Authority
Updated Date: 2024-01-01T00:00:00Z
Creation Date: 1995-08-14T00:00:00Z
Registry Expiry Date: 2030-08-14T00:00:00Z
Name Server: ns1.example.com
Name Server: ns2.example.com

View File

@ -0,0 +1,3 @@
[[inputs.whois]]
domains = ["example.com"]
timeout = "5s"

View File

@ -0,0 +1,188 @@
//go:generate ../../../tools/readme_config_includer/generator
package whois
import (
_ "embed"
"errors"
"fmt"
"regexp"
"strings"
"time"
"github.com/likexian/whois"
"github.com/likexian/whois-parser"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/plugins/inputs"
)
//go:embed sample.conf
var sampleConfig string
type Whois struct {
Domains []string `toml:"domains"`
Server string `toml:"server"`
Timeout config.Duration `toml:"timeout"`
ReferralChainQuery bool `toml:"referral_chain_query"`
Log telegraf.Logger `toml:"-"`
client *whois.Client
}
func (*Whois) SampleConfig() string {
return sampleConfig
}
func (w *Whois) Init() error {
if len(w.Domains) == 0 {
return errors.New("no domains configured")
}
if w.Timeout <= 0 {
return errors.New("timeout has to be greater than zero")
}
w.client = whois.NewClient()
w.client.SetTimeout(time.Duration(w.Timeout))
w.client.SetDisableReferralChain(!w.ReferralChainQuery)
if w.Server == "" {
w.Server = "whois.iana.org"
}
return nil
}
var domainRegex = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9-]{0,253}[a-zA-Z0-9]\.[a-zA-Z]{2,}$`)
func isValidDomain(domain string) bool {
return domainRegex.MatchString(domain)
}
func (w *Whois) Gather(acc telegraf.Accumulator) error {
for _, domain := range w.Domains {
if !isValidDomain(domain) {
acc.AddError(fmt.Errorf("invalid domain format: %q", domain))
continue
}
w.Log.Tracef("Fetching WHOIS data for %q using WHOIS server %q with timeout: %v", domain, w.Server, w.Timeout)
// Fetch WHOIS raw data
raw, err := w.client.Whois(domain, w.Server)
if err != nil {
acc.AddError(fmt.Errorf("whois query failed for %q: %w", domain, err))
continue
}
// Parse WHOIS data using whois-parser
data, err := whoisparser.Parse(raw)
if err != nil {
// Skip metric recording for these errors
if errors.Is(err, whoisparser.ErrDomainDataInvalid) {
acc.AddError(fmt.Errorf("whois parsing failed for %q: %w", domain, err))
continue
}
var status string
switch {
case errors.Is(err, whoisparser.ErrNotFoundDomain):
status = "not found"
case errors.Is(err, whoisparser.ErrReservedDomain):
status = "reserved"
case errors.Is(err, whoisparser.ErrPremiumDomain):
status = "premium"
case errors.Is(err, whoisparser.ErrBlockedDomain):
status = "blocked"
case errors.Is(err, whoisparser.ErrDomainLimitExceed):
status = "limit exceeded"
default:
status = "unknown"
}
acc.AddFields(
"whois",
map[string]interface{}{
"error": err.Error(),
},
map[string]string{
"domain": domain,
"status": status,
},
)
continue
}
// Extract expiration date
var expirationTimestamp int64
var expiry int64
if data.Domain.ExpirationDateInTime != nil {
expirationTimestamp = data.Domain.ExpirationDateInTime.Unix()
// Calculate expiry in seconds
expiry = int64(time.Until(*data.Domain.ExpirationDateInTime).Seconds())
}
// Extract creation date
var creationTimestamp int64
if data.Domain.CreatedDateInTime != nil {
creationTimestamp = data.Domain.CreatedDateInTime.Unix()
}
// Extract updated date
var updatedTimestamp int64
if data.Domain.UpdatedDateInTime != nil {
updatedTimestamp = data.Domain.UpdatedDateInTime.Unix()
}
// Extract registrar name (handle nil)
registrar := "not set"
if data.Registrar != nil {
registrar = data.Registrar.Name
}
// Extract registrant name (handle nil)
registrant := "not set"
if data.Registrant != nil {
registrant = data.Registrant.Name
}
// Extract status (handle empty)
status := "unknown"
if len(data.Domain.Status) > 0 {
status = strings.Join(data.Domain.Status, ",")
}
// Add metrics
fields := map[string]interface{}{
"creation_timestamp": creationTimestamp,
"dnssec_enabled": data.Domain.DNSSec,
"expiration_timestamp": expirationTimestamp,
"expiry": expiry,
"updated_timestamp": updatedTimestamp,
"registrar": registrar,
"registrant": registrant,
"name_servers": strings.Join(data.Domain.NameServers, ","),
}
tags := map[string]string{
"domain": domain,
"status": status,
}
acc.AddFields("whois", fields, tags)
}
return nil
}
// Plugin registration
func init() {
inputs.Add("whois", func() telegraf.Input {
return &Whois{
Timeout: config.Duration(30 * time.Second),
}
})
}

View File

@ -0,0 +1,227 @@
package whois
import (
// "errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/testutil"
)
// Make sure Whois implements telegraf.Input
var _ telegraf.Input = &Whois{}
func TestInit(t *testing.T) {
// Setup the plugin
plugin := &Whois{
Domains: []string{"example.com", "google.com"},
Server: "whois.example.org",
Timeout: config.Duration(5 * time.Second),
Log: testutil.Logger{},
}
// Test init
require.NoError(t, plugin.Init())
}
func TestInitFail(t *testing.T) {
tests := []struct {
name string
domains []string
server string
timeout config.Duration
expected string
}{
{
name: "missing domains",
timeout: config.Duration(5 * time.Second),
expected: "no domains configured",
},
{
name: "invalid timeout",
domains: []string{"example.com"},
timeout: config.Duration(0),
expected: "timeout has to be greater than zero",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Setup the plugin
plugin := &Whois{
Domains: tt.domains,
Server: tt.server,
Timeout: tt.timeout,
Log: testutil.Logger{},
}
// Test for the expected error message
require.ErrorContains(t, plugin.Init(), tt.expected)
})
}
}
func TestCases(t *testing.T) {
// Get all directories in testcases
folders, err := os.ReadDir("testcases")
require.NoError(t, err, "failed to read testcases directory")
// Prepare the influx parser for expectations
parser := &influx.Parser{}
require.NoError(t, parser.Init())
for _, f := range folders {
if !f.IsDir() {
continue
}
testcasePath := filepath.Join("testcases", f.Name())
configFilename := filepath.Join(testcasePath, "telegraf.conf")
expectedFilename := filepath.Join(testcasePath, "expected.out")
expectedErrorFilename := filepath.Join(testcasePath, "expected.err")
// Compare options for metrics
options := []cmp.Option{
testutil.IgnoreTime(),
testutil.SortMetrics(),
// Ignore `expiry` due to possibility of fail on tests if CI is under high load
testutil.IgnoreFields("expiry"),
}
t.Run(f.Name(), func(t *testing.T) {
// Create and start a mock WHOIS server
mockServer, err := createMockServer(testcasePath)
require.NoError(t, err, "failed to create mock WHOIS server")
mockServerAddr, err := mockServer.start()
require.NoError(t, err, "failed to start mock WHOIS server")
defer mockServer.stop() // Ensure cleanup
// Read expected output
var expectedMetrics []telegraf.Metric
if _, err := os.Stat(expectedFilename); err == nil {
expectedMetrics, err = testutil.ParseMetricsFromFile(expectedFilename, parser)
require.NoError(t, err)
}
// Read expected errors
var expectedErrors []string
if _, err := os.Stat(expectedErrorFilename); err == nil {
expectedErrors, err = testutil.ParseLinesFromFile(expectedErrorFilename)
require.NoError(t, err)
}
// Load Telegraf plugin config
cfg := config.NewConfig()
require.NoError(t, cfg.LoadConfig(configFilename))
require.Len(t, cfg.Inputs, 1)
// Get WHOIS plugin instance
plugin := cfg.Inputs[0].Input.(*Whois)
plugin.Server = mockServerAddr
require.NoError(t, plugin.Init())
var acc testutil.Accumulator
require.NoError(t, plugin.Gather(&acc))
var actualErrorMsgs []string
for _, err := range acc.Errors {
actualErrorMsgs = append(actualErrorMsgs, err.Error())
}
require.ElementsMatch(t, actualErrorMsgs, expectedErrors)
// Compare expected metrics
actualMetrics := acc.GetTelegrafMetrics()
testutil.RequireMetricsEqual(t, expectedMetrics, actualMetrics, options...)
})
}
}
type server struct {
responses map[string][]byte
listener net.Listener
errors []error
sync.Mutex
}
func createMockServer(path string) (*server, error) {
// Read the input data
matches, err := filepath.Glob(filepath.Join(path, "input_*.txt"))
if err != nil {
return nil, fmt.Errorf("matching input files failed: %w", err)
}
responses := make(map[string][]byte, len(matches))
for _, fn := range matches {
buf, err := os.ReadFile(fn)
if err != nil {
return nil, fmt.Errorf("reading %q failed: %w", fn, err)
}
domain := strings.TrimPrefix(filepath.Base(fn), "input_")
domain = strings.TrimSuffix(domain, ".txt")
responses[domain] = buf
}
return &server{responses: responses}, nil
}
func (s *server) start() (string, error) {
// Create the listener
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return "", fmt.Errorf("starting server failed: %w", err)
}
s.listener = listener
addr := listener.Addr().String()
go func() {
for {
conn, err := s.listener.Accept()
if err != nil {
return // Stop accepting new connections on shutdown
}
go func(c net.Conn) {
defer c.Close()
// Read the requested domain
buf := make([]byte, 1024)
n, err := c.Read(buf)
if err != nil {
return
}
domain := strings.TrimSpace(string(buf[:n]))
// Write the response from the input data or an error if the domain cannot be found
response, found := s.responses[domain]
if !found {
response = []byte("ERROR: No data available\n")
}
if _, err := c.Write(response); err != nil {
s.Lock()
s.errors = append(s.errors, fmt.Errorf("writing response %q failed: %w", domain, err))
s.Unlock()
}
}(conn)
}
}()
return addr, nil
}
func (s *server) stop() {
if s.listener != nil {
s.listener.Close()
s.listener = nil
}
}