test(inputs.zfs): Rework and fix unit-tests for FreeBSD (#14175)

This commit is contained in:
Sven Rebhan 2023-10-24 18:08:54 +02:00 committed by GitHub
parent ab24e78c20
commit 06083bf19b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 233 additions and 235 deletions

View File

@ -0,0 +1 @@
zfs,pools=freenas-boot::red1::temp1::temp2 vdev_cache_stats_delegations=6952i,vdev_cache_stats_hits=465583i,vdev_cache_stats_misses=87789i 1698097327984312950

View File

@ -0,0 +1,12 @@
{
"vdev_cache_stats": [
"kstat.zfs.misc.vdev_cache_stats.misses: 87789",
"kstat.zfs.misc.vdev_cache_stats.hits: 465583",
"kstat.zfs.misc.vdev_cache_stats.delegations: 6952"
],
"zfetchstats": [
"kstat.zfs.misc.zfetchstats.max_streams: 0",
"kstat.zfs.misc.zfetchstats.misses: 0",
"kstat.zfs.misc.zfetchstats.hits: 0"
]
}

View File

@ -0,0 +1,2 @@
[[inputs.zfs]]
kstatMetrics = ["vdev_cache_stats"]

View File

@ -0,0 +1,4 @@
freenas-boot ONLINE 30601641984 2022177280 28579464704 - 6 1.00x
red1 ONLINE 8933531975680 1126164848640 7807367127040 8% 12 1.83x
temp1 ONLINE 2989297238016 1626309320704 1362987917312 38% 54 1.28x
temp2 ONLINE 2989297238016 626958278656 2362338959360 12% 20 1.00x

View File

@ -0,0 +1,5 @@
zfs_pool,health=ONLINE,pool=freenas-boot allocated=2022177280i,capacity=6i,dedupratio=1,fragmentation=0i,free=28579464704i,size=30601641984i 1698097244832121819
zfs_pool,health=ONLINE,pool=red1 allocated=1126164848640i,capacity=12i,dedupratio=1.8300000429153442,fragmentation=8i,free=7807367127040i,size=8933531975680i 1698097244832126568
zfs_pool,health=ONLINE,pool=temp1 allocated=1626309320704i,capacity=54i,dedupratio=1.2799999713897705,fragmentation=38i,free=1362987917312i,size=2989297238016i 1698097244832130759
zfs_pool,health=ONLINE,pool=temp2 allocated=626958278656i,capacity=20i,dedupratio=1,fragmentation=12i,free=2362338959360i,size=2989297238016i 1698097244832134111
zfs,pools=freenas-boot::red1::temp1::temp2 vdev_cache_stats_delegations=6952i,vdev_cache_stats_hits=465583i,vdev_cache_stats_misses=87789i 1698097244832209819

View File

@ -0,0 +1,12 @@
{
"vdev_cache_stats": [
"kstat.zfs.misc.vdev_cache_stats.misses: 87789",
"kstat.zfs.misc.vdev_cache_stats.hits: 465583",
"kstat.zfs.misc.vdev_cache_stats.delegations: 6952"
],
"zfetchstats": [
"kstat.zfs.misc.zfetchstats.max_streams: 0",
"kstat.zfs.misc.zfetchstats.misses: 0",
"kstat.zfs.misc.zfetchstats.hits: 0"
]
}

View File

@ -0,0 +1,3 @@
[[inputs.zfs]]
kstatMetrics = ["vdev_cache_stats"]
poolMetrics = true

View File

@ -0,0 +1,4 @@
freenas-boot ONLINE 30601641984 2022177280 28579464704 - 6 1.00x
red1 ONLINE 8933531975680 1126164848640 7807367127040 8% 12 1.83x
temp1 ONLINE 2989297238016 1626309320704 1362987917312 38% 54 1.28x
temp2 ONLINE 2989297238016 626958278656 2362338959360 12% 20 1.00x

View File

@ -0,0 +1 @@
zfs,datasets=zata::zata/home::zata/import::zata/storage vdev_cache_stats_delegations=6952i,vdev_cache_stats_hits=465583i,vdev_cache_stats_misses=87789i 1698161813995801840

View File

@ -0,0 +1,12 @@
{
"vdev_cache_stats": [
"kstat.zfs.misc.vdev_cache_stats.misses: 87789",
"kstat.zfs.misc.vdev_cache_stats.hits: 465583",
"kstat.zfs.misc.vdev_cache_stats.delegations: 6952"
],
"zfetchstats": [
"kstat.zfs.misc.zfetchstats.max_streams: 0",
"kstat.zfs.misc.zfetchstats.misses: 0",
"kstat.zfs.misc.zfetchstats.hits: 0"
]
}

View File

@ -0,0 +1,2 @@
[[inputs.zfs]]
kstatMetrics = ["vdev_cache_stats"]

View File

@ -0,0 +1,4 @@
zata 10741741326336 8564135526400 0 90112
zata/home 10741741326336 2498560 212992 2285568
zata/import 10741741326336 196608 81920 114688
zata/storage 10741741326336 8556084379648 3601138999296 4954945380352

View File

@ -0,0 +1,5 @@
zfs_dataset,dataset=zata avail=10741741326336i,used=8564135526400i,usedds=90112i,usedsnap=0i 1698161697336342048
zfs_dataset,dataset=zata/home avail=10741741326336i,used=2498560i,usedds=2285568i,usedsnap=212992i 1698161697336345401
zfs_dataset,dataset=zata/import avail=10741741326336i,used=196608i,usedds=114688i,usedsnap=81920i 1698161697336348474
zfs_dataset,dataset=zata/storage avail=10741741326336i,used=8556084379648i,usedds=4954945380352i,usedsnap=3601138999296i 1698161697336352944
zfs,datasets=zata::zata/home::zata/import::zata/storage vdev_cache_stats_delegations=6952i,vdev_cache_stats_hits=465583i,vdev_cache_stats_misses=87789i 1698161697336357134

View File

@ -0,0 +1,12 @@
{
"vdev_cache_stats": [
"kstat.zfs.misc.vdev_cache_stats.misses: 87789",
"kstat.zfs.misc.vdev_cache_stats.hits: 465583",
"kstat.zfs.misc.vdev_cache_stats.delegations: 6952"
],
"zfetchstats": [
"kstat.zfs.misc.zfetchstats.max_streams: 0",
"kstat.zfs.misc.zfetchstats.misses: 0",
"kstat.zfs.misc.zfetchstats.hits: 0"
]
}

View File

@ -0,0 +1,3 @@
[[inputs.zfs]]
kstatMetrics = ["vdev_cache_stats"]
datasetMetrics = true

View File

@ -0,0 +1,4 @@
zata 10741741326336 8564135526400 0 90112
zata/home 10741741326336 2498560 212992 2285568
zata/import 10741741326336 196608 81920 114688
zata/storage 10741741326336 8556084379648 3601138999296 4954945380352

View File

@ -0,0 +1 @@
zfs,pools=freenas-boot::red1::temp1::temp2 vdev_cache_stats_delegations=6952i,vdev_cache_stats_hits=465583i,vdev_cache_stats_misses=87789i 1698157432411179772

View File

@ -0,0 +1,12 @@
{
"vdev_cache_stats": [
"kstat.zfs.misc.vdev_cache_stats.misses: 87789",
"kstat.zfs.misc.vdev_cache_stats.hits: 465583",
"kstat.zfs.misc.vdev_cache_stats.delegations: 6952"
],
"zfetchstats": [
"kstat.zfs.misc.zfetchstats.max_streams: 0",
"kstat.zfs.misc.zfetchstats.misses: 0",
"kstat.zfs.misc.zfetchstats.hits: 0"
]
}

View File

@ -0,0 +1,2 @@
[[inputs.zfs]]
kstatMetrics = ["vdev_cache_stats"]

View File

@ -0,0 +1,4 @@
freenas-boot ONLINE 30601641984 2022177280 28579464704 - 6 1.00x
red1 ONLINE 8933531975680 1126164848640 7807367127040 8% 12 1.83x
temp1 ONLINE 2989297238016 1626309320704 1362987917312 38% 54 1.28x
temp2 ONLINE 2989297238016 626958278656 2362338959360 12% 20 1.00x

View File

@ -0,0 +1 @@
zfs,pools=freenas-boot::red1::temp1::temp2 vdev_cache_stats_delegations=6952i,vdev_cache_stats_hits=465583i,vdev_cache_stats_misses=87789i,zfetchstats_hits=0i,zfetchstats_max_streams=0i,zfetchstats_misses=0i 1698157264111374629

View File

@ -0,0 +1,12 @@
{
"vdev_cache_stats": [
"kstat.zfs.misc.vdev_cache_stats.misses: 87789",
"kstat.zfs.misc.vdev_cache_stats.hits: 465583",
"kstat.zfs.misc.vdev_cache_stats.delegations: 6952"
],
"zfetchstats": [
"kstat.zfs.misc.zfetchstats.max_streams: 0",
"kstat.zfs.misc.zfetchstats.misses: 0",
"kstat.zfs.misc.zfetchstats.hits: 0"
]
}

View File

@ -0,0 +1,2 @@
[[inputs.zfs]]
kstatMetrics = ["zfetchstats", "vdev_cache_stats"]

View File

@ -0,0 +1,4 @@
freenas-boot ONLINE 30601641984 2022177280 28579464704 - 6 1.00x
red1 ONLINE 8933531975680 1126164848640 7807367127040 8% 12 1.83x
temp1 ONLINE 2989297238016 1626309320704 1362987917312 38% 54 1.28x
temp2 ONLINE 2989297238016 626958278656 2362338959360 12% 20 1.00x

View File

@ -0,0 +1 @@
zfs,pools=temp2 vdev_cache_stats_delegations=6952i,vdev_cache_stats_hits=465583i,vdev_cache_stats_misses=87789i 1698098260907108724

View File

@ -0,0 +1,12 @@
{
"vdev_cache_stats": [
"kstat.zfs.misc.vdev_cache_stats.misses: 87789",
"kstat.zfs.misc.vdev_cache_stats.hits: 465583",
"kstat.zfs.misc.vdev_cache_stats.delegations: 6952"
],
"zfetchstats": [
"kstat.zfs.misc.zfetchstats.max_streams: 0",
"kstat.zfs.misc.zfetchstats.misses: 0",
"kstat.zfs.misc.zfetchstats.hits: 0"
]
}

View File

@ -0,0 +1,2 @@
[[inputs.zfs]]
kstatMetrics = ["vdev_cache_stats"]

View File

@ -0,0 +1 @@
temp2 UNAVAIL - - - - - -

View File

@ -0,0 +1,2 @@
zfs_pool,health=UNAVAIL,pool=temp2 size=0i 1698098260907383061
zfs,pools=temp2 vdev_cache_stats_delegations=6952i,vdev_cache_stats_hits=465583i,vdev_cache_stats_misses=87789i 1698098260907393118

View File

@ -0,0 +1,12 @@
{
"vdev_cache_stats": [
"kstat.zfs.misc.vdev_cache_stats.misses: 87789",
"kstat.zfs.misc.vdev_cache_stats.hits: 465583",
"kstat.zfs.misc.vdev_cache_stats.delegations: 6952"
],
"zfetchstats": [
"kstat.zfs.misc.zfetchstats.max_streams: 0",
"kstat.zfs.misc.zfetchstats.misses: 0",
"kstat.zfs.misc.zfetchstats.hits: 0"
]
}

View File

@ -0,0 +1,3 @@
[[inputs.zfs]]
kstatMetrics = ["vdev_cache_stats"]
poolMetrics = true

View File

@ -0,0 +1 @@
temp2 UNAVAIL - - - - - -

View File

@ -14,7 +14,6 @@ import (
)
func (z *Zfs) gatherPoolStats(acc telegraf.Accumulator) (string, error) {
lines, err := z.zpool()
if err != nil {
return "", err
@ -38,11 +37,8 @@ func (z *Zfs) gatherPoolStats(acc telegraf.Accumulator) (string, error) {
fields := map[string]interface{}{}
if tags["health"] == "UNAVAIL" {
fields["size"] = int64(0)
} else {
size, err := strconv.ParseInt(col[2], 10, 64)
if err != nil {
return "", fmt.Errorf("Error parsing size: %s", err)
@ -98,7 +94,6 @@ func (z *Zfs) gatherDatasetStats(acc telegraf.Accumulator) (string, error) {
datasets := []string{}
for _, line := range lines {
col := strings.Split(line, "\t")
datasets = append(datasets, col[0])
}
@ -139,12 +134,17 @@ func (z *Zfs) Gather(acc telegraf.Accumulator) error {
if err != nil {
return err
}
if poolNames != "" {
tags["pools"] = poolNames
}
datasetNames, err := z.gatherDatasetStats(acc)
if err != nil {
return err
}
if datasetNames != "" {
tags["datasets"] = datasetNames
}
fields := make(map[string]interface{})
for _, metric := range kstatMetrics {

View File

@ -3,250 +3,90 @@
package zfs
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
"github.com/influxdata/telegraf/testutil"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/testutil"
)
// $ zpool list -Hp -o name,health,size,alloc,free,fragmentation,capacity,dedupratio
var zpool_output = []string{
"freenas-boot ONLINE 30601641984 2022177280 28579464704 - 6 1.00x",
"red1 ONLINE 8933531975680 1126164848640 7807367127040 8% 12 1.83x",
"temp1 ONLINE 2989297238016 1626309320704 1362987917312 38% 54 1.28x",
"temp2 ONLINE 2989297238016 626958278656 2362338959360 12% 20 1.00x",
// Generate testcase-data via
//
// zpool.txt: $ zpool list -Hp -o name,health,size,alloc,free,fragmentation,capacity,dedupratio
// zdataset.txt: $ zfs list -Hp -o name,avail,used,usedsnap,usedds
// sysctl.json: $ sysctl -q kstat.zfs.misc.<kstat metrics>
func TestCases(t *testing.T) {
// Get all testcase directories
testpath := filepath.Join("testcases", "freebsd")
folders, err := os.ReadDir(testpath)
require.NoError(t, err)
// Register the plugin
inputs.Add("zfs", func() telegraf.Input { return &Zfs{} })
for _, f := range folders {
// Only handle folders
if !f.IsDir() {
continue
}
func mock_zpool() ([]string, error) {
return zpool_output, nil
}
t.Run(f.Name(), func(t *testing.T) {
testcasePath := filepath.Join(testpath, f.Name())
configFilename := filepath.Join(testcasePath, "telegraf.conf")
inputSysctlFilename := filepath.Join(testcasePath, "sysctl.json")
inputZPoolFilename := filepath.Join(testcasePath, "zpool.txt")
inputZDatasetFilename := filepath.Join(testcasePath, "zdataset.txt")
expectedFilename := filepath.Join(testcasePath, "expected.out")
// $ zpool list -Hp -o name,health,size,alloc,free,fragmentation,capacity,dedupratio
var zpool_output_unavail = []string{
"temp2 UNAVAIL - - - - - -",
}
// Load the input data
buf, err := os.ReadFile(inputSysctlFilename)
require.NoError(t, err)
var sysctl map[string][]string
require.NoError(t, json.Unmarshal(buf, &sysctl))
zpool, err := testutil.ParseLinesFromFile(inputZPoolFilename)
require.NoError(t, err)
zdataset, err := testutil.ParseLinesFromFile(inputZDatasetFilename)
require.NoError(t, err)
func mock_zpool_unavail() ([]string, error) {
return zpool_output_unavail, nil
}
// Prepare the influx parser for expectations
parser := &influx.Parser{}
require.NoError(t, parser.Init())
// $ zfs list -Hp -o name,avail,used,usedsnap,usedds
var zdataset_output = []string{
"zata 10741741326336 8564135526400 0 90112",
"zata/home 10741741326336 2498560 212992 2285568",
"zata/import 10741741326336 196608 81920 114688",
"zata/storage 10741741326336 8556084379648 3601138999296 4954945380352",
}
// Read the expected output
expected, err := testutil.ParseMetricsFromFile(expectedFilename, parser)
require.NoError(t, err)
func mock_zdataset() ([]string, error) {
return zdataset_output, nil
}
// Configure the plugin
cfg := config.NewConfig()
require.NoError(t, cfg.LoadConfig(configFilename))
require.Len(t, cfg.Inputs, 1)
// sysctl -q kstat.zfs.misc.arcstats
// Setup the plugin
plugin := cfg.Inputs[0].Input.(*Zfs)
plugin.sysctl = func(metric string) ([]string, error) {
if r, found := sysctl[metric]; found {
return r, nil
}
return nil, fmt.Errorf("invalid argument")
}
plugin.zpool = func() ([]string, error) { return zpool, nil }
plugin.zdataset = func(_ []string) ([]string, error) { return zdataset, nil }
plugin.Log = testutil.Logger{}
// sysctl -q kstat.zfs.misc.vdev_cache_stats
var kstat_vdev_cache_stats_output = []string{
"kstat.zfs.misc.vdev_cache_stats.misses: 87789",
"kstat.zfs.misc.vdev_cache_stats.hits: 465583",
"kstat.zfs.misc.vdev_cache_stats.delegations: 6952",
}
// sysctl -q kstat.zfs.misc.zfetchstats
var kstat_zfetchstats_output = []string{
"kstat.zfs.misc.zfetchstats.max_streams: 0",
"kstat.zfs.misc.zfetchstats.misses: 0",
"kstat.zfs.misc.zfetchstats.hits: 0",
}
func mock_sysctl(metric string) ([]string, error) {
if metric == "vdev_cache_stats" {
return kstat_vdev_cache_stats_output, nil
}
if metric == "zfetchstats" {
return kstat_zfetchstats_output, nil
}
return []string{}, fmt.Errorf("Invalid arg")
}
func TestZfsPoolMetrics(t *testing.T) {
// Gather and test
var acc testutil.Accumulator
require.NoError(t, plugin.Gather(&acc))
z := &Zfs{
KstatMetrics: []string{"vdev_cache_stats"},
sysctl: mock_sysctl,
zpool: mock_zpool,
}
err := z.Gather(&acc)
require.NoError(t, err)
require.False(t, acc.HasMeasurement("zfs_pool"))
acc.Metrics = nil
z = &Zfs{
KstatMetrics: []string{"vdev_cache_stats"},
PoolMetrics: true,
sysctl: mock_sysctl,
zpool: mock_zpool,
}
err = z.Gather(&acc)
require.NoError(t, err)
//one pool, all metrics
tags := map[string]string{
"pool": "freenas-boot",
"health": "ONLINE",
}
poolMetrics := getFreeNasBootPoolMetrics()
acc.AssertContainsTaggedFields(t, "zfs_pool", poolMetrics, tags)
}
func TestZfsPoolMetrics_unavail(t *testing.T) {
var acc testutil.Accumulator
z := &Zfs{
KstatMetrics: []string{"vdev_cache_stats"},
sysctl: mock_sysctl,
zpool: mock_zpool_unavail,
}
err := z.Gather(&acc)
require.NoError(t, err)
require.False(t, acc.HasMeasurement("zfs_pool"))
acc.Metrics = nil
z = &Zfs{
KstatMetrics: []string{"vdev_cache_stats"},
PoolMetrics: true,
sysctl: mock_sysctl,
zpool: mock_zpool_unavail,
}
err = z.Gather(&acc)
require.NoError(t, err)
//one pool, UNAVAIL
tags := map[string]string{
"pool": "temp2",
"health": "UNAVAIL",
}
poolMetrics := getTemp2PoolMetrics()
acc.AssertContainsTaggedFields(t, "zfs_pool", poolMetrics, tags)
}
func TestZfsDatasetMetrics(t *testing.T) {
var acc testutil.Accumulator
z := &Zfs{
KstatMetrics: []string{"vdev_cache_stats"},
sysctl: mock_sysctl,
zdataset: mock_zdataset,
}
err := z.Gather(&acc)
require.NoError(t, err)
require.False(t, acc.HasMeasurement("zfs_dataset"))
acc.Metrics = nil
z = &Zfs{
KstatMetrics: []string{"vdev_cache_stats"},
DatasetMetrics: true,
sysctl: mock_sysctl,
zdataset: mock_zdataset,
}
err = z.Gather(&acc)
require.NoError(t, err)
//one pool, all metrics
tags := map[string]string{
"dataset": "zata",
}
datasetMetrics := getZataDatasetMetrics()
acc.AssertContainsTaggedFields(t, "zfs_dataset", datasetMetrics, tags)
}
func TestZfsGeneratesMetrics(t *testing.T) {
var acc testutil.Accumulator
z := &Zfs{
KstatMetrics: []string{"vdev_cache_stats"},
sysctl: mock_sysctl,
zpool: mock_zpool,
}
err := z.Gather(&acc)
require.NoError(t, err)
//four pool, vdev_cache_stats metrics
tags := map[string]string{
"pools": "freenas-boot::red1::temp1::temp2",
}
intMetrics := getKstatMetricsVdevOnly()
acc.AssertContainsTaggedFields(t, "zfs", intMetrics, tags)
acc.Metrics = nil
z = &Zfs{
KstatMetrics: []string{"zfetchstats", "vdev_cache_stats"},
sysctl: mock_sysctl,
zpool: mock_zpool,
}
err = z.Gather(&acc)
require.NoError(t, err)
//four pool, vdev_cache_stats and zfetchstats metrics
intMetrics = getKstatMetricsVdevAndZfetch()
acc.AssertContainsTaggedFields(t, "zfs", intMetrics, tags)
}
func getFreeNasBootPoolMetrics() map[string]interface{} {
return map[string]interface{}{
"allocated": int64(2022177280),
"capacity": int64(6),
"dedupratio": float64(1),
"free": int64(28579464704),
"size": int64(30601641984),
"fragmentation": int64(0),
}
}
func getTemp2PoolMetrics() map[string]interface{} {
return map[string]interface{}{
"size": int64(0),
}
}
func getZataDatasetMetrics() map[string]interface{} {
return map[string]interface{}{
"avail": int64(10741741326336),
"used": int64(8564135526400),
"usedsnap": int64(0),
"usedds": int64(90112),
}
}
func getKstatMetricsVdevOnly() map[string]interface{} {
return map[string]interface{}{
"vdev_cache_stats_misses": int64(87789),
"vdev_cache_stats_hits": int64(465583),
"vdev_cache_stats_delegations": int64(6952),
}
}
func getKstatMetricsVdevAndZfetch() map[string]interface{} {
return map[string]interface{}{
"vdev_cache_stats_misses": int64(87789),
"vdev_cache_stats_hits": int64(465583),
"vdev_cache_stats_delegations": int64(6952),
"zfetchstats_max_streams": int64(0),
"zfetchstats_misses": int64(0),
"zfetchstats_hits": int64(0),
actual := acc.GetTelegrafMetrics()
testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime())
})
}
}