From 54422f4f1dd0aa7816fa73b7bdc9aaf1f042e4a2 Mon Sep 17 00:00:00 2001 From: Dae-Ho Kim Date: Wed, 20 Jan 2021 11:45:24 +0900 Subject: [PATCH] add kafka connect example to jolokia2 input (#8709) * feat: add kafka connect example to jolokia2 input * docs: add kafka connect example link * chore: polishing --- plugins/inputs/jolokia2/README.md | 1 + .../jolokia2/examples/kafka-connect.conf | 90 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 plugins/inputs/jolokia2/examples/kafka-connect.conf diff --git a/plugins/inputs/jolokia2/README.md b/plugins/inputs/jolokia2/README.md index 4a7b8f420..a944949db 100644 --- a/plugins/inputs/jolokia2/README.md +++ b/plugins/inputs/jolokia2/README.md @@ -179,6 +179,7 @@ Both `jolokia2_agent` and `jolokia2_proxy` plugins support default configuration - [Java JVM](/plugins/inputs/jolokia2/examples/java.conf) - [JBoss](/plugins/inputs/jolokia2/examples/jboss.conf) - [Kafka](/plugins/inputs/jolokia2/examples/kafka.conf) +- [Kafka Connect](/plugins/inputs/jolokia2/examples/kafka-connect.conf) - [Tomcat](/plugins/inputs/jolokia2/examples/tomcat.conf) - [Weblogic](/plugins/inputs/jolokia2/examples/weblogic.conf) - [ZooKeeper](/plugins/inputs/jolokia2/examples/zookeeper.conf) diff --git a/plugins/inputs/jolokia2/examples/kafka-connect.conf b/plugins/inputs/jolokia2/examples/kafka-connect.conf new file mode 100644 index 000000000..d84f5fd58 --- /dev/null +++ b/plugins/inputs/jolokia2/examples/kafka-connect.conf @@ -0,0 +1,90 @@ +[[inputs.jolokia2_agent]] + urls = ["http://localhost:8080/jolokia"] + name_prefix = "kafka.connect." + + [[processors.enum]] + [[processors.enum.mapping]] + field = "status" + + [processors.enum.mapping.value_mappings] + paused = 0 + running = 1 + unassigned = 2 + failed = 3 + destroyed = 4 + + [inputs.jolokia2_agent.tags] + input_type = "kafka-connect" + + # https://kafka.apache.org/documentation/#connect_monitoring + [[inputs.jolokia2_agent.metric]] + name = "connectWorkerMetrics" + mbean = "kafka.connect:type=connect-worker-metrics" + paths = ["connector-count", "connector-startup-attempts-total", "connector-startup-failure-percentage", "connector-startup-failure-total", "connector-startup-success-percentage", "connector-startup-success-total", "task-count", "task-startup-attempts-total", "task-startup-failure-percentage", "task-startup-failure-total", "task-startup-success-percentage", "task-startup-success-total"] + + [[inputs.jolokia2_agent.metric]] + name = "connectWorkerMetrics" + mbean = "kafka.connect:type=connect-worker-metrics,connector=*" + paths = ["connector-destroyed-task-count", "connector-failed-task-count", "connector-paused-task-count", "connector-running-task-count", "connector-total-task-count", "connector-unassigned-task-count"] + tag_keys = ["connector"] + + [[inputs.jolokia2_agent.metric]] + name = "connectWorkerRebalanceMetrics" + mbean = "kafka.connect:type=connect-worker-rebalance-metrics" + paths = ["completed-rebalances-total", "connect-protocol", "epoch", "leader-name", "rebalance-avg-time-ms", "rebalance-max-time-ms", "rebalancing", "time-since-last-rebalance-ms"] + + [[inputs.jolokia2_agent.metric]] + name = "connectorMetrics" + mbean = "kafka.connect:type=connector-metrics,connector=*" + paths = ["connector-class", "connector-version", "connector-type", "status"] + tag_keys = ["connector"] + + [[inputs.jolokia2_agent.metric]] + name = "connectorTaskMetrics" + mbean = "kafka.connect:type=connector-task-metrics,connector=*,task=*" + paths = ["batch-size-avg", "batch-size-max", "offset-commit-avg-time-ms", "offset-commit-failure-percentage", "offset-commit-max-time-ms", "offset-commit-success-percentage", "pause-ratio", "running-ratio", "status"] + tag_keys = ["connector", "task"] + + [[inputs.jolokia2_agent.metric]] + name = "sinkTaskMetrics" + mbean = "kafka.connect:type=sink-task-metrics,connector=*,task=*" + paths = ["offset-commit-completion-rate", "offset-commit-completion-total", "offset-commit-seq-no", "offset-commit-skip-rate", "offset-commit-skip-total", "partition-count", "put-batch-avg-time-ms", "put-batch-max-time-ms", "sink-record-active-count", "sink-record-active-count-avg", "sink-record-active-count-max", "sink-record-lag-max", "sink-record-read-rate", "sink-record-read-total", "sink-record-send-rate", "sink-record-send-total"] + tag_keys = ["connector", "task"] + + [[inputs.jolokia2_agent.metric]] + name = "sourceTaskMetrics" + mbean = "kafka.connect:type=source-task-metrics,connector=*,task=*" + paths = ["poll-batch-avg-time-ms", "poll-batch-max-time-ms", "source-record-active-count", "source-record-active-count-avg", "source-record-active-count-max", "source-record-poll-rate", "source-record-poll-total", "source-record-write-rate", "source-record-write-total"] + tag_keys = ["connector", "task"] + + [[inputs.jolokia2_agent.metric]] + name = "taskErrorMetrics" + mbean = "kafka.connect:type=task-error-metrics,connector=*,task=*" + paths = ["deadletterqueue-produce-failures", "deadletterqueue-produce-requests", "last-error-timestamp", "total-errors-logged", "total-record-errors", "total-record-failures", "total-records-skipped", "total-retries"] + tag_keys = ["connector", "task"] + + # https://kafka.apache.org/documentation/#selector_monitoring + [[inputs.jolokia2_agent.metric]] + name = "connectMetrics" + mbean = "kafka.connect:type=connect-metrics,client-id=*" + paths = ["connection-close-rate", "connection-close-total", "connection-creation-rate", "connection-creation-total", "network-io-rate", "network-io-total", "outgoing-byte-rate", "outgoing-byte-total", "request-rate", "request-total", "request-size-avg", "request-size-max", "incoming-byte-rate", "incoming-byte-rate", "incoming-byte-total", "response-rate", "response-total", "select-rate", "select-total", "io-wait-time-ns-avg", "io-wait-ratio", "io-time-ns-avg", "io-ratio", "connection-count", "successful-authentication-rate", "successful-authentication-total", "failed-authentication-rate", "failed-authentication-total", "successful-reauthentication-rate", "successful-reauthentication-total", "reauthentication-latency-max", "reauthentication-latency-avg", "failed-reauthentication-rate", "failed-reauthentication-total", "successful-authentication-no-reauth-total"] + tag_keys = ["client-id"] + + # https://kafka.apache.org/documentation/#common_node_monitoring + [[inputs.jolokia2_agent.metric]] + name = "connectNodeMetrics" + mbean = "kafka.connect:type=connect-node-metrics,client-id=*,node-id=*" + paths = ["outgoing-byte-rate", "outgoing-byte-total", "request-rate", "request-total", "request-size-avg", "request-size-max", "incoming-byte-rate", "incoming-byte-total", "request-latency-avg", "request-latency-max", "response-rate", "response-total"] + tag_keys = ["client-id", "node-id"] + + [[inputs.jolokia2_agent.metric]] + name = "appInfo" + mbean = "kafka.connect:type=app-info,client-id=*" + paths = ["start-time-ms", "commit-id", "version"] + tag_keys = ["client-id"] + + [[inputs.jolokia2_agent.metric]] + name = "connectCoordinatorMetrics" + mbean = "kafka.connect:type=connect-coordinator-metrics,client-id=*" + paths = ["join-time-max", "failed-rebalance-rate-per-hour", "rebalance-latency-total", "sync-time-avg", "join-rate", "sync-rate", "failed-rebalance-total", "rebalance-total", "last-heartbeat-seconds-ago", "heartbeat-rate", "join-time-avg", "sync-total", "rebalance-latency-max", "sync-time-max", "last-rebalance-seconds-ago", "rebalance-rate-per-hour", "assigned-connectors", "heartbeat-total", "assigned-tasks", "heartbeat-response-time-max", "rebalance-latency-avg", "join-total"] + tag_keys = ["client-id"] \ No newline at end of file