想監視卡夫卡消息傳送指標 - 許多消息大多是如何消耗或損失，延遲，最終消費者偏移等等...Apache spark - 如何監控Kafka消息傳遞？
發現Apache的火花，但我真的不明白，如果它可以「無需編寫任何（java）代碼即可輕鬆插入Kafka」。
我只想在一些配置文件中更改Kafka連接的配置，在那裏存儲消息傳遞指標並在一些用戶界面中顯示和查詢它。
與從頭開始編寫（java）應用程序相比，是否存在使用Spark-Kafka消息傳遞監控的「更加自動化」的方式？
是否有一些已經完成的應用程序，可以自由使用？
如何設置鏈卡夫卡 - 火花 - （數據庫？） - UI在最簡單和自動的方式？
Can Spark還可以監控其他度量標準 - 集羣，代理，主題，虛擬副本，分區數量......等等？
在使用不同軟件進行實驗後，卡夫卡監視器顯示出最適合我的情況。
它通過整個羣集上的分區創建主題，並提供有關生產和消費的指標，如消息丟失，重複和端到端延遲。
開始卡夫卡顯示器像這樣：
cd <path_to_kafka_monitor>
export JMX_PORT=<some_port>
./bin/kafka-monitor-start.sh config/kafka-monitor.properties
指標與jmxTrans收集。使用的查詢：
{
"servers": [{
"port": "9999",
"host": "127.0.0.1",
"queries": [{
"obj": "kafka.consumer:type=consumer-coordinator-metrics,client-id=*",
"attr": ["assigned-partitions","commit-latency-avg","commit-rate","heartbeat-rate","heartbeat-response-time-max",
"join-rate","join-time-avg","join-time-max","last-heartbeat-seconds-ago","sync-rate","sync-time-avg","sync-time-max"],
"resultAlias": "monitor-consumer.consumer-coordinator-metrics;client-id",
"outputWriters": [{
"@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
"url": "http://127.0.0.1:8086/",
"username": "root",
"password": "root",
"database": "monitor",
"typeNames": ["client-id"]
}]
},
{
"obj": "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*",
"attr": ["bytes-consumed-rate","fetch-latency-avg","fetch-latency-max","records-consumed-rate","records-lag-max",
"records-per-request-avg"],
"resultAlias": "monitor-consumer.consumer-fetch-manager-metrics,client-id",
"outputWriters": [{
"@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
"url": "http://127.0.0.1:8086/",
"username": "root",
"password": "root",
"database": "monitor",
"typeNames": ["client-id"]
}]
},
{
"obj": "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*,topic=*",
"attr": ["bytes-consumed-rate","fetch-size-avg","fetch-size-max","records-consumed-rate","records-per-request-avg"],
"resultAlias": "monitor-consumer-fetch-manager-metrics;client-id,topic",
"outputWriters": [{
"@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
"url": "http://127.0.0.1:8086/",
"username": "root",
"password": "root",
"database": "monitor",
"typeNames": ["client-id","topic"]
}]
},
{
"obj": "kafka.consumer:type=consumer-metrics,client-id=*",
"attr": [],
"resultAlias": "monitor-consumer-metrics;client-id",
"outputWriters": [{
"@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
"url": "http://127.0.0.1:8086/",
"username": "root",
"password": "root",
"database": "monitor",
"typeNames": ["client-id"]
}]
},
{
"obj": "kafka.consumer:type=consumer-node-metrics,client-id=*,node-id=*",
"attr": ["incoming-byte-rate","outgoing-byte-rate","request-latency-avg","request-rate","request-size-avg","response-rate"],
"resultAlias": "monitor-consumer-node-metrics;client-id,node-id",
"outputWriters": [{
"@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
"url": "http://127.0.0.1:8086/",
"username": "root",
"password": "root",
"database": "monitor",
"typeNames": ["client-id","node-id"]
}]
},
{
"obj": "kafka.producer:type=producer-metrics,client-id=*",
"attr": [],
"resultAlias": "monitor-producer-metrics;client-id",
"outputWriters": [{
"@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
"url": "http://127.0.0.1:8086/",
"username": "root",
"password": "root",
"database": "monitor",
"typeNames": ["client-id"]
}]
},
{
"obj": "kafka.producer:type=producer-node-metrics,client-id=*,node-id=*",
"attr": ["incoming-byte-rate","outgoing-byte-rate","request-latency-avg","request-rate","request-size-avg","response-rate"],
"resultAlias": "monitor-producer-node-metrics;client-id,node-id",
"outputWriters": [{
"@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
"url": "http://127.0.0.1:8086/",
"username": "root",
"password": "root",
"database": "monitor",
"typeNames": ["client-id","node-id"]
}]
},
{
"obj": "kafka.producer:type=producer-topic-metrics,client-id=*,topic=*",
"attr": [],
"resultAlias": "monitor-producer-topic-metrics;client-id,topic",
"outputWriters": [{
"@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
"url": "http://127.0.0.1:8086/",
"username": "root",
"password": "root",
"database": "monitor",
"typeNames": ["client-id","topic"]
}]
},
{
"obj": "kmf.services:type=consume-service,name=single-cluster-monitor",
"attr": [],
"resultAlias": "monitor-consume-service.single-cluster-monitor",
"outputWriters": [{
"@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
"url": "http://127.0.0.1:8086/",
"username": "root",
"password": "root",
"database": "monitor"
}]
},
{
"obj": "kmf.services:type=produce-service,name=single-cluster-monitor",
"attr": [],
"resultAlias": "monitor-produce-service.single-cluster-monitor",
"outputWriters": [{
"@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory",
"url": "http://127.0.0.1:8086/",
"username": "root",
"password": "root",
"database": "monitor"
}]
}
]
}]
}
嗨，您可以使用kafka manager項目。它將使得在便捷模式下管理所有卡夫卡服務
請問您可以分享鏈接，您看到apache-spark可用於監控apache-kafka matrics嗎？ – berrytchaks
主要來自這裏：http：//spark.apache.org/docs/latest/streaming-programming-guide.html#monitoring-applications，但現在，在第二次閱讀時，我看到接收器不是卡夫卡消費者。我的錯，但是，仍然，你知道是否可以在沒有付費軟件的情況下監控kafka郵件傳送嗎？謝謝 – milkamar
你有興趣在ELK堆棧中使用節拍嗎？ – berrytchaks