kafka日常操作命令
kafka基础操作
启动：
[root@sinleka ~]# /opt/softs/zookeeper-3.4.10/bin/zkServer.sh start
[root@sinleka ~]# /opt/softs/kafka_2.11-0.11.0.0/bin/kafka-server-start.sh -daemon /opt/softs/kafka_2.11-0.11.0.0/config/server.properties
zk信息：192.168.19.154:2181，192.168.19.157:2181，192.168.19.160:2181
kfk信息：192.168.19.154:9092，192.168.19.157:9092，192.168.19.160:9092
生产者生产消息
[root@www ~]# /opt/softs/kafka_2.11-0.11.0.0/bin/kafka-console-producer.sh --broker-list 192.168.19.154:9092 --topic weiwei
消费组消费（--from-beginning从最开始端消费数据）
/opt/softs/kafka_2.11-0.11.0.0/bin/kafka-console-consumer.sh -zookeeper 192.168.19.157:2181 --topic weiwei --from-beginning
查看集群topic信息
[root@www ~]# /opt/softs/kafka_2.11-0.11.0.0/bin/kafka-topics.sh --zookeeper 192.168.19.160:2181/ --topic weiwei --describe
Topic:weiwei PartitionCount:4 ReplicationFactor:3 Configs:
Topic: weiwei Partition: 0 Leader: 1 Replicas: 1,2,0 Isr: 1,2,0
Topic: weiwei Partition: 1 Leader: 2 Replicas: 2,0,1 Isr: 2,0,1
Topic: weiwei Partition: 2 Leader: 0 Replicas: 0,1,2 Isr: 2,0,1
Topic: weiwei Partition: 3 Leader: 1 Replicas: 1,0,2 Isr: 1,2,0
查看消费组信息
[root@sinleka ~]# /opt/softs/kafka_2.11-0.11.0.0/bin/kafka-consumer-groups.sh --zookeeper 192.168.19.157:2181 --list
Note: This will only show information about consumers that use ZooKeeper (not those using the Java consumer API).
console-consumer-4863
[mwopr@1a02vla2485zzzz ~]$ /app/kafka/bin/kafka-consumer-offset-checker.sh --zookeeper 100.86.129.134:2181/kafka/scs_oms2_core_fah61tzb01 --group SCS_OMS2_CORE_FAH61TZB01_SYNC --topic KAFKA_BROKER_CHECK_NEW
flink数据查找offset信息
看不到存活的消费组，但是可以看到offset在变化，说明消费组存在
[appdeploy@cnsz17vl4398:/home/appdeploy]$/app/kafka/bin/kafka-consumer-offset-checker.sh --zookeeper 100.86.131.222:2181/kafka/eos_fms_tdms_tcjty97v01 --group FOP_DMP_DSP2 --topic FOP_TDOP_CORE_PALLET
Group Topic Pid Offset logSize Lag Owner
FOP_DMP_DSP2 FOP_TDOP_CORE_PALLET 0 7397270 7397501 231 none
FOP_DMP_DSP2 FOP_TDOP_CORE_PALLET 1 7395898 7396134 236 none
FOP_DMP_DSP2 FOP_TDOP_CORE_PALLET 2 7396422 7396666 244 none
FOP_DMP_DSP2 FOP_TDOP_CORE_PALLET 3 7398751 7398997 246 none
找zk位置
[appdeploy@cnsz17vl4398:/home/appdeploy]$/app/zookeeper-3.4.6/bin/zkCli.sh -server 100.86.131.222:2181
[zk: 100.86.131.222:2181(CONNECTED) 1] ls /kafka/eos_fms_tdms_tcjty97v01/consumers/FOP_DMP_DSP2/offsets/FOP_TDOP_CORE_PALLET
[0, 1, 2, 3]
[zk: 100.86.131.222:2181(CONNECTED) 2] ls /kafka/eos_fms_tdms_tcjty97v01/consumers/FOP_DMP_DSP2/offsets/FOP_TDOP_CORE_PALLET/0
[]
[zk: 100.86.131.222:2181(CONNECTED) 3] get /kafka/eos_fms_tdms_tcjty97v01/consumers/FOP_DMP_DSP2/offsets/FOP_TDOP_CORE_PALLET/0
7397270
cZxid = 0x3038acc3a
ctime = Mon Aug 02 20:12:34 CST 2021
mZxid = 0x3038acc3f
mtime = Mon Aug 02 20:12:34 CST 2021
pZxid = 0x3038acc3a
cversion = 0
dataVersion = 1
aclVersion = 0
ephemeralOwner = 0x0
dataLength = 7
numChildren = 0
[zk: 100.86.131.222:2181(CONNECTED) 4]
查看kafka主题消息。
[mwopr@testa:/home/appdeploy]$/app/kafka/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list 10.110.65.200:9094 --topic SAP_MDMS_JSON_MDMSINFO --time -2 |awk -F ':' '{sum+=$NF} END{print $1,sum}'
SAP_MDMS_JSON_MDMSINFO 3599551
[appdeploy@testa:/home/appdeploy]$/app/kafka/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list 10.110.65.200:9094 --topic SAP_MDMS_JSON_MDMSINFO --time -1 |awk -F ':' '{sum+=$NF} END{print $1,sum}'
SAP_MDMS_JSON_MDMSINFO 3601193
[mwopr@testa:/home/mwopr]$/app/kafka/bin/kafka-console-consumer.sh --zookeeper 10.110.65.200:2181/kafka/other --topic SAP_MDMS_JSON_MDMSINFO --from-beginning >/app/tmp/trq.txt
^CConsumed 1642 messages
[mwopr@testa:/home/mwopr]$wc -l /app/tmp/trq.txt
1642 /app/tmp/trq.txt
[mwopr@testa:/home/mwopr]$grep 100000396883 /app/tmp/trq.txt
<?xml version="1.0" encoding="utf-8"?><n0:MT_Unify_Req xmlns:n0="urn:sf:erp:common:unify" xmlns:prx="urn:sap.com:proxy:EEP:/1SAI/TASA12787EAB91CA4128E9B:740"><
