CentOS6 Install kafka
一.zookeeper集群安装
原文链接：http://www.cnblogs.com/caoguo/p/5958608.html
# 环境
192.168.190.152 192.168.190.153 192.168.190.154
Install jdk
# tar zxvf jdk-8u65-linux-x64.tar.gz # mv jdk1.8.0_65 /usr/local/ # vi /etc/profile.d/jdk.sh JAVA_HOME=/usr/local/jdk1.8.0_65 export JAVA_HOME PATH=$JAVA_HOME/bin:$PATH export PATH # source /etc/profile
Install zookeeper
# wget http://www-eu.apache.org/dist/zookeeper/zookeeper-3.4.9/zookeeper-3.4.9.tar.gz # tar zxvf zookeeper-3.4.9.tar.gz # mv zookeeper-3.4.9 /usr/local/ # useradd zookeeper # chown -R zookeeper.zookeeper /usr/local/zookeeper-3.4.9 # cd /usr/local/zookeeper-3.4.9/conf/ # cp zoo_sample.cfg zoo.cfg # mkdir /etc/zookeeper # ln -sf /usr/local/zookeeper-3.4.9/conf/zoo.cfg /etc/zookeeper/ # mkdir -p /var/lib/zookeeper # chown -R zookeeper:zookeeper /var/lib/zookeeper # 用来标识主机 #echo 1 > /var/lib/zookeeper/myid #echo 2 > /var/lib/zookeeper/myid #echo 3 > /var/lib/zookeeper/myid
# vi /etc/zookeeper/zoo.cfg tickTime=2000 initLimit=10 syncLimit=5 dataDir=/var/lib/zookeeper clientPort=2181 maxClientCnxns=0 server.1=192.168.190.152:2888:3888 server.2=192.168.190.153:2888:3888 server.3=192.168.190.154:2888:3888
[root@localhost ~]# vi /etc/init.d/zookeeper #! /bin/sh # # chkconfig: 2345 90 10 # description: zookeeper daemon . /etc/init.d/functions # You will probably want to change only two following lines. BASEDIR="/usr/local/zookeeper-3.4.9" USER="zookeeper" PROG="zookeeper" CMD="bin/zkServer.sh" RETVAL=0 start () { echo -n $"Starting ${PROG}: " runuser ${USER} -c "cd ${BASEDIR} ${CMD} start > /dev/null &" echo } stop () { echo -n $"Stopping ${PROG}: " runuser ${USER} -c "cd ${BASEDIR} ${CMD} stop > /dev/null &" echo } restart () { stop start } # See how we were called. case "$1" in start) start ;; stop) stop ;; restart) restart ;; *) echo $"Usage: $0 {start|stop|status}" RETVAL=2 ;; esac exit $RETVAL # chmod 755 /etc/init.d/zookeeper # /etc/init.d/zookeeper start # chkconfig zookeeper on
# ./zkCli.sh # 查看当前节点列表 [zk: localhost:2181(CONNECTED) 6] ls / [zookeeper] # 创建节点 [zk: localhost:2181(CONNECTED) 7] create /test "test" Created /test [zk: localhost:2181(CONNECTED) 8] ls / [zookeeper, test] # 查看节点数据 [zk: localhost:2181(CONNECTED) 9] get /test test cZxid = 0x100000004 ctime = Thu Oct 13 08:35:04 CST 2016 mZxid = 0x100000004 mtime = Thu Oct 13 08:35:04 CST 2016 pZxid = 0x100000004 cversion = 0 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 4 numChildren = 0 # 设置节点数据 [zk: localhost:2181(CONNECTED) 10] set /test "111111" cZxid = 0x100000004 ctime = Thu Oct 13 08:35:04 CST 2016 mZxid = 0x100000005 mtime = Thu Oct 13 08:35:41 CST 2016 pZxid = 0x100000004 cversion = 0 dataVersion = 1 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 6 numChildren = 0 # 删除节点数据 [zk: localhost:2181(CONNECTED) 11] delete /test [zk: localhost:2181(CONNECTED) 12] ls / [zookeeper]
二.kafka集群安装
# 环境
192.168.190.152 192.168.190.153 192.168.190.154
wget http://mirrors.cnnic.cn/apache/kafka/0.10.0.1/kafka_2.11-0.10.0.1.tgz # tar zxvf kafka_2.11-0.10.0.1.tgz # mv kafka_2.11-0.10.0.1 /usr/local/ # cd /usr/local/kafka_2.11-0.10.0.1/
# vi config/server.properties broker.id=0 # id唯一 0 1 2 port=9092 host.name=192.168.190.152 #192.168.190.153 192.168.190.154 num.network.threads=3 num.io.threads=8 socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 socket.request.max.bytes=104857600 log.dirs=/tmp/kafka-logs num.partitions=1 default.replication.factor=2 num.recovery.threads.per.data.dir=1 log.retention.hours=168 log.segment.bytes=1073741824 log.retention.check.interval.ms=300000 zookeeper.connect=192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 zookeeper.connection.timeout.ms=6000
# useradd kafka # touch /etc/init.d/kafka # chmod 755 /etc/init.d/kafka
# vi /etc/init.d/kafka #!/bin/sh # # chkconfig: 345 99 01 # description: Kafka # # File : Kafka # # Description: Starts and stops the Kafka server # source /etc/rc.d/init.d/functions KAFKA_HOME=/usr/local/kafka_2.11-0.10.0.1 KAFKA_USER=kafka # See how we were called. case "$1" in start) echo -n "Starting Kafka:" /sbin/runuser $KAFKA_USER -c "nohup $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties > /var/log/kafka/server.out 2> /var/log/kafka/server.err &" echo " done." exit 0 ;; stop) echo -n "Stopping Kafka: " /sbin/runuser $KAFKA_USER -c "ps -ef | grep kafka.Kafka | grep -v grep | awk '{print \$2}' | xargs kill" echo " done." exit 0 ;; hardstop) echo -n "Stopping (hard) Kafka: " /sbin/runuser $KAFKA_USER -c "ps -ef | grep kafka.Kafka | grep -v grep | awk '{print \$2}' | xargs kill -9" echo " done." exit 0 ;; status) c_pid=`ps -ef | grep kafka.Kafka | grep -v grep | awk '{print $2}'` if [ "$c_pid" = "" ] ; then echo "Stopped" exit 3 else echo "Running $c_pid" exit 0 fi ;; restart) stop start ;; *) echo "Usage: cassandra {start|stop|hardstop|status|restart}" exit 1 ;; esac
# mkdir -p /var/log/kafka # chown -R kafka.kafka /var/log/kafka # chown -R kafka.kafka /usr/local/kafka_2.11-0.10.0.1/logs # 启动服务 # /etc/init.d/kafka start # 创建topic # bin/kafka-topics.sh --create --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --replication-factor 2 --partitions 1 --topic test333 # 删除 # bin/kafka-topics.sh --delete --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --topic test333 # 查看topic # bin/kafka-topics.sh --list --zookeeper localhost:2181 # 开启生产端 # bin/kafka-console-producer.sh --broker-list 192.168.190.152:9092,192.168.190.153:9092,192.168.190.154:9092 --topic test333 # 开启消费端 # bin/kafka-console-consumer.sh --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --topic test333 --from-beginning # bin/kafka-topics.sh --describe --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --topic test333
# 错误处理
[2016-10-14 06:36:18,401] WARN Error while fetching metadata with correlation id 0 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:19,543] WARN Error while fetching metadata with correlation id 1 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:19,680] WARN Error while fetching metadata with correlation id 2 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:19,908] WARN Error while fetching metadata with correlation id 3 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:20,116] WARN Error while fetching metadata with correlation id 4 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:20,334] WARN Error while fetching metadata with correlation id 5 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:20,505] WARN Error while fetching metadata with correlation id 6 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:20,757] WARN Error while fetching metadata with correlation id 7 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
config/server.properties kafka必须配置hostname
