Kafka 安装

1. download kafka_2.11-0.10.1.0.tgz, upload to /usr/local

cd /usr/local

tar -zxvf kafka_2.11-0.10.1.0.tgz

mv kafka_2.11-0.10.1.0 kafka

 

2. config 

vi ~/.bashrc

export KAFKA_HOME=/usr/local/kafka

export PATH=     :$KAFKA_HOME/bin

source ~/.bashrc

vi kafka/config/server.properties

broker.id=0

log.dirs=/root/data/kafka/logs

zookeeper.connect=centos1:2181,centos2:2181,centos3:2181

 

3. add slf4j

download slf4j-1.7.21.zip, unzip it, and upload slf4j-nop-1.7.21.jar to /usr/local/kafka/libs

 

4. copy kafka to contos2, centos3

scp ~/.bashrc root@centos2:~/.bashrc

scp ~/.bashrc root@centos3:~/.bashrc

scp -r /usr/local/kafka root@centos2:/usr/local/

scp -r /usr/local/kafka root@centos3:/usr/local/

ssh contos2

source ~/.bashrc

vi /usr/local/kafka/config/server.properties

broker.id=1

ssh contos3

source ~/.bashrc

vi /usr/local/kafka/config/server.properties

broker.id=2

 

5 start kafka on centos1 and check if there is any error

cd /usr/local/kafka

bin/kafka-server-start.sh config/server.properties

error: can't allocate enough memory

solution: 

vi bin/kafka-server-start.sh 

modify: export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G" to  export KAFKA_HEAP_OPTS="-Xmx1G -Xms128M"

modify bin/kafka-server-start.sh on centos2 and centos3 

 

6. start zookeeper and kafka in backend on each server

ssh centos1:

zkServer.sh start

nohup /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties &

ssh centos2:

zkServer.sh start

nohup /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties &

ssh centos3:

zkServer.sh start

nohup /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties &

 

7 test for send and consumer messages

cd /usr/local/kafka/

bin/kafka-topics.sh --create --zookeeper centos1:2181,centos2:2181,centos3:2181 --replication-factor 1 --partitions 1 --topic testTopic

bin/kafka-console-producer.sh --broker-list centos1:9092,centos2:9092,centos3:9092 --topic testTopic

open another console window and login centos1:

cd /usr/local/kafka/

bin/kafka-console-consumer.sh --bootstrap-server centos1:9092,centos2:9092,centos3:9092 --topic testTopic --from-beginning

(if you print message in console window1, it will be received in console window2)