Redis集群管理脚本

最近在电信系统大量使用redis集群，之前都是小打小闹，搞个一两台机器，部署五六个redis实例。今天接到一个项目，要部署7台机器，由于每台有512G内存，按单实例24G内存来计算，每台需要部署10个实例，一共70个实例。。

好吧，不得不写个脚本来管理一下了。

准备工作：
1.从7台服务器选择一台作为主控机，用来管理整个群集
2.为了方便管理，需要可以从主控机使用公钥无密码ssh到其他6台机器
3.因为机器是全新的，有些系统参数需要修改，而且开了防火墙需要打开相关端口，root权限也是必须的


ok，脚本出炉：

#!/bin/sh

#redis的主目录，目录结构：
#    bin/
#        redis-server
#        redis-cli
#        redis-trib.rb
#    conf/
#        redis.conf
#    data/
REDIS_HOME=/home/kvuser/redis

#redis数据库存放位置
#由于会在一台机器上部署多个实例，各个实例的数据尽量存储在不同的硬盘上，以加快数据存储速度
DATA_DIRS=(/data1/redis /data2/redis /data3/redis /data4/redis /data5/redis)

#所有服务器的IP地址列表
HOSTS=(192.168.110.201 192.168.110.202 192.168.110.203 192.168.110.204 192.168.110.205 192.168.110.206 192.168.110.207)

#每台机器上生成的redis实例数量
NODES_PER_HOST=10

#每台机器的redis起始端口号，10台机器就是6379-6388
START_PORT=6379

#每组redis的slave机器的数量
REPLICAS=1

#是否拥有sudo权限，如果没有sudo权限或者登录账号不为root，需要root权限的操作都只会提示问题，不进行实际配置
SUDOABLE=1


USERNAME=`whoami`
IF_CONFIG=`/sbin/ifconfig`
LOCAL_IP=
SUDO=

#获得本机IP地址
for host in ${HOSTS[*]}
do
	i=`echo $IF_CONFIG|grep -c $host`
	if [ $i -eq 1 ]; then
		LOCAL_IP=$host
		break
	fi
done

#判断是否拥有root权限
if [ "$USERNAME" == "root" ]; then
	SUDOABLE=1
	SUDO=""
else
	SUDO=sudo
fi

redis_cmd() {
	for host in ${HOSTS[*]}
        do
                echo "[[$1 redis servers on $host]]"
                if [ "$LOCAL_IP" != "$host" ]; then
                        ssh -t -q $host "$REDIS_HOME/redis-cluster local$1"
                else
                        redis_local_$1
                fi
        done
}

redis_call(){
        for host in ${HOSTS[*]}
        do
                echo "[[call \"$@\" on $host]]"
                if [ "$LOCAL_IP" != "$host" ]; then
                        ssh -t -q $host "$REDIS_HOME/redis-cluster localcall $@"
                else
                        redis_local_call $@
                fi
        done
}

redis_start() {
	for host in ${HOSTS[*]}
	do
		echo "[[start redis servers on $host]]"
                if [ "$LOCAL_IP" != "$host" ]; then
			ssh -t -q $host "$REDIS_HOME/redis-cluster localstart"
		else
			redis_local_start
		fi
	done
}

redis_stop() {
        for host in ${HOSTS[*]}
        do
                echo "[[stop redis servers on $host]]"
                if [ "$LOCAL_IP" != "$host" ]; then
                	ssh -t -q $host "$REDIS_HOME/redis-cluster localstop"
                else
                        redis_local_stop
                fi
        done
}

redis_clean() {
        for host in ${HOSTS[*]}
        do
                echo "[[clean redis servers on $host]]"
                if [ "$LOCAL_IP" != "$host" ]; then
                        ssh -t -q $host "$REDIS_HOME/redis-cluster localclean"
                else
                        redis_local_clean
                fi
        done
}

redis_init() {
	if [ -f $REDIS_HOME.tar.gz ]; then
		rm -f $REDIS_HOME.tar.gz
	fi
	tar -C `dirname $REDIS_HOME` -czf $REDIS_HOME.tar.gz `basename $REDIS_HOME`/ --exclude `basename $REDIS_HOME`/data*
        for host in ${HOSTS[*]}
        do
		echo "[[init redis servers on $host]]"
                if [ "$LOCAL_IP" != "$host" ]; then
			scp $REDIS_HOME.tar.gz $host:$REDIS_HOME.tar.gz
                	ssh -t -q $host "tar xzf $REDIS_HOME.tar.gz && $REDIS_HOME/redis-cluster localinit"
		else
			redis_local_init
		fi
        done
}

redis_create() {
        CLUSTER=""
        for host in ${HOSTS[*]}
        do
                for(( i=0; i<$NODES_PER_HOST; i++))
                do
                        PORT=$((START_PORT+i))
                        CLUSTER="$CLUSTER $host:$PORT"
                done
        done
        echo "[create cluster $CLUSTER]"
        $REDIS_HOME/bin/redis-trib.rb create --replicas $REPLICAS $CLUSTER
}

redis_update() {
	for host in ${HOSTS[*]}
	do
                echo "[[update redis servers on $host]]"
                if [ "$LOCAL_IP" != "$host" ]; then
                        scp $REDIS_HOME/redis-cluster $host:$REDIS_HOME/
                fi
        done
}

redis_local_start() {
	j=0
	l=${#DATA_DIRS[@]}
	for(( i=0; i<$NODES_PER_HOST; i++))
	do
		PORT=$((START_PORT+i))
		echo "start redis at port $PORT"
		if [ ! -L $REDIS_HOME/data/$PORT ]; then
                        echo "make symbolic link to ${DATA_DIRS[j]}/$PORT"
                        ln -s ${DATA_DIRS[j]}/$PORT $REDIS_HOME/data/$PORT
                fi
		$REDIS_HOME/bin/redis-server $REDIS_HOME/conf/redis.conf --dir $REDIS_HOME/data/$PORT --port $PORT --bind $LOCAL_IP
		j=$(((j+1) % l))
	done
}

redis_local_stop() {
	for(( i=0; i<$NODES_PER_HOST; i++))
        do
                PORT=$((START_PORT+i))
		echo "stop redis at port $PORT"
                $REDIS_HOME/bin/redis-cli -h $LOCAL_IP -p $PORT shutdown
        done
}

redis_local_call() {
	for(( i=0; i<$NODES_PER_HOST; i++))
        do
                PORT=$((START_PORT+i))
		echo "[call $@ on port $PORT]"
		$REDIS_HOME/bin/redis-cli -h $LOCAL_IP -p $PORT $@
	done
}

redis_local_init() {
	j=0
	l=${#DATA_DIRS[@]}
	mkdir -p $REDIS_HOME/data
	for(( i=0; i<$NODES_PER_HOST; i++))
        do
                PORT=$((START_PORT+i))
                echo "[init redis at port $PORT]"

		if [ ! -d ${DATA_DIRS[j]}/$PORT ]; then
			echo "make data dir at ${DATA_DIRS[j]}/$PORT"
			mkdir -p ${DATA_DIRS[j]}/$PORT 2&> /dev/null || ( $SUDO mkdir -p ${DATA_DIRS[j]}/$PORT && $SUDO chown $USERNAME:$USERNAME ${DATA_DIRS[j]}/$PORT )
		fi
		j=$(((j+1) % l))
        done
	i=`/sbin/sysctl vm.overcommit_memory | grep -c "vm.overcommit_memory = 1"`
	if [ $i -eq 0 ]; then
		if [ $SUDOABLE -eq 1 ]; then
			echo "[set vm.overcommit_memory = 1]"
			$SUDO sh -c 'echo "vm.overcommit_memory = 1" >> /etc/sysctl.conf'
			$SUDO sysctl vm.overcommit_memory=1
		else
			echo "[!!! should set vm.overcommit_memory = 1 !!!]"
		fi
	fi
	
	i=`cat /sys/kernel/mm/transparent_hugepage/enabled|grep -c "\[never\]"`
	if [ $i -eq 0 ]; then
		if [ $SUDOABLE -eq 1 ]; then
			echo "[set /sys/kernel/mm/transparent_hugepage/enabled to never]"
			$SUDO sh -c 'echo never > /sys/kernel/mm/transparent_hugepage/enabled'
		else
			echo "[!!! /sys/kernel/mm/transparent_hugepage/enabled is \"always\", should set to \"never\". !!!]"
		fi
	fi

	i=`ulimit -n`
	if [ $i -lt 65536 ]; then
		if [ $SUDOABLE -eq 1 ]; then
			echo "[set open files to 65536]"
			line="$USERNAME	soft	nofile	65536"
			$SUDO sh -c "echo $line >> /etc/security/limits.d/$USERNAME.conf"
			line="$USERNAME	hard	nofile	65536"
			$SUDO sh -c "echo $line >> /etc/security/limits.d/$USERNAME.conf"
		else
			echo "[!!! max open file is $i, may be too small. !!!]"
		fi
	fi

	if [ $SUDOABLE -eq 1 ]; then
		s=`$SUDO service iptables status`
		f=0
		for(( i=0; i<$NODES_PER_HOST; i++))
        	do
        	        PORT=$((START_PORT+i))
			n=`echo $s|grep -c $PORT`
			if [ $n -eq 0 ]; then
				echo "[allow tcp connect on port $PORT from ${HOSTS[0]}/24]"
				$SUDO iptables -I INPUT 5 -m state --state NEW -s ${HOSTS[0]}/24 -p tcp --dport $PORT -j ACCEPT
				f=1
			fi
			PORT=$((PORT+10000))
        	        n=`echo $s|grep -c $PORT`
        	        if [ $n -eq 0 ]; then
        	                echo "[allow tcp connect on port $PORT from ${HOSTS[0]}/24]"
        	                $SUDO iptables -I INPUT 5 -m state --state NEW -s ${HOSTS[0]}/24 -p tcp --dport $PORT -j ACCEPT
        	                f=1
        	        fi
		done
		if [ $f -eq 1 ]; then
			echo "[save iptables]"
			$SUDO service iptables save
		fi
	else
		echo "[@@@ please check iptables for redis PORT and PORT+10000 @@@]"
	fi
}

redis_local_clean() {
        j=0
        l=${#DATA_DIRS[@]}
        for(( i=0; i<$NODES_PER_HOST; i++))
        do
                PORT=$((START_PORT+i))
                echo "[clean redis at port $PORT]"
		rm -fr ${DATA_DIRS[j]}/$PORT/*
		j=$(((j+1) % l))
	done
	rm -f $REDIS_HOME/data/*
}

redis_local_tail() {
	for(( i=0; i<$NODES_PER_HOST; i++))
        do
                PORT=$((START_PORT+i))
		echo "[tail $REDIS_HOME/data/$PORT/redis.log]"
		tail $REDIS_HOME/data/$PORT/redis.log
		echo ""
        done
}

case "$1" in
	localstart)
#启动本机的所有redis实例
		redis_local_start
	;;
	localstop)
#停止本机的所有redis实例
		redis_local_stop
	;;
	localinit)
#初始化本机配置
		redis_local_init
	;;
	localclean)
#清除本机所有redis实例的数据
		redis_local_clean
	;;
	localtail)
#查看本机所有redis实例日志的最后10行
		redis_local_tail
	;;
	localcall)
#在本机所有redis实例上执行redis命令
		redis_local_call $2 $3 $4 $5 $6 $7 $8 $9
	;;
	create)
#初始化redis集群
		redis_create
	;;
	start)
#启动集群的所有redis实例
		redis_cmd start
	;;
	stop)
#停止集群所有redis实例
		redis_cmd stop
	;;
	tail)
#查看集群所有redis实例日志的最后10行
		redis_cmd tail
	;;
	init)
#初始化集群所有redis实例
		redis_init
	;;
	clean)
#清除集群所有redis实例的数据
		redis_cmd clean
	;;
	update)
#更新集群上的redis-cluster脚本
		redis_update
	;;
	call)
#在集群所有redis实例上执行redis命令
		redis_call $2 $3 $4 $5 $6 $7 $8 $9
	;;
	*)
		echo $"Usage: $0 {init|create|start|stop|init|clean|tail|update|localstart|localstop|localinit|localclean|localtail}"
		exit 1
esac

redis.conf文件中不需要配置port, bind, dir这三个参数，这三个参数会直接在命令行上配置

好，现在就可以轻松管理这70个redis实例了：
./redis-cluster init
./redis-cluster start
./redis-cluster create
执行以上三条命令，集群就初始化完成了，如果有root权限，什么防火墙啊，系统参数啊，统统搞定

YES