`
wbj0110
  • 浏览: 1603277 次
  • 性别: Icon_minigender_1
  • 来自: 上海
文章分类
社区版块
存档分类
最新评论

Install Storm on CentOS

阅读更多

Centos 安装Storm 脚本 保存于此,备忘.

 

什么是Storm?

 

  Storm is a free and open source distributed realtime computation system. Storm makes it easy to reliably process unbounded streams of data, doing for realtime processing what Hadoop did for batch processing. Storm is simple, can be used with any programming language, and is a lot of fun to use!

  Storm has many use cases: realtime analytics, online machine learning, continuous computation, distributed RPC, ETL, and more. Storm is fast: a benchmark clocked it at over a million tuples processed per second per node. It is scalablefault-tolerantguarantees your data will be processed, and is easy to set up and operate.

 

[官网] http://storm-project.net/

 

安装脚本

安装daemontools-0.76.tar.gz的脚本:

复制代码
yum install patch
sudo mkdir -p /package
sudo chmod 1755 /package/
cd /package/
sudo wget http://cr.yp.to/daemontools/daemontools-0.76.tar.gz
sudo tar xzf daemontools-0.76.tar.gz
sudo wget http://www.qmail.org/moni.csi.hu/pub/glibc-2.3.1/daemontools-0.76.errno.patch
cd admin/daemontools-0.76
sudo patch -p1 < http://www.cnblogs.com/daemontools-0.76.errno.patch
sudo rm http://www.cnblogs.com/daemontools-0.76.errno.patch http://www.cnblogs.com/daemontools-0.76.tar.gz
sudo ./package/install
复制代码

 

安装Storm的脚本是在 nathanmarz 写的脚本的基础上修改的,去掉了两个大文件的下载,依赖项修改为yum安装,代码略长 

复制代码
#!/bin/bash

pp() {
    echo -e "\e[00;32m"$1"\e[00m"
}

HOST=`hostname`

#########################################
# Clean up old installation.
#########################################

cleanup() {
    pp "Cleaning up previous installation..."
    rm -rf $BASEDIR
    mkdir $BASEDIR
    echo "#!/bin/bash" > $START_SH
    chmod +x $START_SH
    echo "#!/bin/bash" > $STOP_SH
    chmod +x $STOP_SH
}

#########################################
# System dependencies.
#########################################

deps() {
    pp "Checking system dependencies..."
    echo
       yum  install gcc gcc-c++.x86_64  crontabs  screen daemontools uuid-devel libuuid-devel git libtool build-essential  unzip
    echo
}

#########################################
# ZooKeeper.
#########################################

zookeeper() {
    echo "Current Host $HOST "

#    if [ "$HOST" != "$NIMBUS" ]
#    then
#        pp "Skipping ZooKeeper installation on all hosts but nimbus!"
#        return
#    fi

    ZK_VERSION="3.3.6"
    ZK_DIR=$BASEDIR"/zookeeper"
    ZK_CONFIGFILE="zoo.conf"
    ZK_CONF=$ZK_DIR"/"$ZK_CONFIGFILE
    ZK_RUN=$ZK_DIR"/run"
    ZK_PURGE=$ZK_DIR"/purge.sh"
    ZK_DATADIR=$ZK_DIR"/data"
    ZK_TARBALL_URL="http://apache.openmirror.de/zookeeper/zookeeper-"$ZK_VERSION"/zookeeper-"$ZK_VERSION".tar.gz"
    ZK_TARBALL=$ZK_DIR/"zookeeper.tar.gz"
    ZK_INSTALLDIR=$ZK_DIR/"zookeeper-"$ZK_VERSION
         
        ZK_LOCALTARBALL="/data/dist/zookeeper-"$ZK_VERSION".tar.gz"

        ZK_FILENAME=$BASEDIR"/zookeeper/zookeeper-"$ZK_VERSION.tar.gz
    pp "Installing ZooKeeper "$ZK_VERSION" on nimbus host '"$HOST"'..."

    mkdir $ZK_DIR &>/dev/null
    mkdir $ZK_DATADIR &>/dev/null

        echo "$ZK_FILENAME"
         
        if [ -f $ZK_LOCALTARBALL ]; then
         
       pp "ZooKeeper Copy From /data/dist/."
          cp $ZK_LOCALTARBALL $ZK_TARBALL
        else 
     pp "Downloading ZooKeeper..."
         wget $ZK_TARBALL_URL -q -O $ZK_TARBALL
        fi

    tar xzf $ZK_TARBALL -C $ZK_DIR
    rm $ZK_TARBALL

    pp "Configuring ZooKeeper..."

    # Cluster config.
    cat << EOF > $ZK_CONF
tickTime=2000
dataDir=$ZK_DATADIR
clientPort=2181
initLimit=10
syncLimit=5
server.1=192.168.0.101:2888:3888
server.2=192.168.0.102:2888:3888
server.3=192.168.0.103:2888:3888
EOF

#Read the myid
read  -p "Please select the myid for this instance:  " MYID 
if [ ! `echo $MYID | egrep "^[0-9]+\$"`  ] ; then
        echo "                       _  _                     "  
        echo "                _     ( \/ )                    "
        echo "                |              |               "
        echo "                                                "

        echo " Are you kidding ? That is number? Did you learnt math from P.E teacher?  Selecting default: 1"
                MYID=1 
                fi
    # This host's id.
    echo $MYID > $ZK_DATADIR/myid

    # Run script.
    ZK_CP=$ZK_INSTALLDIR/zookeeper-$ZK_VERSION.jar:$ZK_INSTALLDIR/lib/log4j-1.2.15.jar:$ZK_INSTALLDIR/conf
    cat << EOF > $ZK_RUN
#!/bin/bash
_JAVA_OPTIONS="-Xmx1024M -Xms1024M"
java -cp $ZK_CP org.apache.zookeeper.server.quorum.QuorumPeerMain $ZK_CONFIGFILE
EOF
    chmod +x $ZK_RUN

    # Purge script to cleanup zookeeper log files.
    cat << EOF > $ZK_PURGE
mkdir $ZK_DIR/snap
java -cp $ZK_CP org.apache.zookeeper.server.PurgeTxnLog $ZK_DATADIR $ZK_DIR/snap -n 3
rm -r $ZK_DIR/snap
EOF
    chmod +x $ZK_PURGE

    # Run purge.sh via cron job.
    echo "@hourly $ZK_PURGE" | crontab -

    # Update global start/stop scripts.
    echo "supervise $ZK_DIR &" >> $START_SH
    echo "svc -x $ZK_DIR" >> $STOP_SH
}

#########################################
# Storm dependency: ZeroMQ
#########################################

zeromq() {
    ZMQ_VERSION="2.1.7"
    ZMQ_DIR=$BASEDIR"/zeromq"
    ZMQ_TARBALL_URL="http://download.zeromq.org/zeromq-"$ZMQ_VERSION".tar.gz"
    ZMQ_TARBALL=$ZMQ_DIR"/zeromq.tar.gz"

    pp "Installing ZeroMQ "$ZMQ_VERSION" (storm dependency)..."
    mkdir $ZMQ_DIR

    pp "Downloading ZeroMQ..."
    wget $ZMQ_TARBALL_URL -q -O $ZMQ_TARBALL
    tar zxf $ZMQ_TARBALL -C $ZMQ_DIR
    rm $ZMQ_TARBALL

    pp "Compiling ZeroMQ..."
    echo
    pushd $ZMQ_DIR/zeromq-$ZMQ_VERSION
    ./configure && make && sudo make install
    popd
    echo
}

#########################################
# Storm dependency 2: JZMQ,
# Java bindings for ZeroMQ.
#
# This is where things get tricky.
# Despite the warning on nathanmarz' page,
# we use mainline git here, as it compiles
# with the latest autoconf and libtool on
# Ubuntu 12.04.
#########################################

jzmq() {
    JZMQ_DIR=$BASEDIR"/jzmq"
#    JZMQ_REPO="https://githiub.com/zeromq/jzmq.git"
#    JZMQ_COMMIT="e2dd66"

    pp "Installing JZMQ (Java bindings for ZeroMQ) from Github..."

    git clone -q https://github.com/nathanmarz/jzmq.git $JZMQ_DIR

    pp "Compiling JZMQ..."

    echo
    pushd $JZMQ_DIR
    git checkout $JZMQ_COMMIT
    ./autogen.sh && ./configure --with-zeromq=/usr/local/lib && make && sudo make install
    popd
    echo
}

#########################################
# Storm itself.
#########################################

storm() {
    STORM_VERSION="0.8.1"
    STORM_DIR=$BASEDIR"/storm"
    STORM_ZIP_URL="https://github.com/downloads/nathanmarz/storm/storm-"$STORM_VERSION".zip"
    STORM_ZIP=$STORM_DIR"/storm.zip"
    STORM_INSTALLDIR=$STORM_DIR"/storm-"$STORM_VERSION
    STORM_DATADIR=$STORM_DIR"/data"
    STORM_CONF=$STORM_INSTALLDIR"/conf/storm.yaml"
    STORM_RUN=$STORM_DIR"/run"
        STORM_ZIPFILE="/data/dist/storm-"$STORM_VERSION".zip"

    pp "Installing Storm "$STORM_VERSION"..."
    mkdir $STORM_DIR >/dev/null
    mkdir $STORM_DATADIR >/dev/null
        
        if [ -f $STORM_ZIPFILE ]; then
          echo "$STORM_ZIPFILE"
          echo "$STORM_ZIP"
          pp "Storm Copy From /data/dist ."
          cp $STORM_ZIPFILE $STORM_ZIP
        else
          pp "Downloading Storm..."
          wget $STORM_ZIP_URL -q -O $STORM_ZIP        
        fi

    
    unzip -qq $STORM_ZIP -d $STORM_DIR
    rm $STORM_ZIP





    pp "Configuring Storm..."
    echo "storm.local.dir: \""$STORM_DATADIR"\"" > $STORM_CONF
    echo "storm.zookeeper.servers:" >> $STORM_CONF
    echo " - \"192.168.10.101\"" >> $STORM_CONF
    echo " - \"192.168.10.102\"" >> $STORM_CONF
    echo " - \"192.168.10.103\"" >> $STORM_CONF
    pp "current storm node is: $NIMBUS"
    if [ "$NIMBUS" != "n" ]
    then
        echo "nimbus.host: \"192.168.10.101\"" >> $STORM_CONF
    fi

    echo "storm.zookeeper.port: 2181" >> $STORM_CONF

    # Supervisor directories/scripts + global start/stop scripts.
    # Note: If we're NIMBUS, we run the 'nimbis' action instead.
    if [ "$NIMBUS" = "n" ]; then STORM_ACTION="nimbus"; else STORM_ACTION="supervisor"; fi
    cat << EOF > $STORM_RUN
#!/bin/bash
$STORM_INSTALLDIR/bin/storm $STORM_ACTION
EOF
    chmod +x $STORM_RUN
    echo "supervise $STORM_DIR &" >> $START_SH
    echo "svc -x $STORM_DIR" >> $STOP_SH
}

#########################################
# Main app.
#########################################

PHASES=("cleanup" "deps" "zookeeper" "zeromq" "jzmq" "storm")

execute() {
    case "$1" in
    "0")
        cleanup
        ;;
    "1")
        deps
        ;;
    "2")
        zookeeper
        ;;
    "3")
        zeromq
        ;;
    "4")
        jzmq
        ;;
    "5")
        storm
        ;;
    esac
}

if [ $# -eq 3 ]
then

    NIMBUS=$2
    BASEDIR=$3
    START_SH=$BASEDIR"/start.sh"
    STOP_SH=$BASEDIR"/stop.sh"

    if [ "$1" = "all" ]
    then
        # Run everything.
        for ((p=0;p<${#PHASES[@]};p++))
        do
            execute $p
        done

        pp "Installation complete."
        pp "Be sure to carefully read the log."
        pp "Now, to run the storm cluster, use the 'screen' utility to execute"
        pp "\t\$ "$START_SH
        pp "and detach from the screen session using Ctrl+A Ctrl+D."
    else
        execute $1

        pp "Phase installation complete."
    fi
else
    echo "Usage: ./install_storm <number_of_phase>/all n/s <installdir>"
    echo "   "
    echo "Options:    "
    echo "n/s  n-> nimbus s-> supervisor"
    echo " "
    echo "Phases:"
    for ((i=0;i<${#PHASES[@]};i++))
    do
        echo -e "\t"$i": "${PHASES[$i]}
    done
fi
复制代码

 

 

 

分集剧情:

To install ZeroMQ, run:

wget http://download.zeromq.org/historic/zeromq-2.1.7.tar.gz
tar -xzf zeromq-2.1.7.tar.gz
cd zeromq-2.1.7
./configure
make
sudo make install

 

To install JZMQ, run:

git clone https://github.com/nathanmarz/jzmq.git
cd jzmq
./autogen.sh
./configure
make
sudo make install
分享到:
评论

相关推荐

    Install Cobbler on CentOS 7 -.mp4

    Install Cobbler on CentOS 7 Install Cobbler on CentOS 7 Install Cobbler on CentOS 7 Install Cobbler on CentOS 7

    Install of MS2019 on centos8.2

    1. Centos8.2启动盘制作;2. Centos8.2系统安装;3. Centos8.2系统环境设置;4. MS2019环境设置;5. 普通用户安装

    How To Install nginx on CentOS 6 with yum

    "安装 Nginx 服务器 trên CentOS 6 使用 yum" Nginx 是一个高性能的 Web 服务器软件,一般来说,它比 Apache 软件更加灵活和轻量级。在本教程中,我们将指导您如何在 CentOS 6 上使用 yum 安装 Nginx 服务器。 ...

    How to install innotop on centos

    标题 "如何在CentOS上安装innotop" 涉及到的是在Linux环境中,特别是CentOS操作系统上安装一个名为innotop的工具的过程。innotop是一款强大的MySQL和InnoDB性能监视工具,它提供了丰富的实时监控功能,帮助数据库...

    Auto install Mysql on CentOS

    1,将下载的镜像文件放到./usr/local 2,创建目录/data/mysql #mkdir -p /data/mysql 3,修改脚本内容,将“MYSQL='mysql-5.7.35-linux-glibc2.12-x86_64.tar.gz'” 改成对应的镜像名 4,sudo ln -s /usr/lib64/...

    Install_Oracle9204_on_CentOS4.2

    【标题】"Install_Oracle9204_on_CentOS4.2" 指的是在 CentOS 4.2 操作系统上安装 Oracle 9i Release 2 (9.2.0.4) 数据库的过程。Oracle 9i 是 Oracle 公司在 2001 年发布的数据库产品,它提供了数据存储、管理和...

    Install of Zacros on centos8.2

    5. 如果未安装make工具,则需要通过命令"dnf install make"来安装。dnf是CentOS 8的包管理工具,用于安装、更新、删除和管理软件包。 6. 接下来是Zacros的安装过程。首先需要解压Zacros的安装包,使用"unzip zacros...

    How to install VNC server on Centos 7

    如何在CentOS 7上安装VNC服务器 一、引言 VNC(Virtual Network Computing)是一种用于远程访问和控制桌面环境的图形化系统。它允许用户通过客户端软件(如VNC Viewer)来连接并控制远程计算机上的桌面环境。本文将...

    install ntop-ng 5.4 on centos 7.5 step by step

    install ntop-ng 5.4 on centos 7.5 step by step: 1. tar -xf ntop-ng5.4-centos7.tar 2. cd ng 3. yum install *.rpm 4. service redis start 5. ntopng -i interface-name -w 3000 5. open admin web in chrome ...

    Oracle11g install on CentOS 6.x

    1. TXT文档安装可直接拷贝 2. word文档供查看 3. 含打补丁步骤

    Install of cp2k on centos8.2.pdf

    CP2K is a quantum chemistry and solid state physics software package that can perform atomistic simulations of solid state, liquid, molecular, periodic, material, crystal, and biological systems.

    Installation OpenMeetings 4.0.10 on Centos 7.pdf

    安装的 Apache的 OpenMeetings 4.0.10在CentOS 7。Openmeetings提供视频会议,即时消息传递,白板,协作文档编辑和其他组件软件工具。它使用Media Server的API函数进行远程处理和流传输(Red5或Kurento)。

    Network Bonding On CentOS 7

    Network Bonding On CentOS 7

    SQLserver 2019 on centos 7

    [sudo] yum install -y mssql-server ``` 此命令将自动解决依赖关系并安装SQL Server 2019。安装过程中会列出待安装的软件包及它们的版本号。例如,根据提供的部分内容显示,将安装以下组件: - `mssql-server.x86...

    Centos7.0-RDO-install-Openstack

    Centos7.0-RDO-install-Openstack

    Oracle for Centos7 autoinstall

    整理编写相对比较全面的一个Oracle 单实例的自动安装脚本,使用环境仅限centos7,其他环境未测试。脚本自动配置Oracle的环境,在挂载本地光盘的前提下,自动验证光盘,从本地光盘yum 支持包,本地光盘没有的将忽略。...

    TensorFlow1.3.0 on centos6.5

    在本主题中,我们将深入探讨如何在 CentOS 6.5 操作系统上安装和使用 TensorFlow 1.3.0 版本。TensorFlow 是一个强大的开源机器学习库,由谷歌大脑团队开发,广泛用于深度学习和人工智能应用。CentOS 6.5 是一个基于...

    Centos7.4 VMware 12.5.5 install patch

    VMware Workstation for Linux v12.5.7-5813279 don't install on RedHat Enterprise Linux 7.4 Beta1 ( kernel: 3.10.0-663.el7 ) centos 7.4 下面安装VMWARE 12.5.5 成功后,在重新编译之前,将附件中的文件覆盖原...

    在CentOS上安装kubernetes详细指南

    ### 在CentOS上安装Kubernetes详细指南 #### 环境准备与角色分配 本指南主要介绍了如何在CentOS 7.2上安装Kubernetes集群。为了更好地理解安装过程,首先需要了解各节点的角色分配: - **Master/Node** (172.20.0...

Global site tag (gtag.js) - Google Analytics