hadoop环境搭建

192.168.23.231 server1
192.168.23.234 server2
192.168.23.239 server3


http://mirrors.163.com/centos/6/isos/


关闭防火墙
service iptables stop
chkconfig iptables off
停闭selinux
vim /etc/sysconfig/selinux
SELINUX=disables


export JAVA_HOME=/usr/jdk1.8.0_25
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin

BASE_HOME=/myhome/usr
JAVA_HOME=/usr/jdk1.8.0_25
CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/jre/lib/rt.jar
PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH
export PATH

ssh-keygen -t rsa
cat id_rsa.pub>> authorized_keys
ssh root@192.168.23.234 cat ~/.ssh/id_rsa.pub>> authorized_keys
ssh root@192.168.23.239 cat ~/.ssh/id_rsa.pub>> authorized_keys

scp authorized_keys root@server2:~/.ssh/
scp authorized_keys root@server3:~/.ssh/


mkdir -p /myhome/usr#已安装的目录
mkdir -p /myhome/root
mkdir -p /myhome/tmp
mkdir -p /myhome/src
mkdir -p /myhome/logs
mkdir -p /myhome/dev
mkdir -p /myhome/bin
mkdir -p /myhome/usr/hadoop/yarn/local
mkdir -p /myhome/usr/hadoop/yarn/logs
mkdir -p /myhome/usr/hadoop/journal/zjenterprise
mkdir -p /myhome/usr/hadoop/hdfs/name
mkdir -p /myhome/usr/hadoop/hdfs/data
mkdir -p /myhome/usr/hadoop/tmp
mkdir -p /myhome/usr/hadoop/journal/
mkdir -p /myhome/usr/zookeeper/data/


touch /myhome/usr/zookeeper/data/myid
echo 1 > /myhome/usr/zookeeper/data/myid


core-site.xml
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://192.168.23.231:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/myhome/usr/hadoop/tmp</value>
    </property>
    <property>
        <name>io.file.buffer.size</name>
        <value>131702</value>
    </property>
</configuration>

hdfs-site.xml
<configuration>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/myhome/usr/hadoop/dfs/name</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:/myhome/usr/hadoop/dfs/data</value>
    </property>
    <property>
        <name>dfs.replication</name>
        <value>2</value>
    </property>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>192.168.23.231:9001</value>
    </property>
    <property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
    </property>
</configuration>

mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>192.168.23.231:10020</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>192.168.23.231:19888</value>
    </property>
</configuration>


yarn-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>
    <property>
        <name>yarn.resourcemanager.address</name>
        <value>192.168.23.231:8032</value>
    </property>
    <property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>192.168.23.231:8030</value>
    </property>
    <property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>192.168.23.231:8031</value>
    </property>
    <property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>192.168.23.231:8033</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>192.168.23.231:8088</value>
    </property>
    <property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>768</value>
    </property>
</configuration>


scp -r /myhome/usr/hadoop 192.168.23.234:/myhome/usr
scp -r /myhome/usr/hadoop 192.168.23.239:/myhome/usr
在Master服务器启动hadoop，从节点会自动启动，进入/home/hadoop/hadoop-2.7.0目录
(1)初始化，输入命令，bin/hdfs namenode -format
(2)全部启动sbin/start-all.sh，也可以分开sbin/start-dfs.sh、sbin/start-yarn.sh
(3)停止的话，输入命令，sbin/stop-all.sh
(4)输入命令，jps，可以看到相关信息