CentOS6.4安装Hadoop2.1beta

1.安装JDK7 rpm到/usr/java/jdk1.7.0_40

[root@server-308 ~]# rpm -ivh jdk-7u40-linux-x64.rpm
Preparing...                ########################################### [100%]
   1:jdk                    ########################################### [100%]
Unpacking JAR files...
        rt.jar...
        jsse.jar...
        charsets.jar...
        tools.jar...
        localedata.jar...
        jfxrt.jar...
[root@server-308 ~]# java -version
java version "1.7.0_40"
Java(TM) SE Runtime Environment (build 1.7.0_40-b43)
Java HotSpot(TM) 64-Bit Server VM (build 24.0-b56, mixed mode)

2.下载hadoop2.1beta并解压至/opt/hadoop-2.1.0-beta，并建立软链/opt/hadoop接到/opt/hadoop-2.1.0-beta

[root@server-308 ~]# cd /opt
[root@server-308 opt]# tar -zxvf /root/hadoop-2.1.0-beta.tar.gz
  [root@server-308 opt]# ln -s hadoop-2.1.0-beta hadoop
  [root@server-308 opt]# ls
  cedar hadoop hadoop-2.1.0-beta rh

3. 设置环境变量、

[root@server-308 opt]# vim /etc/profile

在文件末尾添加如下内容:

export JAVA_HOME=/usr/java/default
export PATH=${JAVA_HOME}/bin:${PATH}

export HADOOP_PREFIX=/opt/hadoop
export PATH=$PATH:$HADOOP_PREFIX/bin
export PATH=$PATH:$HADOOP_PREFIX/sbin
export HADOOP_MAPRED_HOME=${HADOOP_PREFIX}
export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
export YARN_HOME=${HADOOP_PREFIX}

使上述修改生效:[root@server-308 opt]# source /etc/profile
4.建立用户和组

[root@server-308 opt]# groupadd hadoop
[root@server-308 opt]# useradd -g hadoop yarn
[root@server-308 opt]# useradd -g hadoop hdfs
[root@server-308 opt]# useradd -g hadoop mapred

5.创建相关路径和文件

[root@server-306 opt]# mkdir -p /var/data/hadoop/hdfs/nn
[root@server-306 opt]# mkdir -p /var/data/hadoop/hdfs/snn
[root@server-306 opt]# mkdir -p /var/data/hadoop/hdfs/dn
[root@server-306 opt]# mkdir -p /opt/hadoop/logs
[root@server-306 opt]# mkdir -p /var/log/hadoop/yarn
[root@server-306 opt]# cd /opt/hadoop/logs
[root@server-306 logs]# chown hdfs:hadoop /var/data/hadoop/hdfs -R
[root@server-306 logs]# chown yarn:hadoop /var/log/hadoop/yarn -R
[root@server-306 logs]# chmod g+w /opt/hadoop/logs
[root@server-306 logs]# chown yarn:hadoop . -R

6.修改hadoop配置文件
vim /opt/hadoop/etc/hadoop/core-site.xml

<configuration>
  <property>
    <name>fs.default.name</name>
    <value>hdfs://192.168.32.31:8020</value>
  </property>
  <property>
    <name>hadoop.http.staticuser.user</name>
    <value>hdfs</value>
  </property>
</configuration>

7.修改hdfs-site.xml
vim /opt/hadoop/etc/hadoop/hdfs-site.xml

<configuration>
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>file:/var/data/hadoop/hdfs/nn</value>
  </property>
  <property>
    <name>fs.checkpoint.dir</name>
    <value>file:/var/data/hadoop/hdfs/snn</value>
  </property>
  <property>
    <name>fs.checkpoint.edits.dir</name>
    <value>file:/var/data/hadoop/hdfs/snn</value>
  </property>
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>file:/var/data/hadoop/hdfs/dn</value>
  </property>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
  <property>
    <name>dfs.permissions</name>
    <value>false</value>
  </property>
</configuration>

8.修改mapred-site.xml
#cp mapred-site.xml.template mapred-site.xml
#vi mapred-site.xml

<configuration>
  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
</configuration>

9.修改yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce.shuffle</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
  </property>
</configuration>

格式化集群

su - hdfs
cd /opt/hadoop
./hdfs namenode -format

启动hdfs

cd ../sbin
./hadoop-daemon.sh start namenode
./hadoop-daemon.sh start secondarynamenode
./hadoop-daemon.sh start datanode

启动yarn

su - yarn
cd /opt/hadoop/sbin
./yarn-daemon.sh start resourcemanager
./yarn-daemon.sh start nodemanager