[实验]集群hadoop配置

环境
hadoop1.2.0
CentOS release 6.4
jdk1.7.0_25

node04vm01	192.168.15.146	master	NameNode,SecondaryNameNode,JobTracker
node04vm02	192.168.15.147	slave	DataNode,TaskTracker
node04vm03	192.168.15.148	slave	DataNode,TaskTracker
node04vm04	192.168.15.149	slave	DataNode,TaskTracker
node04vm05	192.168.15.150	slave	DataNode,TaskTracker

配置
配置ssh互信,java环境等

<configuration>
        <property>
                <name>fs.default.name</name>
                <value>hdfs://node04vm01:9000</value>
        </property>
</configuration>

<configuration>
        <property>
                <name>dfs.replication</name>
                <value>2</value>
        </property>
        <property>
                <name>dfs.permissions</name>
                <value>false</value>
        </property>
        <property>
                <name>dfs.name.dir</name>
                <value>/home/hue/hdfs/name</value>
        </property>
        <property>
                <name>dfs.name.edits.dir</name>
                <value>/home/hue/hdfs/edits</value>
        </property>
        <property>
                <name>dfs.data.dir</name>
                <value>/home/hue/hdfs/data</value>
        </property>
</configuration>

<configuration>
        <property>
                <name>mapred.job.tracker</name>
                <value>node04vm01:9001</value>
        </property>
</configuration>

node04vm01

node04vm02
node04vm03
node04vm04
node04vm05

初始化

bin/hadoop namenode -format

13/08/28 16:40:04 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG: Starting NameNode
STARTUP_MSG:   host = node04vm01/192.168.15.146
STARTUP_MSG:   args = [-format]
STARTUP_MSG:   version = 1.2.0
STARTUP_MSG:   build = https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.2 -r 1479473; compiled by 'hortonfo' on Mon May  6 06:59:37 UTC 2013
STARTUP_MSG:   java = 1.7.0_25
************************************************************/
13/08/28 16:40:04 INFO util.GSet: Computing capacity for map BlocksMap
13/08/28 16:40:04 INFO util.GSet: VM type       = 64-bit
13/08/28 16:40:04 INFO util.GSet: 2.0% max memory = 932118528
13/08/28 16:40:04 INFO util.GSet: capacity      = 2^21 = 2097152 entries
13/08/28 16:40:04 INFO util.GSet: recommended=2097152, actual=2097152
13/08/28 16:40:05 INFO namenode.FSNamesystem: fsOwner=hue
13/08/28 16:40:05 INFO namenode.FSNamesystem: supergroup=supergroup
13/08/28 16:40:05 INFO namenode.FSNamesystem: isPermissionEnabled=false
13/08/28 16:40:05 INFO namenode.FSNamesystem: dfs.block.invalidate.limit=100
13/08/28 16:40:05 INFO namenode.FSNamesystem: isAccessTokenEnabled=false accessKeyUpdateInterval=0 min(s), accessTokenLifetime=0 min(s)
13/08/28 16:40:05 INFO namenode.FSEditLog: dfs.namenode.edits.toleration.length = 0
13/08/28 16:40:05 INFO namenode.NameNode: Caching file names occuring more than 10 times
13/08/28 16:40:05 INFO common.Storage: Image file of size 109 saved in 0 seconds.
13/08/28 16:40:05 INFO common.Storage: Storage directory /home/hue/hdfs/name has been successfully formatted.
13/08/28 16:40:05 INFO namenode.FSEditLog: closing edit log: position=4, editlog=/home/hue/hdfs/edits/current/edits
13/08/28 16:40:05 INFO namenode.FSEditLog: close success: truncate to 4, editlog=/home/hue/hdfs/edits/current/edits
13/08/28 16:40:05 INFO common.Storage: Storage directory /home/hue/hdfs/edits has been successfully formatted.
13/08/28 16:40:05 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at node04vm01/192.168.15.146
************************************************************/

启动

bin/start-all.sh

验证
http://node04vm01:50030
http://node04vm01:50070

bin/hadoop fs -mkdir /test/input
bin/hadoop fs -put ~/test/senseandsensibility/chapter1.txt /test/input/
bin/hadoop jar hadoop-examples-1.2.0.jar wordcount /test/input /test/output