Slave2

1.mount

slave1@slave1-VirtualBox:~$ sudo mkdir /mnt/shared
[sudo] password for slave1:
mkdir: cannot create directory ‘/mnt/shared’: File exists
slave1@slave1-VirtualBox:~$ sudo mount -t vboxsf slave1 /mnt/shared/
slave1@slave1-VirtualBox:~$ ls /mnt/shared/

2.jdk

sudo mkdir /usr/local/jvm

sudo cp /mnt/shared/jdk-7u79-linux-x64.tar.gz /usr/local/jvm/
 cd /usr/local/jvm/

slave1@slave1-VirtualBox:/usr/local/jvm$ sudo tar zxf jdk-7u79-linux-x64.tar.gz

slave1@slave1-VirtualBox:/usr/local/jvm$ cd ~
slave1@slave1-VirtualBox:~$ sudo gedit .bashrc

export JAVA_HOME=/usr/local/jvm/jdk1.7.0_79
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH

source .bashrc

lave1@slave1-VirtualBox:~$ sudo update-alternatives --install /usr/bin/java java /usr/local/jvm/jdk1.7.0_79/bin/java 300
update-alternatives: using /usr/local/jvm/jdk1.7.0_79/bin/java to provide /usr/bin/java (java) in auto mode
slave1@slave1-VirtualBox:~$  sudo update-alternatives --install /usr/bin/javac javac /usr/local/jvm/jdk1.7.0_79/bin/javac 300
slave1@slave1-VirtualBox:~$ sudo update-alternatives --install /usr/bin/jar jar /usr/local/jvm/jdk1.7.0_79/bin/jar 300
slave1@slave1-VirtualBox:~$  sudo update-alternatives --install /usr/bin/jar jar /usr/local/jvm/jdk1.7.0_79/bin/jar 300
slave1@slave1-VirtualBox:~$ java -version

3.SSH

sudo apt-get install ssh

Package ssh is not available, but is referred to by another package

这个问题的原因是ubuntu的/etc/apt/source.list中的源比较旧了，需要更新一下，更新方法：

$ sudo apt-get -y update

更新完毕之后，在使用apt-get就没有问题了。

slave1@slave1-VirtualBox:~$ ps -e |grep ssh

slave1@slave1-VirtualBox:~$  ssh-keygen -t rsa -P ""

slave2@slave2-VirtualBox:~$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
slave1@slave1-VirtualBox:~$ ssh localhost

slave1@slave1-VirtualBox:~$ exit
4.rsync
slave1@slave1-VirtualBox:~$ sudo apt-get install rsync
5.Hadoop

slave1@slave1-VirtualBox:~$ sudo cp /mnt/shared/hadoop-2.6.0.tar.gz /usr/local/

slave1@slave1-VirtualBox:~$ cd /usr/local/
slave1@slave1-VirtualBox:/usr/local$ sudo tar zxf hadoop-2.6.0.tar.gz

lave1@slave1-VirtualBox:/usr/local$ cd hadoop-2.6.0/

slave2@slave2-VirtualBox:/usr/local/hadoop-2.6.0$ sudo mkdir tmp dfs dfs/name dfs/data
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ cd etc/hadoop/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/etc/hadoop$ sudo gedit hadoop-env.sh

export JAVA_HOME=/usr/local/jvm/jdk1.7.0_79
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/etc/hadoop$ cd ~
slave1@slave1-VirtualBox:~$ sudo gedit .bashrc

export PATH=${JAVA_HOME}/bin:/usr/local/hadoop-2.6.0/bin:$PATH

slave1@slave1-VirtualBox:~$ source .bashrc

slave1@slave1-VirtualBox:~$ hadoop version

slave1@slave1-VirtualBox:~$ cd /usr/local/hadoop-2.6.0/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo mkdir input
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo cp README.txt input/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.0.jar wordcount input output

 

error:

java.lang.Exception: java.io.IOException: Mkdirs failed to create file:/usr/local/hadoop-2.6.0/output/_temporary/0/_temporary/attempt_local2127094074_0001_r_000000_0 (exists=false, cwd=file:/usr/local/hadoop-2.6.0)
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ cd ..
slave1@slave1-VirtualBox:/usr/local$ sudo chmod -R 777 hadoop-2.6.0
slave1@slave1-VirtualBox:/usr/local$ sudo chown -R slave1:adm hadoop-2.6.0
slave1@slave1-VirtualBox:/usr/local$ cd hadoop-2.6.0/

slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ cat output/*
6.pre-sudo distribute

slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ cd etc/hadoop/

slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/etc/hadoop$ sudo gedit core-site.xml

<configuration>
    <property> 
        <name>hadoop.tmp.dir</name> 
        <value>/usr/local/hadoop-2.6.0/tmp</value> 
        <description>Abase for other temporary directories.</description> 
    </property> 
    <property> 
        <name>fs.defaultFS</name> 
        <value>hdfs://localhost:9000</value> 
    </property>
</configuration>

 slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/etc/hadoop$ sudo gedit hdfs-site.xml
<!-- Put site-specific property overrides in this file. -->

<configuration>
<property> 
        <name>dfs.replication</name> 
        <value>1</value> 
    </property> 
    <property> 
        <name>dfs.namenode.name.dir</name> 
        <value>file:/usr/local/hadoop-2.6.0/dfs/name</value> 
    </property> 
    <property> 
        <name>dfs.datanode.data.dir</name> 
        <value>file:/usr/local/hadoop-2.6.0/dfs/data</value> 
    </property> 
    <property>                 //这个属性节点是为了防止后面eclopse存在拒绝读写设置的 
            <name>dfs.permissions</name> 
            <value>false</value> 
     </property>
</configuration>

slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/etc/hadoop$ cd ../..
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo mkdir tmp
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo mkdir dfs dfs/data dfs/name
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo chmod -R 777 dfs/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo chmod -R 777 tmp/

slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo chmod -R 777 dfs/data/

slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hdfs namenode -format
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sbin/start-all.sh

slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ jps
NameNode - http://localhost:50070/

slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$  bin/hdfs dfs -mkdir /user
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hdfs dfs -mkdir /user/slave1
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hdfs dfs -mkdir /user/slave1/input
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hdfs dfs -put README.txt input
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.0.jar wordcount input output
Examine the output files:

Copy the output files from the distributed filesystem to the local filesystem and examine them:

  $ bin/hdfs dfs -get output output
  $ cat output/*

or

View the output files on the distributed filesystem:

  $ bin/hdfs dfs -cat output/*

When you're done, stop the daemons with:

  $ sbin/stop-dfs.sh