Oozie

#hadoop

export HADOOP_HOME=/home/oozie/hadoop-2.7.3

export PATH=$PATH:$HADOOP_HOME/bin

 

#oozie

export OOZIE_HOME=/home/oozie/oozie-5.1.0

export OOZIE_CONFIG=$OOZIE_HOME/conf

export CLASSPATH=$CLASSPATH:$OOZIE_HOME/bin

export PATH=$PATH:$OOZIE_HOME/bin

export OOZIE_URL=http://hadoop-senior.guppy.com:11000/oozie

 

wget http://archive.cloudera.com/gplextras/misc/ext-2.2.zip

wget https://www-us.apache.org/dist/oozie/5.1.0/oozie-5.1.0.tar.gz

tar -zxvf /home/software/oozie-5.1.0.tar.gz -C /home/oozie/

bin/mkdistro.sh -DskipTests -Puber

mv oozie-5.1.0 oozie-5.1.0_

tar -zxvf oozie-5.1.0_/distro/target/oozie-5.1.0-distro.tar.gz  -C /home/oozie/

cd oozie-5.1.0

cp /opt/software/ext-2.2.zip libext/

cd conf/

mkdir spark-conf

cd spark-conf/

vim spark-defaults.conf

# spark on yarn log
spark.yarn.historyServer   http://xx:18080
spark.eventLog.dir      hdfs://xx:9000/spark/historyLog
spark.eventLog.enabled    true

 

vim oozie-site.xml

<property>
   <name>oozie.service.ProxyUserService.proxyuser.root.hosts</name>
   <value>*</value>
</property>
<property>
<name>oozie.service.ProxyUserService.proxyuser.root.groups</name>
  <value>*</value>
</property>
<property>
 <name>oozie.service.JPAService.create.db.schema</name>
 <value>false</value>
</property>
<property>
<name>oozie.service.JPAService.jdbc.driver</name>
<value>com.mysql.jdbc.Driver</value>
</property>
<property>
        <name>oozie.service.JPAService.jdbc.url</name>
        <value>jdbc:mysql://192.168.0.189:3306/oozie?createDatabaseIfNotExist=true</value>
</property>
<property>
        <name>oozie.service.JPAService.jdbc.username</name>
        <value>oozie</value>
</property>
<property>
        <name>oozie.service.JPAService.jdbc.password</name>
        <value>123456</value>
</property>
<property>
        <name>oozie.service.HadoopAccessorService.hadoop.configurations</name>
        <value>*=/home/oozie/hadoop-2.7.3/etc/hadoop</value>
 </property>
<property>
        <name>oozie.service.HadoopAccessorService.action.configurations</name>
        <value>*=/home/oozie/hadoop-2.7.3/etc/hadoop</value>
</property>
<property>
         <name>oozie.service.SparkConfigurationService.spark.configurations</name>
         <value>*=spark-conf</value>
</property>
<property>
         <name>oozie.service.WorkflowAppService.system.libpath</name>
         <value>/user/root/share/lib</value>
</property>
<property>
        <name>oozie.use.system.libpath</name>
        <value>true</value>
</property>
<property>
        <name>oozie.subworkflow.classpath.inheritance</name>
        <value>true</value>
</property>

 

tar -zxvf /home/software/hadoop-2.7.3.tar.gz -C /home/oozie/

cd hadoop-2.7.3/

vim etc/hadoop/hadoop-env.sh 

export JAVA_HOME=/etc/alternatives/java_sdk_1.8.0/

 

vim etc/hadoop/hdfs-site.xml

     <!-- 指定HDFS副本的数量 -->
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>

    <property>
        <name>dfs.secondary.http.address</name>
        <value>192.168.0.189:50090</value>
    </property>

 

vim etc/hadoop/core-site.xml

    <!-- 指定HADOOP所使用的文件系统schema（URI），HDFS的老大（NameNode）的地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://192.168.0.189:9000</value>
    </property>
    <!-- 指定hadoop运行时产生文件的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/hadoop/hadoop-2.7.3/tmp</value>
    </property>
<!-- OOZIE -->
<property>
<name>hadoop.proxyuser.root.hosts</name>
<value>master</value>
</property>
<property>
<name>hadoop.proxyuser.root.groups</name>
<value>*</value>
</property>

 

bin/hdfs namenode -format

sbin/start-all.sh

 

bin/hadoop dfs -mkdir /user/root/examples

bin/hadoop dfs -put /home/oozie/oozie-5.1.0/examples/* /user/root/examples

 

vim oozie-5.1.0/conf/oozie-site.xml

bin/oozie-setup.sh

 

bin/oozie-setup.sh sharelib create -fs hdfs://192.168.0.189:9000 -locallib oozie-sharelib-5.1.0.tar.gz

cp /opt/modules/hive-2.3.6/lib/mysql-connector-java-5.1.46.jar ./lib

bin/ooziedb.sh create -sqlfile oozie.sql -run

bin/oozied.sh start

tar zxvf oozie-examples.tar.gz

vim examples/apps/spark/job.properties

vim examples/apps/spark/workflow.xml

bin/oozie job -oozie http://192.168.0.189:11000/oozie -config examples/apps/spark/job.properties -run