spark任务启动脚本备忘

#!/bin/bash

# Here, I am assuming that you want to run your Spark program in "Spark Cluster"
# Assuming that your Spark master is running on server "myserver100"
# This script is a kind of template ...
#   --------------------------------------------------------------------------------
#   1. You have installed the data-algorithms-book  in /home/mp/data-algorithms-book (BOOK_HOME)
#   2. Spark 1.5.2 is installed at /usr/local/spark-1.5.2
#   3. And you have built the source code and generated $DAB/dist/data_algorithms_book.jar
#   4. And you have two input parameters identified as P1 and P2
#   5. You need to modify spark-submit parameters accordingly
#   --------------------------------------------------------------------------------
#
export JAVA_HOME=/home/nianhua/soft/jdk1.8.0_45
# java is defined at $JAVA_HOME/bin/java
export BOOK_HOME=/data/spark/demo3
export SPARK_HOME=/home/nianhua/soft/spark-1.3.0-bin-hadoop2.4
export SPARK_MASTER=spark://tuijian-mnger.cando.site:7077
#export SPARK_JAR=$BOOK_HOME/lib/spark-assembly-1.5.2-hadoop2.6.0.jar
export APP_JAR=$BOOK_HOME/sparkwordcount.jar
#
# build all other dependent jars in OTHER_JARS
JARS=`find $BOOK_HOME/lib -name '*.jar'`
OTHER_JARS=""
for J in $JARS ; do
   OTHER_JARS=$J,$OTHER_JARS
done

#
echo $JAVA_HOME
echo ${JAVA_HOME}
P1=local
P2=1
DRIVER_CLASS_NAME=$1
nohup  $SPARK_HOME/bin/spark-submit --class $DRIVER_CLASS_NAME     --master $SPARK_MASTER     --num-executors 10     --driver-memory 2g     --executor-memory 5g  --total-executor-cores 12   --executor-cores 10 --driver-java-options "-Dspark.akka.frameSize=25"    --jars $OTHER_JARS        $APP_JAR $P1 $P2 &