#!/bin/bash
export JAVA_HOME=/usr/java/jdk1.8.0_202
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH
#set hadoop env
export HADOOP_HOME=/opt/cloudera/parcels/CDH
export PATH=${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:$PATH
#提交任務
spark2-submit \
--master yarn \
--num-executors 6 \
--executor-memory 8G \
--executor-cores 2 \
--driver-memory 16G \
--conf spark.default.parallelism=30 \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.storage.memoryFraction=0.5 \
--conf spark.shuffle.memoryFraction=0.3 \
--conf spark.dynamicAllocation.enable=false \
--conf "spark.driver.extraJavaOptions=-XX:+UseG1GC -Dlog4j.debug=true" \
--conf "spark.driver.extraJavaOptions=-Dlog4j.configuration=file:/xxx/log4j.properties" \
--conf "spark.executor.extraJavaOptions=-Dlog4j.configuration=file:/xxx/log4j.properties" \
--class xx xxx.jar xxx.properties