#spark-env.sh JAVA_HOME=/home/hadoop/app/jdk1.7.0_60 SCALA_HOME=/home/hadoop/app/scala-2.10.3 SPARK_HOME=/home/hadoop/app/spark-1.4.0 SPARK_PID_DIR=$SPARK_HOME/tmp HADOOP_CONF_DIR=/home/hadoop/app/hadoop/etc/hadoop SPARK_CLASSPATH=$SPARK_HOME/conf/:$SPARK_HOME/lib/*:/home/hadoop/app/hadoop/share/hadoop/common/lib/hadoop-lzo-0.4.19.jar:/home/hadoop/app/hbase/conf:/home/hadoop/app/hadoop/lib/native:$SPARK_CLASSPATH SPARK_JAVA_OPTS="$SPARK_JAVA_OPTS -Dspark.akka.askTimeout=300 -Dspark.ui.retainedStages=1000 -Dspark.eventLog.enabled=true -Dspark.eventLog.dir=hdfs://sparkcluster/user/spark_history_logs -Dspark.shuffle.spill=false -Dspark.shuffle.manager=hash -Dspark.yarn.max.executor.failures=99999 -Dspark.worker.timeout=300" SPARK_LOCAL_DIRS=/data1/hadoop/spark_local_dir,/data2/hadoop/spark_local_dir,/data3/hadoop/spark_local_dir,/data4/hadoop/spark_local_dir,/data5/hadoop/spark_local_dir,/data6/hadoop/spark_local_dir,/data7/hadoop/spark_local_dir,/data8/hadoop/spark_local_dir,/data9/hadoop/spark_local_dir,/data10/hadoop/spark_local_dir SPARK_MASTER_PORT=4050 SPARK_WORKER_CORES=30 SPARK_WORKER_MEMORY=60g SPARK_WORKER_INSTANCES=6 SPARK_DRIVER_MEMORY=12g SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=spark1:2181,spark2:2181,spark3:2181 $SPARK_DAEMON_JAVA_OPTS"
如何根據機器的情況合理的設置 SPARK_WORKER_CORES AND SPARK_WORKER_MEMORY?
查看機器的cpu信息：
# 總核數 = 物理CPU個數 X 每顆物理CPU的核數 # 總邏輯CPU數 = 物理CPU個數 X 每顆物理CPU的核數 X 超線程數 ## 查看物理CPU個數 cat /proc/cpuinfo| grep "physical id"| sort| uniq| wc -l # 查看邏輯CPU的個數 cat /proc/cpuinfo| grep "processor"| wc -l