spark累计器
因为task的执行是在多个Executor中执行，所以会出现计算总量的时候，每个Executor只会计算部分数据，不能全局计算。
累计器是可以实现在全局中进行累加计数。
注意：
累加器只能在driver端定义，driver端读取，不能在Executor端读取。
广播变量只能在driver端定义，在Executor端读取，Executor不能修改。
下面是实践的代码：
package SparkStreaming; import org.apache.commons.collections.iterators.ArrayListIterator; import org.apache.commons.io.LineIterator; import org.apache.spark.Accumulator; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function; import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFunction; import scala.Tuple2; import java.util.Iterator; import java.util.List; public class totalization_device { public static void main(String[] args) { SparkConf conf = new SparkConf() .setMaster("local[2]") .setAppName("totalization_device"); JavaSparkContext sc = new JavaSparkContext(conf); /* * 定义一个累加器 * */ Accumulator<Integer> accumulator = sc.accumulator(0); JavaRDD<String> fileRDD = sc.textFile("E:/2018_cnic/learn/wordcount.txt"); JavaRDD<String> fileRDD1 = fileRDD.flatMap(new FlatMapFunction<String, String>() { @Override public Iterator<String> call(String s) throws Exception { accumulator.add(1); return new ArrayListIterator(s.split(" ")); } }); JavaPairRDD<String, Integer> pairRDD = fileRDD1.mapToPair(new PairFunction<String, String, Integer>() { @Override public Tuple2<String, Integer> call(String s) throws Exception { return new Tuple2<String, Integer>(s,1); } }); JavaPairRDD<String, Integer> reducebykeyRDD = pairRDD.reduceByKey(new Function2<Integer, Integer, Integer>() { @Override public Integer call(Integer integer, Integer integer2) throws Exception { return integer + integer2; } }); List<Tuple2<String, Integer>> collect = reducebykeyRDD.collect(); for(Tuple2 tup:collect){ System.out.println(tup); } Integer num = accumulator.value(); System.out.println("一共有："+num+"行"); sc.close(); } }
结果输出：
19/04/30 15:11:49 INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 3) in 41 ms on localhost (executor driver) (1/2) 19/04/30 15:11:49 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 2) in 42 ms on localhost (executor driver) (2/2) 19/04/30 15:11:49 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 19/04/30 15:11:49 INFO DAGScheduler: ResultStage 1 (collect at totalization_device.java:50) finished in 0.051 s 19/04/30 15:11:49 INFO DAGScheduler: Job 0 finished: collect at totalization_device.java:50, took 0.273877 s (4,1) (authentication,1) (Registered,3) (is,1) (Found,3) (master.Master:,3) (spark.SecurityManager:,5) (util.log:,1) (19,1) (modify,4) (classes,1) (6,1) ([jar:file:/opt/workspace/hive-3.1.0/lib/log4j-slf4j-impl-2.10.0.jar!/org/slf4j/impl/StaticLoggerBinder.class],1) (8080.,1) (type,1) (with,5) (INFO,17) (permissions:,3) (groups,4) (using,1) (19/04/24,18) (Class,1) (@1326ms,1) (WARN,1) (root,2) (signal,3) ('MasterUI',1) (13,1) (24,1) (Set(root);,2) (version,1) (11,1) (ui,1) (8,1) (load,1) (Set();,1) (20,1) (15,1) (14:23:51,14) (Actual,1) (initialized,1) (server.Server:,2) (master,1) (,34) (multiple,1) (56130C,1) (handler,3) (22,1) (26,1) (TERM,1) (17,1) (daemon,1) (bindings.,1) (builtin-java,1) (server.AbstractConnector:,1) (users,2) ([jar:file:/opt/workspace/hbase-1.4.6/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class],1) (http://www.slf4j.org/codes.html#multiple_bindings,1)
(105L,,1) (Starting,1) (jetty-9.3.z-SNAPSHOT,1) (Spark,2) (14,1) (SLF4J,1) (platform...,1) (7,1) (util.NativeCodeLoader:,1) (Successfully,2) (on,2) ('sparkMaster',1) (library,1) (service,2) (16,1) (at,1) (in,3) (9,1) (443265@master1,1) (See,1) (7077.,1) (Logging,1) (missions:,1) (util.Utils:,2) (spark://master1:7077,1)
(for,5) (Changing,4) (25,1) (native-hadoop,1) (port,2) (Running,1) (explanation.,1) (your,1) (view,4) (acls,5) (10,1) (Unable,1) (binding,4) (to:,4) (disabled;,2) (contains,1) (util.SignalUtils:,3) (process,1) (21,1) (SLF4J:,6) (ServerConnector@1cbf22af{HTTP/1.1,[http/1.1]}{0.0.0.0:8080},1) (23,1) (5,1) (18,1) (SecurityManager:,1) (Started,3) (INT,1) (Set(),1) ("spark-root-org.apache.spark.deploy.master.Master-1-master1.out",1) (to,1) (applicable,1) (HUP,1) (started,2) (of,1) (path,1) (where,1) (12,1) (an,1) ([jar:file:/opt/workspace/hadoop-2.9.1/share/hadoop/common/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class],1) ([org.slf4j.impl.Log4jLoggerFactory],1) (2.3.0,1) (14:23:50,4) (@1280ms,1) (name:,1) (per,1) 一共有：25行 19/04/30 15:11:49 INFO SparkUI: Stopped Spark web UI at http://hadoop:4040
19/04/30 15:11:49 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
19/04/30 15:11:49 INFO MemoryStore: MemoryStore cleared 19/04/30 15:11:49 INFO BlockManager: BlockManager stopped 19/04/30 15:11:49 INFO BlockManagerMaster: BlockManagerMaster stopped 19/04/30 15:11:49 INFO