Storm WordCount Topology学习

1，分布式单词计数的流程

首先要有数据源，在SentenceSpout中定义了一个字符串数组sentences来模拟数据源。字符串数组中的每句话作为一个tuple发射。其实，SplitBolt接收SentenceSpout发射的tuple，它将每句话分割成每个单词，并将每个单词作为tuple发射。再次，WordCountBolt接收SplitBolt发送的tuple，它将接收到的每一个单词统计计数，并将 <单词：出现次数> 作为tuple发射。最后，ReportBolt接收WordCountBolt发送的tuple，将统计的结果存入HashMap中，并打印出结果。

流程图如下：

 

2，Topology的组成类

 ISpout、IComponent、IBolt三个接口定义了一些最基本的方法，BaseRichSpout、BaseRichBolt是接口的实现类，自定义的Spout与Bolt通过继承实现类来完成工作。

 详细解释参考代码里面的注释。参考《Storm分布式实时计算模式》第一章中的例子。

1， SentenceSpou.java分析：

private String[] sentences = { "my dog has fleas", "i like cold beverages",
             "the dog ate my homework", "don't have a cow man",
             "i don't think i like fleas" };

定义了待发射的数据源。Spout从该字符串数组一次取一个字符串生成tuple进行发射。

 

32     public void open(@SuppressWarnings("rawtypes") Map conf, 33 TopologyContext context, SpoutOutputCollector collector) { 34 // TODO Auto-generated method stub 35 this.collector = collector; 36 }

open函数，在ISpout接口中定义，所有的Spout组件在初始化时调用这个方法。在open()中初始化了发射器。

 

55     public void declareOutputFields(OutputFieldsDeclarer declarer) {
56         // TODO Auto-generated method stub
57         declarer.declare(new Fields("sentence"));// 标记SentenceSpout发送的tuple的键为
58                                                     // sentence
59     }

declareOutputFields函数标记了该Spout发射的tuple的(字段值)键值。下游的Bolt可以通过该键值来接收它发出的tuple

 

41     public void nextTuple() {
42         // TODO Auto-generated method stub
43         // 以字符串数组sentences 中的每个字符串 作为参数 构造tuple
44         this.collector.emit(new Values(sentences[index]));// 通过emit方法将构造好的tuple发送出去
45         index++;
46         if (index >= sentences.length) {
47             index = 0;
48         }
49         Utils.sleep(100);
50     }

nextTuple()是所有Spout的核心方法。Storm通过调用这个方法向collector发射tuple。Values.java 继承了ArrayList，new Values(...)构造了一个List对象，并将之作为emit的参数通过collector发射出去。

这里的发射规则是：每次发射其中一个字符串，阻塞100ms。当发射完整个字符串数组时，将索引(index)重新置0。可以继续发射。除非显示终止Topology，否则它不会停止。

 

 SentenceSpou.java代码如下：

 1 package org.apache.storm.storm_core; 2 3 import java.util.Map; 4 5 import backtype.storm.spout.SpoutOutputCollector; 6 import backtype.storm.task.TopologyContext; 7 import backtype.storm.topology.OutputFieldsDeclarer; 8 import backtype.storm.topology.base.BaseRichSpout; 9 import backtype.storm.tuple.Fields; 10 import backtype.storm.tuple.Values; 11 import backtype.storm.utils.Utils; 12 13 public class SentenceSpout extends BaseRichSpout { 14 /** 15 * 16 */ 17 private static final long serialVersionUID = 3444934973982660864L; 18 private SpoutOutputCollector collector;// 用来向其他Spout发射tuple 19 private String[] sentences = { "my dog has fleas", "i like cold beverages", 20 "the dog ate my homework", "don't have a cow man", 21 "i don't think i like fleas" }; 22 23 private int index = 0; 24 25 /* 26 * open() 方法在所有的Spout组件初始化时被调用 27 * 28 * @param Map conf storm 配置信息 29 * 30 * @context TopologyContext topology 组件信息 31 */ 32 public void open(@SuppressWarnings("rawtypes") Map conf, 33 TopologyContext context, SpoutOutputCollector collector) { 34 // TODO Auto-generated method stub 35 this.collector = collector; 36 } 37 38 /* 39 * Values.java extends ArrayList Storm 调用该方法向输出的collector发射tuple 40 */ 41 public void nextTuple() { 42 // TODO Auto-generated method stub 43 // 以字符串数组sentences 中的每个字符串 作为参数 构造tuple 44 this.collector.emit(new Values(sentences[index]));// 通过emit方法将构造好的tuple发送出去 45 index++; 46 if (index >= sentences.length) { 47 index = 0; 48 } 49 Utils.sleep(100); 50 } 51 52 /* 53 * SentenceSpout 发送的tuple它是一个包含键值对的List，该方法声明了List中包含的键值对的键为 sentence 54 */ 55 public void declareOutputFields(OutputFieldsDeclarer declarer) { 56 // TODO Auto-generated method stub 57 declarer.declare(new Fields("sentence"));// 标记SentenceSpout发送的tuple的键为 58 // sentence 59 } 60 }

 

 

SplitBolt.java代码如下：

 1 package org.apache.storm.storm_core; 2 3 import java.util.Map; 4 5 import backtype.storm.task.OutputCollector; 6 import backtype.storm.task.TopologyContext; 7 import backtype.storm.topology.OutputFieldsDeclarer; 8 import backtype.storm.topology.base.BaseRichBolt; 9 import backtype.storm.tuple.Fields; 10 import backtype.storm.tuple.Tuple; 11 import backtype.storm.tuple.Values; 12 13 public class SplitSentenceBolt extends BaseRichBolt { 14 /** 15 * 16 */ 17 private static final long serialVersionUID = -2107029392155190729L; 18 private OutputCollector collector;// 用来向其他Spout发射tuple的发射器 19 20 /* 21 * (non-Javadoc) prepare方法类似于open方法，prepare在bolt初始化时被调用 22 */ 23 public void prepare(Map stormConf, TopologyContext context, 24 OutputCollector collector) { 25 // TODO Auto-generated method stub 26 this.collector = collector;// 发射器初始化 27 28 } 29 30 public void execute(Tuple input) { 31 // TODO Auto-generated method stub 32 // 接收从SentenceSpout的发射器发射过来的tuple,因为SentenceSpout中声明的tuple字段为sentence,故getStringByField方法的参数为sentence 33 String sentence = input.getStringByField("sentence");// 该tuple是一个包含 34 // 键为sentence 35 // 值为字符串 36 // 的列表List<Map<sentence,String>> 37 String[] words = sentence.split(" ");// 将字符串分解成一个个的单词 38 for (String word : words) 39 this.collector.emit(new Values(word));// 将每个单词构造成tuple并发送给下一个Spout 40 } 41 42 public void declareOutputFields(OutputFieldsDeclarer declarer) { 43 // TODO Auto-generated method stub 44 declarer.declare(new Fields("word"));// 定义SplitSentenceBolt发送的tuple的字段("键值")为 word 45 } 46 }

 

WordCountBolt.java

package org.apache.storm.storm_core;

import java.util.HashMap;
import java.util.Map;

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;

public class WordCountBolt extends BaseRichBolt{

    private OutputCollector collector;
    private HashMap<String, Long>counts = null;//统计每个单词出现的次数,放到HashMap中保存起来
    
    public void prepare(Map stormConf, TopologyContext context,
            OutputCollector collector) {
        // TODO Auto-generated method stub
        this.collector = collector;
        this.counts = new HashMap<String, Long>();//初始化HashMap,因为prepare会被自动调用的
    }

    public void execute(Tuple input) {
        // TODO Auto-generated method stub
        String word = input.getStringByField("word");
        Long count = this.counts.get(word);
        if(count == null)//HashMap中没有word这个单词
            count = 0L;
        count++;
        this.counts.put(word, count);//更新该单词在HashMap中的统计次数
        //此处发射的tuple包含了两个元素:单词和计数，它每次发送的是一个长度为2的List，
        //可理解为：List.add(new HashMap("word",word)); List.add(new HashMap(("count",count));
        this.collector.emit(new Values(word, count));//第一个元素的键为 "word"，值为该单词(a string)，第二个键为 "count",值为单词的计数
    }

    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        // TODO Auto-generated method stub
        declarer.declare(new Fields("word", "count"));
    }
}

 

 

 ReportBolt.java如下：

package org.apache.storm.storm_core; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichBolt; import backtype.storm.tuple.Tuple; public class ReportBolt extends BaseRichBolt{ /** * */ private static final long serialVersionUID = 4921144902730095910L; // private OutputCollector collector; ReportBolt不需要发射tuple了 private HashMap<String, Long> counts = null; public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { // TODO Auto-generated method stub this.counts = new HashMap<String, Long>(); } public void execute(Tuple input) { // TODO Auto-generated method stub String word = input.getStringByField("word"); Long count = input.getLongByField("count"); this.counts.put(word, count); } public void declareOutputFields(OutputFieldsDeclarer declarer) { // TODO Auto-generated method stub //不需要发出任何数据流  } //Topology在storm集群中运行时，cleanup方法是不可靠的,并不能保证它一定会执行 public void cleanup(){ System.out.println("------ print counts ------"); List<String> keys = new ArrayList<String>(); keys.addAll(counts.keySet());//将HashMap中所有的键都添加到一个集合中 Collections.sort(keys);//对键(单词)进行排序 for(String key : keys)//输出排好序的每个单词的出现次数 System.out.println(key + " : " + this.counts.get(key)); System.out.println("--------bye----------"); } }

 

 

WordCountTopology.java如下：

 1 package org.apache.storm.storm_core;
 2 
 3 import backtype.storm.Config;
 4 import backtype.storm.LocalCluster;
 5 import backtype.storm.topology.TopologyBuilder;
 6 import backtype.storm.tuple.Fields;
 7 import backtype.storm.utils.Utils;
 8 
 9 public class WordCountTopology {
10     private static final String SENTENCE_SPOUT_ID = "sentence-spout";
11     private static final String SPLIT_BOLT_ID = "split-bolt";
12     private static final String COUNT_BOLT_ID = "count-bolt";
13     private static final String REPORT_BOLT_ID = "report-bolt";
14     private static final String TOPOLOGY_NAME = "word-count-topology";
15     
16     public static void main(String[] args) throws Exception{
17         SentenceSpout spout = new SentenceSpout();
18         SplitSentenceBolt splitBolt = new SplitSentenceBolt();
19         WordCountBolt countBolt = new WordCountBolt();
20         ReportBolt reportBolt = new ReportBolt();
21         
22         TopologyBuilder builder = new TopologyBuilder();
23         builder.setSpout(SENTENCE_SPOUT_ID, spout);
24         builder.setBolt(SPLIT_BOLT_ID, splitBolt).shuffleGrouping(SENTENCE_SPOUT_ID);
25         builder.setBolt(COUNT_BOLT_ID, countBolt).fieldsGrouping(SPLIT_BOLT_ID, new Fields("word"));
26         builder.setBolt(REPORT_BOLT_ID, reportBolt).globalGrouping(COUNT_BOLT_ID);
27         
28         Config config = new Config();
29         LocalCluster cluster = new LocalCluster();
30         
31         cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
32         Utils.sleep(1000);
33         cluster.killTopology(TOPOLOGY_NAME);
34         cluster.shutdown();
35         
36     }
37 }