java 对文件内容进行分词统计

本文记录了我在面试过程中感觉有用的问题，方便日后参考。

---

问题描述：
给定一个文本文件，按以下要求进行分词统计：

时间限制:5000ms
内存限制:256MB

要求1：读取文本信息（input.txt），设置分词大小，输出相应词频信息
要求2：统计一个单词在文本中的出现频率（一个单词出现次数/总单词数），排序输出结果

文本内容大致如下:

程序实现:

import java.util.Map;
import java.util.HashMap;
import java.util.Scanner;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
import java.math.BigDecimal;

public class Participle {
	
	public static void solutionOne(){
		Scanner in = new Scanner(System.in);
		print("请输入分词大小: ");
		int inputPartiSize = 0;
		while(in.hasNextInt()){
			inputPartiSize = in.nextInt();
			break;
		}
		
		final Map<String,Integer> dictFreq = new HashMap<String,Integer>();
		final int partiSize = inputPartiSize;
		
		readInput(new LineSolution(){
		
			public void solveLine(String line){
				String[] lineDicts = lineParser(line, partiSize);
				if(lineDicts != null){
					for(int i=0; i<lineDicts.length; i++){
						String word = lineDicts[i];
						if(dictFreq.containsKey(word)){
							int num = dictFreq.get(word);
							dictFreq.put(word, ++num);
						}
						else {
							dictFreq.put(word, 1);
						}
					}
				}
			}
		});
		
		for(Map.Entry<String,Integer> entry : dictFreq.entrySet()){
			print(entry.getKey() + "\t times: " + entry.getValue() + '\n');
		}
	}
	
	public static void solutionTwo(){
		print("方案二:\n");
		
		final Map<String,Integer> singleDictFreq = new HashMap<String,Integer>();
		final Map<String,Integer> callResult = new HashMap<String,Integer>();
		callResult.put("sum", 0);
		
		readInput(new LineSolution(){
		
			public void solveLine(String line){
				String[] lineDicts = lineParser(line, 1);
				if(lineDicts != null){
					callResult.put("sum", callResult.get("sum") + lineDicts.length);
					
					for(int i=0; i<lineDicts.length; i++){
						String word = lineDicts[i];
						if(singleDictFreq.containsKey(word)){
							int num = singleDictFreq.get(word);
							singleDictFreq.put(word, ++num);
						}
						else {
							singleDictFreq.put(word, 1);
						}
					}
				}
			}
		});
		
		Map<String,Double> singleDictFreqCalc = new HashMap<String,Double>();
		
		int sum = callResult.get("sum");
		for(Map.Entry<String,Integer> entry : singleDictFreq.entrySet()){
			singleDictFreqCalc.put(entry.getKey(), divide(entry.getValue(), sum));
		}
		
		for(Map.Entry<String,Double> entry : singleDictFreqCalc.entrySet()){
			print(entry.getKey() + "\t frequency: " + entry.getValue() + '\n');
		}
	}
	
	private static String[] lineParser(String line, int scale){
		String[] lineDicts = null;
		if(line != null && !"".equals(line.trim())){
			String[] spliter = line.split("\\s+|,");
			List<String> container = new ArrayList<String>();
			
			for(int i=0; i<spliter.length; i += scale){
				StringBuilder phase = new StringBuilder("");
				
				for(int j=0; (i+j) < spliter.length && j<scale; j++){
					phase.append(spliter[i+j]).append(" ");
				} 
				
				String phaseStr = phase.toString().trim();
				if("".equals(phaseStr)){
					continue;
				}
				container.add(phaseStr);
			}
			lineDicts = new String[container.size()];
			container.toArray(lineDicts);
		}
		return lineDicts;
	} 
	
	private static void readInput(LineSolution solution){
		try{
			String dir = System.getProperty("java.class.path");
			FileReader fr = new FileReader(dir + "/input.txt");
			BufferedReader br = new BufferedReader(fr);
			
			String line = "";
			while(line != null){
				line = br.readLine();
				if(line == null){
					break;
				}
				
				solution.solveLine(line);
			}
			
		}catch(IOException e){
			e.printStackTrace();
		}
	}
	
	private static double divide(double divisor, double dividend){
		return new BigDecimal(divisor)
			.divide(new BigDecimal(dividend), 5, BigDecimal.ROUND_HALF_UP).doubleValue();
	}
	
	public static void main(String[] args){
		
		String Q1 = "1：读取文本信息（input.txt），设置分词大小，输出相应词频信息";
		String Q2 = "2：统计一个单词在文本中的出现频率（一个单词出现次数/总单词数），排序输出结果";
		
		print(Q1+'\n');
		print(Q2+'\n');
		print("\n");
		
		print("请输入问题序号: ");
		Scanner in = new Scanner(System.in);
		
		while(in.hasNextInt()){
			int num = in.nextInt();
			if(num == 1){
				solutionOne();
				break;
			}
			else if(num == 2){
				solutionTwo();
				break;
			}
			else {
				print("\n请输入有效问题序号: ");
			}
		}
		
	}
	
	private static void print(String str){
		System.out.print(str);
	}
}

interface LineSolution {
	void solveLine(String line);
}

结果输出:

---