大數據排序的實現代碼， 理論上支持幾百億沒問題吧

先上說說思路，

 

1， 把一個bigdata文件拆分成N個小文件，小文件容量小於當前機器的內存

2，對小文件進行排序處理

3，對小文件進行並歸排序，代碼中我是用1 and 1，一個個並歸生成新的排序完成的文件，直到全部並歸完成

 

簡單說說我這里的並歸算法,代碼中的sortBySmallFile，如有文件A有n個元素， 文件B有m個元素

這里的並歸，先取出An0，分別比較Bn0,Bn1.... ，當An0>Bn,把Bn寫入新文件，直至An0<Bn 時把An0寫入結果文件，循環取出An1

 

OK,上代碼

 

package com.ben.file;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;

/**
 * 
 * @author Hejinbin
 * QQ 277803242
 * email [e-mail]
 */
public class BigDataSort {

	public final static String SMALL_FILE_PATH = "D://temp//BigData//";
	public final static int BIG_NUM_LINE = 1000000;
	public final static String ORING_FILE_PATH = "D://temp//BigData//bigData.txt";
	public final static int SMALL_FILE_LINE = 100000; //1M for 1 small file
	private File tempFiles[];

	public BigDataSort() throws IOException {
		createBigsortNums();
		beSmallFileAndSort();
		unitFileToSort();
	}

	private void createBigsortNums() throws IOException {
		BufferedWriter writer = new BufferedWriter(new FileWriter(ORING_FILE_PATH));
		Random random = new Random();
		for (int i = 0; i < BIG_NUM_LINE; i++) {
			writer.write(String.valueOf(random.nextInt(100000000)));
			writer.newLine();// add a new line . in order to show easy by file
		}
		writer.close();
	}

	private void beSmallFileAndSort() throws IOException {
		BufferedReader bigDataFile = new BufferedReader(new FileReader(ORING_FILE_PATH));
		List<Integer> smallLine = null;
		tempFiles = new File[BIG_NUM_LINE / SMALL_FILE_LINE];
		for (int i = 0; i < tempFiles.length; i++) {
			tempFiles[i] = new File(SMALL_FILE_PATH + "sortTempFile" + i + ".temp");
			BufferedWriter smallWtite = new BufferedWriter(new FileWriter(tempFiles[i]));
			smallLine = new ArrayList<Integer>();
			for (int j = 0; j < SMALL_FILE_LINE; j++)
				smallLine.add(Integer.parseInt(bigDataFile.readLine()));
			Collections.sort(smallLine);
			for (Object num : smallLine.toArray())
				smallWtite.write(num + "\n");
			smallWtite.close();
		}
	}

	private void unitFileToSort() throws IOException {
		File tempFile = null;
		for(int  i=1;i<tempFiles.length;i++){
			tempFile=sortBySmallFile(tempFiles[0],tempFiles[i]);
			tempFiles[0].delete();
			tempFiles[0]=tempFile;
		}
		tempFile.renameTo(new File(ORING_FILE_PATH+"sortResult.txt"));
	}

	public static File sortBySmallFile(File fromFile, File toFile) throws IOException {
		BufferedReader fromRd = new BufferedReader(new FileReader(fromFile));
		BufferedReader toTempRd = new BufferedReader(new FileReader(toFile));
		File newSortFile = new File(SMALL_FILE_PATH + fromFile.getName() + ".temp");
		BufferedWriter newSortFileWt = new BufferedWriter(new FileWriter(newSortFile));
		int index = -1;
		int toPoint = -1;
		while (fromRd.ready()) {
			index = Integer.parseInt(fromRd.readLine());
			if (index < toPoint) {
				newSortFileWt.write(String.valueOf(index));
				newSortFileWt.newLine();
				continue;
			} else {
				if (toPoint != -1) {
					newSortFileWt.write(String.valueOf(toPoint));
					newSortFileWt.newLine();
				}
			}
			while (toTempRd.ready()) {
				toPoint = Integer.parseInt(toTempRd.readLine());
				if (toPoint < index) {
					newSortFileWt.write(String.valueOf(toPoint));
					newSortFileWt.newLine();
				} else {
					newSortFileWt.write(String.valueOf(index));
					newSortFileWt.newLine();
					break;
				}
			}

		}
		newSortFileWt.write(String.valueOf(index>toPoint?index:toPoint));
		newSortFileWt.newLine();
		newSortFileWt.close();
		fromRd.close();
		toTempRd.close();
		toFile.delete();
		return newSortFile;
	}

	public static void main(String[] args) throws IOException {
		BigDataSort bigDataSort = new BigDataSort();
	}
}

　　注釋沒寫， 過幾天加上，可能有些許BUG在,歡迎討論