使用java進行Hadoop文件上傳下載和遍歷

使用java操作HDFS

需要使用到的jar包

將hadoop的tar.gz包解壓，里面的lib下的所有jar包，share/hadoop目錄下的common和hdfs文件下的所有jar包

以及Hadoop-common-2.7.7、Hadoop-hdfs-2.7.7、hadoop-client-2.7.7這三個jar包。

常用的操作

1.連接至hdfs

	@Test
    public void connectHDFS() {    
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.5.22:9000");
        try {
            FileSystem fileSystem = FileSystem.get(conf); 
            FileStatus fileStatus = fileSystem.getFileLinkStatus(new Path("/user"));
            System.out.println(fileStatus.isDirectory());    
        } catch (IOException e) {
            // TODO 自動生成的 catch 塊
            e.printStackTrace();
        }
	}

2.創建文件夾

@Test
    public void mkdir() {
        
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.5.22:9000");
        try {
            FileSystem fileSystem = FileSystem.get(conf);
            boolean mkdirs = fileSystem.mkdirs(new Path("/user/test")); //要創建的文件夾或多級文件夾
            System.out.println(mkdirs?"成功":"失敗");
        } catch (IOException e) {
            // TODO 自動生成的 catch 塊
            e.printStackTrace();
        }
    }

3.文件的上傳

①API直接上傳

    @Test
    public void upload1() {
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.5.22:9000");//連接hdfs
        try {
        	FileSystem fileSystem = FileSystem.get(conf);
        	FileInputStream in = new FileInputStream(new File("H:\\test.txt"));//獲取本地文件
        	FSDataOutputStream out = fileSystem.create(new Path("/upload1.txt")); //選擇上傳路徑
        	byte[] b = new byte[1024];
        	int len = 0;
            while((len=in.read(b))!=-1){
                out.write(b,0,len);
            }
            in.close();
            out.close();
        } catch (IOException e) {
            // TODO 自動生成的 catch 塊
            e.printStackTrace();
    }

②通過Hadoop的IOUtils上傳

	@Test 
    public void  upload2(){
        Configuration conf = new Configuration();
        try {
        	FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.5.22:9000"), conf);
       		FileInputStream in = new FileInputStream(new File("H:\\test.txt"));//獲取本地文件
        	FSDataOutputStream out = fileSystem.create(new Path("/upload2.txt")); //選擇上傳路徑
        	IOUtils.copyBytes(in,out,conf);
			in.close();
        	out.close();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

4.文件下載

①API直接下載

@Test
    public void download1(){
        //添加配置文件
        Configuration conf = new Configuration();
        try {
            FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.5.22:9000"), conf);
            FSDataInputStream in = fileSystem.open(new Path("/upload.txt"));//獲取服務器文件
            FileOutputStream out = new FileOutputStream(new File("H:\\download1.txt"));//選擇下載路徑，下載文件名
			byte[] b = new byte[1024];
            int len = 0;
            while((len=in.read(b))!=-1){
                out.write(b,0,len);
            }
			in.close();
            out.close();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

②通過IOUtils下載

@Test
    public void download1(){
        //添加配置文件
        Configuration conf = new Configuration();
        try {
            FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.5.22:9000"), conf);
            FSDataInputStream in = fileSystem.open(new Path("/upload.txt"));//獲取服務器文件
            FileOutputStream out = new FileOutputStream(new File("H:\\download2.txt"));//選擇下載路徑，下載文件名
			IOUtils.copyBytes(in,out,conf);
			in.close();
            out.close();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

5.修改文件名

@Test
    public void rename(){
        Configuration conf = new Configuration();
        try {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.5.22:9000"),conf);
            boolean rename = fileSystem.rename(new Path("/upload1.txt"), new Path("/upload3.txt"));
            System.out.println(rename?"修改成功":"修改失敗");
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

6.遍歷所有文件、文件夾

@Test
    public void lsr() {

        Configuration conf = new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.5.22:9000");
        try {
            FileSystem fileSystem = FileSystem.get(conf);
            FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));//獲取文件列表
            for (FileStatus fileStatus : listStatus) {
                isDir(fileStatus, fileSystem);//遍歷文件列表，判斷是文件還是文件夾
            }
        } catch (IOException e) {
            // TODO 自動生成的 catch 塊
            e.printStackTrace();
        }
    }

    public void isDir(FileStatus fileStatus,FileSystem fileSystem) {
        //如果是文件夾，則獲取該文件夾下的文件列表，遍歷判斷 遞歸調用
        if(fileStatus.isDirectory()) {
            String dirname = fileStatus.getPath().getName();
            System.out.println("Directory："+dirname);
            FileStatus[] listStatus;
            try {
                listStatus = fileSystem.listStatus(new Path("/"+dirname));
                for (FileStatus fileStatus2 : listStatus) {
                    isDir(fileStatus2,fileSystem);
                }
            } catch (Exception e) {
                // TODO 自動生成的 catch 塊
                e.printStackTrace();
            }
        }else {
            String dirname = fileStatus.getPath().getName();
            System.out.println("File:"+dirname);
        }
    }