hadoop之HDFS核心類Filesystem的使用

1.導入jar包，要使用hadoop的HDFS就要導入hadoop-2.7.7\share\hadoop\common下的3個jar包和lib下的依賴包、hadoop-2.7.7\share\hadoop\hdfs下的3個jar包和lib下的依賴包

2.-ls 查看目錄下的所有文件和文件夾

        @Test
       	public void listStatus() {
		Configuration conf = new Configuration();
        //使用hdfs的fs功能，客戶端就會訪問core-site.xml配置文件
        //這里是設置core-site.xml中的屬性fs.defaultFS和屬性值hdfs://192.168.xx.xx:9000
        //注意寫自己的ip地址
		conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
		try {
			FileSystem fileSystem = FileSystem.get(conf);
                        //查看的路徑
			FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
			for(int i = 0; i < listStatus.length; ++i)
            {		String dpath = listStatus[i].getPath().toString();
                    System.out.println(dpath);
            }
			fileSystem.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

3.-lsr 或者 -ls -R 遞歸查看

    @Test
	public void lsrtest() {
                // 要遞歸遍歷的路徑
		lsr("/");
	}
	
	public static List<String> lsr(String path) {
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
		try {
			FileSystem fileSystem = FileSystem.get(conf);
			FileStatus[] listStatus = fileSystem.listStatus(new Path(path));
			for(int i = 0; i < listStatus.length; ++i)
            {
					String abpath = listStatus[i].getPath().toString();
					System.out.println(abpath);
                    boolean directory = listStatus[i].isDirectory();
                    if (directory) {
						lsr(abpath);
					}
            }
			fileSystem.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return null;
	}

4.-mkdir 創建文件夾

	@Test
	public void mkdir() {
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
		try {
			FileSystem fileSystem = FileSystem.get(conf);
			boolean mkdirs = fileSystem.mkdirs(new Path("/lyx02/lyx002/lyx0002"));
			System.out.println(mkdirs?"創建成功":"創建失敗");
		}catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

5.-touchz 創建文件

        @Test
	public void createNewFile() {
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
		try {
			FileSystem fileSystem = FileSystem.get(conf);
			boolean createNewFile = fileSystem.createNewFile(new Path("/lyx02/lyx002/22.txt"));
			System.out.println(createNewFile?"創建成功":"創建失敗");
		}catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

6.-put 下載hdfs下的文件到主機

@Test
	public void put() {
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
		try {
			FileSystem fileSystem = FileSystem.get(conf);
        //hdfs下的路徑
			FSDataInputStream in = fileSystem.open(new Path("/1.txt"));
			FileOutputStream destFile = new FileOutputStream(new File("D:\\111.txt"));
			BufferedOutputStream out = new BufferedOutputStream(destFile);
			
			int count = -1;
			byte[]buffer = new byte[1024 *8];
			while((count=in.read(buffer))!=-1) {
				out.write(buffer,0,count);
			}
			in.close();
			out.close();
		}catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

7.-get 上傳主機文件到hdfs

	@Test
	public void get() {
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
		try {
			FileSystem fileSystem = FileSystem.get(conf);
			FSDataOutputStream out = fileSystem.create(new Path("/555.txt"));
			FileInputStream srcFile = new FileInputStream(new File("D:\\111.txt"));
			BufferedInputStream in = new BufferedInputStream(srcFile);
			
			int count = -1;
			byte[]buffer = new byte[1024 *8];
			while((count=in.read(buffer))!=-1) {
				out.write(buffer,0,count);
			}
		}catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

8.-copyFromLocalFile 同get

	@Test
	public void copyFromLocalFile() {
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
		try {
			FileSystem fileSystem = FileSystem.get(conf);
			fileSystem.copyFromLocalFile(new Path("D:\\111.txt"), new Path("/666.txt"));
		}catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

9.-copyToLocalFile 同put

@Test
	public void copyToLocalFile() {
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.xx.xx:9000");
		try {
			FileSystem fileSystem = FileSystem.get(conf);
			fileSystem.copyToLocalFile(false,new Path("/666.txt"),new Path("D:\\666.txt"),true);
		}catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}