import java.net.URI;
import java.util.Iterator;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;
/**
*
* 客戶端去操作hdfs時，是有一個用戶身份的
* 默認情況下，hdfs客戶端api會從jvm中獲取一個參數來作為自己的用戶身份：-DHADOOP_USER_NAME=hadoop
*
* 也可以在構造客戶端fs對象時，通過參數傳遞進去
* @author
*
*/
public class HdfsClientDemo {
FileSystem fs = null;
Configuration conf = null;
@Before
public void init() throws Exception{
conf = new Configuration();
// conf.set("fs.defaultFS", "hdfs://mini1:9000");
conf.set("dfs.replication", "5");
//拿到一個文件系統操作的客戶端實例對象
fs = FileSystem.get(conf);
//可以直接傳入 uri和用戶身份
fs = FileSystem.get(new URI("hdfs://mini1:9000"),conf,"hadoop");
}
/**
* 上傳文件
* @throws Exception
*/
@Test
public void testUpload() throws Exception {
fs.copyFromLocalFile(new Path("c:/access.log"), new Path("/access.log.copy"));
fs.close();
}
/**
* 下載文件
* @throws Exception
*/
@Test
public void testDownload() throws Exception {
fs.copyToLocalFile(new Path("/access.log.copy"), new Path("d:/"));
}
/**
* 打印參數
*/
@Test
public void testConf(){
Iterator<Entry<String, String>> it = conf.iterator();
while(it.hasNext()){
Entry<String, String> ent = it.next();
System.out.println(ent.getKey() + " : " + ent.getValue());
}
}
@Test
public void testMkdir() throws Exception {
boolean mkdirs = fs.mkdirs(new Path("/testmkdir/aaa/bbb"));
System.out.println(mkdirs);
}
@Test
public void testDelete() throws Exception {
boolean flag = fs.delete(new Path("/testmkdir/aaa"), true);
System.out.println(flag);
}
/**
* 遞歸列出指定目錄下所有子文件夾中的文件
* @throws Exception
*/
@Test
public void testLs() throws Exception {
RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
while(listFiles.hasNext()){
LocatedFileStatus fileStatus = listFiles.next();
System.out.println("blocksize: " +fileStatus.getBlockSize());
System.out.println("owner: " +fileStatus.getOwner());
System.out.println("Replication: " +fileStatus.getReplication());
System.out.println("Permission: " +fileStatus.getPermission());
System.out.println("Name: " +fileStatus.getPath().getName());
System.out.println("------------------");
BlockLocation[] blockLocations = fileStatus.getBlockLocations();
for(BlockLocation b:blockLocations){
System.out.println("塊起始偏移量: " +b.getOffset());
System.out.println("塊長度:" + b.getLength());
//塊所在的datanode節點
String[] datanodes = b.getHosts();
for(String dn:datanodes){
System.out.println("datanode:" + dn);
}
}
}
}
@Test
public void testLs2() throws Exception {
FileStatus[] listStatus = fs.listStatus(new Path("/"));
for(FileStatus file :listStatus){
System.out.println("name: " + file.getPath().getName());
System.out.println((file.isFile()?"file":"directory"));
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://mini1:9000");
//拿到一個文件系統操作的客戶端實例對象
FileSystem fs = FileSystem.get(conf);
fs.copyFromLocalFile(new Path("c:/access.log"), new Path("/access.log.copy"));
fs.close();
}
}