客戶端遠程訪問高可用（HA）hdfs

當hadoop namenode是HA集群時，hdfs可能動態切換hdfs主節點，客戶端遠程訪問hdfs有兩種實現方法：

方法1：配置多個hdfs地址，每次操作先判斷可用的hdfs地址。

形如：hdfs://192.168.2.102:9000,hdfs://192.168.2.101:9000，以逗號（,）隔開

private void hdfsInit(String hdfs) {
        HdfsPath.setHdfs(hdfs);
        String[] pathes = HdfsPath.getHdfs().split(",");
        for(String path: pathes) {
            Configuration conf = new Configuration();

            conf.set("fs.defaultFS", path);
            conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());


            try (FileSystem fs = FileSystem.newInstance(conf)) {
                fs.listStatus(new Path("/"));
                HdfsPath.setHdfs(path);
                return ;
            } catch (IOException e) {
                logger.warn(path+"拒絕連接；"+e);
            }
        }
        logger.error("hdfs配置錯誤！"+hdfs);
    }

 

方法2：將所有關於namenode的參數寫入Configuration對象中。 代碼：

import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class Demo {

private static void uploadToHdfs() throws Exception {
//本地文件地址
String localSrc = "d:/PowerDesigner15_Evaluation.rar";
//存放在hdfs的目的地址
String dest = "/user/PowerDesigner15_Evaluation.rar";
InputStream in = new BufferedInputStream(new FileInputStream(localSrc));
//得到配置對象
Configuration conf = new Configuration();

conf.set("fs.defaultFS", "hdfs://ns1");
conf.set("dfs.nameservices", "ns1");
conf.set("dfs.ha.namenodes.ns1", "nn1,nn2");
conf.set("dfs.namenode.rpc-address.ns1.nn1", "192.168.2.101:9000");
conf.set("dfs.namenode.rpc-address.ns1.nn2", "192.168.2.102:9000");
conf.set("dfs.client.failover.proxy.provider.ns1", "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");

 

//文件系統
FileSystem fs = FileSystem.get(new URI("hdfs://ns1"), conf, "hadoop");
//輸出流
OutputStream out = fs.create(new Path(dest));
//連接兩個流，形成通道，使輸入流向輸出流傳輸數據
IOUtils.copyBytes(in, out, 4096, true);
}

public static void main(String[] args) throws Exception{
// TODO Auto-generated method stub
uploadToHdfs();
}

}