HDFS的java客戶端編寫

　　

　　注意:下面的所有代碼都是在linux的eclipse中進行編寫。

 

1.首先測試從hdfs中下載文件:

下載文件的代碼:(將hdfs://localhost:9000/jdk-7u65-linux-i586.tar.gz文件下載到本地/opt/download/doload.tgz)

package cn.qlq.hdfs;

import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.commons.compress.utils.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HdfsUtil {
    public static void main(String a[]) throws IOException {
        //to upload a file
        Configuration conf = new Configuration();
        
        FileSystem fs = FileSystem.get(conf);
        
        Path path = new Path("hdfs://localhost:9000/jdk-7u65-linux-i586.tar.gz");
        
        FSDataInputStream input = fs.open(path);
        
        FileOutputStream output = new FileOutputStream("/opt/download/doload.tgz");
        
        IOUtils.copy(input, output);
    }
}

 

 

直接運行報錯:

　　原因是程序不認識   hdfs://localhost:9000/jdk-7u65-linux-i586.tar.gz  這樣的目錄

log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread "main" java.lang.IllegalArgumentException: Wrong FS: hdfs://localhost:9000/jdk-7u65-linux-i586.tar.gz, expected: file:///
    at org.apache.hadoop.fs.FileSystem.checkPath(FileSystem.java:643)
    at org.apache.hadoop.fs.RawLocalFileSystem.pathToFile(RawLocalFileSystem.java:79)
    at org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:506)
    at org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:724)
    at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:501)
    at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:397)
    at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSInputChecker.<init>(ChecksumFileSystem.java:137)
    at org.apache.hadoop.fs.ChecksumFileSystem.open(ChecksumFileSystem.java:339)
    at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:764)
    at cn.qlq.hdfs.HdfsUtil.main(HdfsUtil.java:21)

 

 

 

解決辦法:

• 第一種:　將hadoop安裝目錄下的etc目錄中的core-site.xml拷貝到eclipse的src目錄下。這樣就不會報錯

 

 

運行結果:

[root@localhost download]# ll
total 140224
-rw-r--r--. 1 root root 143588167 Apr 20 05:55 doload.tgz
[root@localhost download]# pwd
/opt/download
[root@localhost download]# ll
total 140224
-rw-r--r--. 1 root root 143588167 Apr 20 05:55 doload.tgz

 

• 第二種:直接在程序中修改

　　我們先查看hdfs-site.xml中的內容:

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://localhost:9000</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/hadoop/hadoop-2.4.1/data/</value>
</property>
</configuration>

 

 

 代碼改為:

    public static void main(String a[]) throws IOException {
        //to upload a filed
        Configuration conf = new Configuration();
        //set hdfs root dir
        conf.set("fs.defaultFS", "hdfs://localhost:9000");
        
        FileSystem fs = FileSystem.get(conf);
        
        Path path = new Path("hdfs://localhost:9000/jdk-7u65-linux-i586.tar.gz");
        
        FSDataInputStream input = fs.open(path);
        
        FileOutputStream output = new FileOutputStream("/opt/download/doload.tgz");
        
        IOUtils.copy(input, output);
    }

 

 

 

 2.下面代碼演示了hdfs的基本操作:

package cn.qlq.hdfs;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.commons.compress.utils.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

public class HdfsUtil {
    private FileSystem fs = null;
    
    @Before
    public void befor() throws IOException, InterruptedException, URISyntaxException{
        //讀取classpath下的xxx-site.xml 配置文件，並解析其內容，封裝到conf對象中
        Configuration conf = new Configuration();
        
        //也可以在代碼中對conf中的配置信息進行手動設置，會覆蓋掉配置文件中的讀取的值
        conf.set("fs.defaultFS", "hdfs://localhost:9000/");
        
        //根據配置信息，去獲取一個具體文件系統的客戶端操作實例對象
        fs = FileSystem.get(new URI("hdfs://localhost:9000/"),conf,"root");
    }
    
    /**
     * 上傳文件，比較底層的寫法
     * 
     * @throws Exception
     */
    @Test
    public void upload() throws Exception {

        Path dst = new Path("hdfs://localhost:9000/aa/qingshu.txt");
        
        FSDataOutputStream os = fs.create(dst);
        
        FileInputStream is = new FileInputStream("/opt/download/haha.txt");
        
        IOUtils.copy(is, os);
        

    }
    

    /**
     * 上傳文件，封裝好的寫法
     * @throws Exception
     * @throws IOException
     */
    @Test
    public void upload2() throws Exception, IOException{
        fs.copyFromLocalFile(new Path("/opt/download/haha.txt"), new Path("hdfs://localhost:9000/aa/qingshu2.txt"));
        
    }
    
    
    
    /**
     * download file
     * @throws IOException
     */
    @Test
    public  void download() throws IOException {

        
        Path path = new Path("hdfs://localhost:9000/jdk-7u65-linux-i586.tar.gz");
        
        FSDataInputStream input = fs.open(path);
        
        FileOutputStream output = new FileOutputStream("/opt/download/doload.tgz");
        
        IOUtils.copy(input, output);
    }
    
    /**
     * 下載文件
     * @throws Exception 
     * @throws IllegalArgumentException 
     */
    @Test
    public void download2() throws Exception {
        fs.copyToLocalFile(new Path("hdfs://localhost:9000/aa/qingshu2.txt"), new Path("/opt/download/haha2.txt"));

    }
    
    /**
     * 查看文件信息
     * @throws IOException 
     * @throws IllegalArgumentException 
     * @throws FileNotFoundException 
     * 
     */
    @Test
    public void listFiles() throws FileNotFoundException, IllegalArgumentException, IOException {

        // listFiles列出的是文件信息，而且提供遞歸遍歷
        RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);
        
        while(files.hasNext()){
            
            LocatedFileStatus file = files.next();
            Path filePath = file.getPath();
            String fileName = filePath.getName();
            System.out.println(fileName);
            
        }
        
        System.out.println("---------------------------------");
        
        //listStatus 可以列出文件和文件夾的信息，但是不提供自帶的遞歸遍歷
        FileStatus[] listStatus = fs.listStatus(new Path("/"));
        for(FileStatus status: listStatus){
            
            String name = status.getPath().getName();
            System.out.println(name + (status.isDirectory()?" is dir":" is file"));
            
        }
        
    }

    /**
     * 創建文件夾
     * @throws Exception 
     * @throws IllegalArgumentException 
     */
    @Test
    public void mkdir() throws IllegalArgumentException, Exception {
        fs.mkdirs(new Path("/aaa/bbb/ccc"));
    }

    /**
     * 刪除文件或文件夾
     * @throws IOException 
     * @throws IllegalArgumentException 
     */
    @Test
    public void rm() throws IllegalArgumentException, IOException {
        fs.delete(new Path("/aa"), true);
    }
    
}