Java操作HDFS

安裝hadoop查看：CentOS安裝Hadoop

 

◆下載winutils.exe和hadoop.dll

   在windows平台下操作hadoop必須使用winutils.exe和hadoop.dll，下載地址：https://github.com/steveloughran/winutils，本文測試使用的是hadoop-2.8.3，雖然這個版本與服務器安裝的版本不一致 ，但經測試是有效的。

   配置環境變量：

    或者在Java代碼中設置：

System.setProperty("hadoop.home.dir", "D:\\hadoop-2.8.3");

 

◆修改core-site.xml配置

    設置fs.defaultFS參數值為hdfs://主機IP:9000

  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://192.168.107.141:9000</value>
  </property>

 

◆pom.xml，盡量讓hadoop-client版本與安裝的hadoop版本一致

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.zhi.test</groupId>
    <artifactId>hadoop-test</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <packaging>jar</packaging>

    <name>hadoop-test</name>
    <url>http://maven.apache.org</url>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <hadoop.version>2.9.2</hadoop.version>
        <!-- Logger -->
        <lg4j2.version>2.12.1</lg4j2.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>log4j</groupId>
                    <artifactId>log4j</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <!-- Logger（log4j2） -->
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>${lg4j2.version}</version>
        </dependency>
        <!-- Log4j 1.x API Bridge -->
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-1.2-api</artifactId>
            <version>${lg4j2.version}</version>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
    </dependencies>
</project>

 

◆Java代碼

package com.zhi.test.hadoop;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

/**
 * HDFS測試
 * 
 * @author zhi
 * @since 2019年9月10日18:28:24
 *
 */
public class HadoopTest {
    private Logger logger = LogManager.getLogger(this.getClass());
    private FileSystem fileSystem = null;

    @Before
    public void before() throws Exception {
//        System.setProperty("hadoop.home.dir", "D:\\hadoop-2.8.3");
        Configuration configuration = new Configuration();
        fileSystem = FileSystem.get(new URI("hdfs://192.168.107.141:9000"), configuration, "root");
    }

    @After
    public void after() throws Exception {
        if (fileSystem != null) {
            fileSystem.close();
        }
    }

    /**
     * 創建文件夾
     */
    @Test
    public void mkdir() {
        try {
            boolean result = fileSystem.mkdirs(new Path("/test"));
            logger.info("創建文件夾結果：{}", result);
        } catch (IllegalArgumentException | IOException e) {
            logger.error("創建文件夾出錯", e);
        }
    }

    /**
     * 上傳文件
     */
    @Test
    public void uploadFile() {
        String fileName = "hadoop.txt";
        InputStream input = null;
        OutputStream output = null;
        try {
            input = new FileInputStream("F:\\" + fileName);
            output = fileSystem.create(new Path("/test/" + fileName));
            IOUtils.copyBytes(input, output, 4096, true);
            logger.error("上傳文件成功");
        } catch (IllegalArgumentException | IOException e) {
            logger.error("上傳文件出錯", e);
        } finally {
            if (input != null) {
                try {
                    input.close();
                } catch (IOException e) {
                }
            }
            if (output != null) {
                try {
                    output.close();
                } catch (IOException e) {
                }
            }
        }
    }

    /**
     * 下載文件
     */
    @Test
    public void downFile() {
        String fileName = "hadoop.txt";
        InputStream input = null;
        OutputStream output = null;
        try {
            input = fileSystem.open(new Path("/test/" + fileName));
            output = new FileOutputStream("F:\\down\\" + fileName);
            IOUtils.copyBytes(input, output, 4096, true);
            logger.error("下載文件成功");
        } catch (IllegalArgumentException | IOException e) {
            logger.error("下載文件出錯", e);
        } finally {
            if (input != null) {
                try {
                    input.close();
                } catch (IOException e) {
                }
            }
            if (output != null) {
                try {
                    output.close();
                } catch (IOException e) {
                }
            }
        }
    }

    /**
     * 刪除文件
     */
    @Test
    public void deleteFile() {
        String fileName = "hadoop.txt";
        try {
            boolean result = fileSystem.delete(new Path("/test/" + fileName), true);
            logger.info("刪除文件結果：{}", result);
        } catch (IllegalArgumentException | IOException e) {
            logger.error("刪除文件出錯", e);
        }
    }

    /**
     * 遍歷文件
     */
    @Test
    public void listFiles() {
        try {
            FileStatus[] statuses = fileSystem.listStatus(new Path("/"));
            for (FileStatus file : statuses) {
                logger.info("掃描到文件或目錄，名稱：{}，是否為文件：{}", file.getPath().getName(), file.isFile());
            }
        } catch (IllegalArgumentException | IOException e) {
            logger.error("遍歷文件出錯", e);
        }
    }
}

訪問http://192.168.107.141:50070/explorer.html#，可查看上傳的文件：