搭建hadoop java開發環境

package hadoopDemo;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;

public class Aa {

    public static void main(String[] args)  throws Exception{
        Aa a=new Aa();
        
       
    } 
    public void upload(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
        //拿到一個文件系統與客戶端一個實例
        FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),
                new Configuration(),"root");
        filesys.copyFromLocalFile(new Path(src), new Path(dst));
        filesys.close();
    }
    public void download(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://linux1:9000"), new Configuration(), "root");
           fileSystem.copyToLocalFile(new Path(src), new Path(dst));
           fileSystem.close();
    }
    
    public void getAllFile() throws IOException, InterruptedException, URISyntaxException{
        Configuration cfg=new Configuration();
        FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
        RemoteIterator<LocatedFileStatus> listFiles=filesys.listFiles(new Path("/"), true);
        
        while(listFiles.hasNext()){
            LocatedFileStatus next2=listFiles.next();
            String name=next2.getPath().getName();
            System.out.println(next2.getPath()+"  "+name);
            
        }
    }
    
    public void mkdir(String dir) throws IOException, InterruptedException, URISyntaxException{
        Configuration cfg=new Configuration();
        FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
        filesys.mkdirs(new Path(dir));
        filesys.close();
    }
    public void mv(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
        Configuration cfg=new Configuration();
//        cfg.set("fs.defaultFs", "hdfs://linux1:9000");
        FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
        filesys.moveFromLocalFile(new Path(src), new Path(dst));
        filesys.close();
    }
    public void delete(String path) throws IOException, InterruptedException, URISyntaxException{
        Configuration cfg=new Configuration();
        FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
        filesys.delete(new Path(path), true);
        filesys.close();
    }
}

 

1.解壓hadoop-2.6.4.tar.gz

 

將此bin文件夾與hadoop-2.6.4文件夾中的bin文件夾合並

將此bin文件夾中的hadoop.dll文件拷貝到C:\Windows\System32目錄中

配置windows環境變量

控制面板——>系統——>更改設置——>高級——>環境變量

新建變量：HADOOP_HOME，路徑：解壓文件夾位置

Path后添加：;%HADOOP_HOME%/bin;%HADOOP_HOME%/sbin

測試生效：

cmd輸入hadoop

保險起見可以重啟電腦

2.創建user library

3.添加jar包

添加完jar包以后進行簡單測試：

 

package hadoopDemo;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;

public class Aa {

    public static void main(String[] args)  throws Exception{
        Aa a=new Aa();
        a.mv("e:/c.txt", "/aaa");　

     　　a.getAllFile();
     
　　　　　a.delete("/aaa");
     
　　　　　a.delete("/b.txt");　　　　       
    } 
    public void upload(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
        //拿到一個文件系統與客戶端一個實例
        FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),
                new Configuration(),"root");
        filesys.copyFromLocalFile(new Path(src), new Path(dst));
        filesys.close();
    }
    public void download(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://linux1:9000"), new Configuration(), "root");
           fileSystem.copyToLocalFile(new Path(src), new Path(dst));
           fileSystem.close();
    }
    
    public void getAllFile() throws IOException, InterruptedException, URISyntaxException{
        Configuration cfg=new Configuration();
        FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
        RemoteIterator<LocatedFileStatus> listFiles=filesys.listFiles(new Path("/"), true);
        
        while(listFiles.hasNext()){
            LocatedFileStatus next2=listFiles.next();
            String name=next2.getPath().getName();
            System.out.println(next2.getPath()+"  "+name);
            
        }
    }
    
    public void mkdir(String dir) throws IOException, InterruptedException, URISyntaxException{
        Configuration cfg=new Configuration();
        FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
        filesys.mkdirs(new Path(dir));
        filesys.close();
    }
    public void mv(String src,String dst) throws IOException, InterruptedException, URISyntaxException{
        Configuration cfg=new Configuration();
//        cfg.set("fs.defaultFs", "hdfs://linux1:9000");
        FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
        filesys.moveFromLocalFile(new Path(src), new Path(dst));
        filesys.close();
    }
    public void delete(String path) throws IOException, InterruptedException, URISyntaxException{
        Configuration cfg=new Configuration();
        FileSystem filesys=FileSystem.get(new URI("hdfs://linux1:9000"),cfg,"root");
        filesys.delete(new Path(path), true);
        filesys.close();
    }
}