Java實現多種方式的http數據抓取

前言：
　　時下互聯網第一波的浪潮已消逝，隨着而來的基於萬千數據的物聯網時代，因而數據成為企業的重要戰略資源之一。基於數據抓取技術，本文介紹了java相關抓取工具，並附上demo源碼供感興趣的朋友測試！

 

1）JDK自帶HTTP連接，獲取頁面或Json

 

2） JDK自帶URL連接，獲取頁面或Json

 

 

3）HttpClient Get工具，獲取頁面或Json

 

 

 4）commons-io工具，獲取頁面或Json

 

 

5） Jsoup工具（通常用於html字段解析），獲取頁面,非Json返回格式】

 

---

完整代碼：

package com.yeezhao.common.http;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.io.IOUtils;
import org.jsoup.Jsoup;

/**
 * http工具對比
 * 
 * @author Administrator -> junhong
 *
 *         2016年12月27日
 */
public class HttpFetchUtil {
    
    /**
     * 獲取訪問的狀態碼
     * @param request
     * @return
     * @throws Exception
     */
    public static int getResponseCode(String request) throws Exception {
        URL url = new URL(request);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        return conn.getResponseCode();

    }
    /**
     * 1）JDK自帶HTTP連接，獲取頁面或Json
     * @param request
     * @param charset
     * @return
     * @throws Exception
     */
    public static String JDKFetch(String request, String charset) throws Exception {
        URL url = new URL(request);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        //模擬瀏覽器參數
        conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36"
                + " (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");
        if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
            InputStream input = conn.getInputStream();
            StringBuffer sb = new StringBuffer();
            BufferedReader reader = new BufferedReader(new InputStreamReader(input, charset));
            String s;
            while ((s = reader.readLine()) != null) {
                sb.append(s + "\n");
            }
            input.close();
            conn.disconnect();
            return sb.toString();
        }
        return "";
    }
    /**
     * 2） JDK自帶URL連接，獲取頁面或Json
     * @param request
     * @param charset
     * @return
     * @throws Exception
     */
    public static String URLFetch(String request, String charset) throws Exception {
        URL url = new URL(request);
        return IOUtils.toString(url.openStream());
    }

    /**
     * 3）HttpClient Get工具，獲取頁面或Json
     * @param url
     * @param charset
     * @return
     * @throws Exception
     */
    public static String httpClientFetch(String url, String charset) throws Exception {
        // GET
        HttpClient httpClient = new HttpClient();
        httpClient.getParams().setContentCharset(charset);
        HttpMethod method = new GetMethod(url);
        httpClient.executeMethod(method);
        return method.getResponseBodyAsString();
    }
    /**
     * 4）commons-io工具，獲取頁面或Json
     * @param url
     * @param charset
     * @return
     * @throws Exception
     */
    public static String commonsIOFetch(String url, String charset) throws Exception {
        return IOUtils.toString(new URL(url), charset);
    }
    
    /**
     * 5） Jsoup工具（通常用於html字段解析），獲取頁面,非Json返回格式
     * @param url
     * @return
     * @throws Exception
     */
    public static String jsoupFetch(String url) throws Exception {
        return Jsoup.parse(new URL(url), 2 * 1000).html();
    }

}

 

測試代碼：

package com.yeezhao.common.http;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;
/**
 * 測試類
 * 3個測試鏈接：
 * 1）百科網頁
 * 2）瀏覽器模擬獲取接口數據
 * 3）獲取普通接口數據
 * @author Administrator -> junhong
 *
 * 2016年12月27日
 */
public class HttpFetchUtilTest {
    String seeds[] = {"http://baike.baidu.com/view/1.htm","http://m.ximalaya.com/tracks/26096131.json","http://remyapi.yeezhao.com/api/query?wd=%E5%91%A8%E6%98%9F%E9%A9%B0%E7%9A%84%E7%94%B5%E5%BD%B1"};
    final static String DEFAULT_CHARSET = "UTF-8";
    @Before
    public void setUp() throws Exception {
    }

    @After
    public void tearDown() throws Exception {
        System.out.println("--- down ---");
    }

    @Test
    public void testGetResponseCode() throws Exception{
        for(String seed:seeds){
            int responseCode = HttpFetchUtil.getResponseCode(seed);
            System.out.println("ret="+responseCode);
        }
    }

    @Test
    public void testJDKFetch() throws Exception{
        for(String seed:seeds){
            String ret = HttpFetchUtil.JDKFetch(seed, DEFAULT_CHARSET);
            System.out.println("ret="+ret);
        }
    }

    @Test
    public void testURLFetch() throws Exception{
        for(String seed:seeds){
            String ret = HttpFetchUtil.URLFetch(seed, DEFAULT_CHARSET);
            System.out.println("ret="+ret);
        }
    }

    @Test
    public void testHttpClientFetch()throws Exception {
        for(String seed:seeds){
            String ret = HttpFetchUtil.httpClientFetch(seed, DEFAULT_CHARSET);
            System.out.println("ret="+ret);
        }
    }

    @Test
    public void testCommonsIOFetch()throws Exception {
        for(String seed:seeds){
            String ret = HttpFetchUtil.commonsIOFetch(seed, DEFAULT_CHARSET);
            System.out.println("ret="+ret);
        }
    }

    @Test
    public void testJsoupFetch() throws Exception{
        for(String seed:seeds){
            String ret = HttpFetchUtil.jsoupFetch(seed);
            System.out.println("ret="+ret);
        }
    }

}

 

 

附：相關jar依賴

...
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.7.3</version>
</dependency>

<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
</dependency>

<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
...

 

后語：
　　現在的數據時代，有着"數據即財富"的理念。因此，數據抓取技術將一直發展更新，基於此后續還將擴充針對POST方法的抓取方式，敬請期待！