HttpClient&Jsoup爬蟲的簡單應用

　　詳細的介紹已經有很多前輩總結，引用一下該篇文章:https://blog.csdn.net/zhuwukai/article/details/78644484

　　下面是一個代碼的示例：

package com.http.client;

import java.io.IOException;

import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
/**
 * 
 * @author oo
 * @date 2018-04-04
 */
public class MyHttpClient {
    private static Logger logger = Logger.getLogger(MyHttpClient.class);

    /**
     * 需求：使用httpclient 爬取 網站數據
     * 
     * @param args
     */
    public static void main(String[] args) {
        // 創建ＨttpClient 對象
        HttpClient hclient = new DefaultHttpClient();

        // 設置響應時間 傳輸源碼時間 代理服務器（設置代理服務器的目的是：防止爬數據被封ip）
        hclient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 20000)
                .setParameter(CoreConnectionPNames.SO_TIMEOUT, 20000)
                .setParameter(ConnRouteParams.DEFAULT_PROXY, new HttpHost("111.155.116.237", 8123));

        HttpGet hGet = new HttpGet("http://www.itcast.cn/");
        String content = "";
        try {
            // 向網站發送請求，獲取網頁源碼
            HttpResponse execute = hclient.execute(hGet);
            // EntityUtils工具類把網頁實體轉換成字符串
            content = EntityUtils.toString(execute.getEntity(), "utf-8");
        } catch (ClientProtocolException e) {
            e.printStackTrace();
            logger.error("********ClientProtocolException" + e);
        } catch (IOException e) {
            e.printStackTrace();
            logger.error("********IOException" + e);
        }

        System.out.println(content);
    }

}

　　使用Jsoup進行請求：

package com.http.client;

import java.io.IOException;

import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class ＭyJsoup {
    private static Logger logger = Logger.getLogger(ＭyJsoup.class);

    public static void main(String[] args) {
        try {
            // 使用jsoup 發送請求
            Document document = Jsoup.connect("http://www.itcast.cn").get();
//            System.out.println(document);
            Elements elements = document.getElementsByTag("a");
            String val = elements.text();
            System.out.println(val);
            
            for (Element element : elements) {
                System.out.println(element.text()+":"+element.attr("href"));
            }
        } catch (IOException e) {
            e.printStackTrace();
            logger.error("***********IOException:　連接失敗" + e);
        }
    }

}

　　HttpClient 結合Jsoup：

package com.http.client;

import java.io.IOException;

import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class HttpCLientAndJsoup {

    public static void main(String[] args) throws ClientProtocolException, IOException {
        // 創建HttpClient對象
        HttpClient hClient = new DefaultHttpClient();
        // 爬蟲URL大部分都是get請求，創建get請求對象
        HttpGet hget = new HttpGet("http://www.itcast.cn/");
        // 向網站發送請求，獲取網頁源碼
        HttpResponse response = hClient.execute(hget);
        // EntityUtils工具類把網頁實體轉換成字符串
        String content = EntityUtils.toString(response.getEntity(), "utf-8");
        // Jsoup負責解析網頁
        Document doc = Jsoup.parse(content);
        // 使用元素選擇器選擇網頁內容
        Elements elements = doc.select("div.salary_con li");
        // System.out.println(elements.text());
        for (Element element : elements) {
            String text = element.text();
            System.out.println(text);
        }

    }

}