webMagic入門案例,啟動流程,及從頁面中獲取元素,並放到Page對象中,並成功拿出


本例作用:

1)首次成功配置,了解軟件運行機制
2)從頁面中獲取元素三種方式 jsoup,xpath,css
3) page.putField("subject","嘗試存入結果項的內容,可從頁面中獲取值");
4)page.getResultItems().get("subject") 從結果集中拿出數據
5)List<String> links =page.getHtml().links().all(); 取出所有超鏈接
6)通過正則表達式找出結果集,放到到page中,並取出打印foreach()

 //找出所有鏈接下詳細招聘格式網址 http://sqrcw.com/zhaopin/job_233460.html
List<String> linkDetails =page.getHtml().links().regex("(^http://sqrcw.com\\/zhaopin\\/job_\\d*\\.html)").all();

        page.putField("linkDetails",linkDetails); //放入到Page對象中
        List<ResultItems> linkDetails_inPage = page.getResultItems().get("linkDetails");

 完整第一天學習代碼:

package com.ibaiqi.study;


import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;
import java.util.List;
import java.util.Map;

/**
 * 快速入門,了解webMagic啟動流程,及從頁面中選擇元素
 * 同時以www.sqrcw.com獲取詳細招聘信息
 */
public class Sqrcw_comProcessor implements PageProcessor {
    Site site=Site.me()
            .addCookie("userName2","zhangdaxu")
            .setRetryTimes(3).setSleepTime(100).setTimeOut(20000);

    @Override
    public void process(Page page) {


        System.out.println(site);
        System.out.println("開始采集"+page.getUrl());
     System.out.println("開始采集超鏈接"+page.getHtml().css(".s1 a"));
        //嘗試向結果項(采集存入項)中寫入數據
        page.putField("subject","嘗試存入結果項的內容");
        //測試從結果項中拿出數據
        System.out.println("取結果項值:"+page.getResultItems().get("subject"));
     
        page.putField("職位",page.getHtml().css(".s1 a"));
        //得到結果項,返回ResultItems
        System.out.println(""+page.getResultItems());
        System.out.println(page.getResultItems().get("subject"));
        //返回json的結果為json
        System.out.println(page.getJson());
        //獲得當前頁面超鏈接
        System.out.println("page.getUrl()");
        System.out.println(page.getUrl().all());
        System.out.println("selectAble() 從站點中查找部份內容");
        Selectable selectable=page.getHtml();
        System.out.println(selectable.$("#main1"));
        String url = page.getRequest().getUrl();
        System.out.println("url"+url);

        Map<String, String> cookies = page.getRequest().getCookies();
//        System.out.println("cookies"+cookies);
//        System.out.println(page.getCharset());
//        System.out.println(site.getCookies());
//        System.out.println(page.getStatusCode());
        System.out.println("得到起鏈接總數:"+page.getHtml().links().all().size());
        System.out.println("得到所有超鏈接:");
       List<String> links =page.getHtml().links().all();
       int i=0;
        for (String link:links ) {
            i++;
            System.out.println(i+"條:"+link);
        }
        System.out.println("正則表達式,找出招聘詳細頁面");
        //找出所有鏈接下詳細招聘格式網址 http://sqrcw.com/zhaopin/job_233460.html
        List<String> linkDetails =page.getHtml().links().regex("(^http://sqrcw.com\\/zhaopin\\/job_\\d*\\.html)").all();
        for (String linkDetail:
                linkDetails) {
            System.out.println(linkDetail);
        }
        System.out.println("詳細頁面總共:"+linkDetails.size());
        //放到page對象中,並取出,目的是為以后放進pipeline對象中,做持久化操作

        page.putField("linkDetails",linkDetails); //放入到Page對象中
        List<ResultItems> linkDetails_inPage = page.getResultItems().get("linkDetails");

        System.out.println("page對象中取出的數據");
        for (Integer j=0 ;j<linkDetails_inPage.size();j++){
            System.out.println(linkDetails_inPage.get(j));
        }

    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        System.out.println("開始采集001");
        //本類是實現 pageProcessor
        PageProcessor pageProcessor = new Sqrcw_comProcessor();
        //從才網查找數據詳細頁面測試
        Spider.create(pageProcessor).addUrl("http://sqrcw.com/job/")
                .addPipeline(new JsonFilePipeline("D:\\data\\webmagic"))
                .run();
        System.out.println("采集結束0001");

    }
}

 


免責聲明!

本站轉載的文章為個人學習借鑒使用,本站對版權不負任何法律責任。如果侵犯了您的隱私權益,請聯系本站郵箱yoyou2525@163.com刪除。



 
粵ICP備18138465號   © 2018-2025 CODEPRJ.COM