本例作用:
1)首次成功配置,了解軟件運行機制
2)從頁面中獲取元素三種方式 jsoup,xpath,css
3) page.putField("subject","嘗試存入結果項的內容,可從頁面中獲取值");
4)page.getResultItems().get("subject") 從結果集中拿出數據
5)List<String> links =page.getHtml().links().all(); 取出所有超鏈接
6)通過正則表達式找出結果集,放到到page中,並取出打印foreach()
//找出所有鏈接下詳細招聘格式網址 http://sqrcw.com/zhaopin/job_233460.html List<String> linkDetails =page.getHtml().links().regex("(^http://sqrcw.com\\/zhaopin\\/job_\\d*\\.html)").all(); page.putField("linkDetails",linkDetails); //放入到Page對象中 List<ResultItems> linkDetails_inPage = page.getResultItems().get("linkDetails");
完整第一天學習代碼:
package com.ibaiqi.study; import us.codecraft.webmagic.Page; import us.codecraft.webmagic.ResultItems; import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Spider; import us.codecraft.webmagic.pipeline.JsonFilePipeline; import us.codecraft.webmagic.processor.PageProcessor; import us.codecraft.webmagic.selector.Selectable; import java.util.List; import java.util.Map; /** * 快速入門,了解webMagic啟動流程,及從頁面中選擇元素 * 同時以www.sqrcw.com獲取詳細招聘信息 */ public class Sqrcw_comProcessor implements PageProcessor { Site site=Site.me() .addCookie("userName2","zhangdaxu") .setRetryTimes(3).setSleepTime(100).setTimeOut(20000); @Override public void process(Page page) { System.out.println(site); System.out.println("開始采集"+page.getUrl()); System.out.println("開始采集超鏈接"+page.getHtml().css(".s1 a")); //嘗試向結果項(采集存入項)中寫入數據 page.putField("subject","嘗試存入結果項的內容"); //測試從結果項中拿出數據 System.out.println("取結果項值:"+page.getResultItems().get("subject")); page.putField("職位",page.getHtml().css(".s1 a")); //得到結果項,返回ResultItems System.out.println(""+page.getResultItems()); System.out.println(page.getResultItems().get("subject")); //返回json的結果為json System.out.println(page.getJson()); //獲得當前頁面超鏈接 System.out.println("page.getUrl()"); System.out.println(page.getUrl().all()); System.out.println("selectAble() 從站點中查找部份內容"); Selectable selectable=page.getHtml(); System.out.println(selectable.$("#main1")); String url = page.getRequest().getUrl(); System.out.println("url"+url); Map<String, String> cookies = page.getRequest().getCookies(); // System.out.println("cookies"+cookies); // System.out.println(page.getCharset()); // System.out.println(site.getCookies()); // System.out.println(page.getStatusCode()); System.out.println("得到起鏈接總數:"+page.getHtml().links().all().size()); System.out.println("得到所有超鏈接:"); List<String> links =page.getHtml().links().all(); int i=0; for (String link:links ) { i++; System.out.println(i+"條:"+link); } System.out.println("正則表達式,找出招聘詳細頁面"); //找出所有鏈接下詳細招聘格式網址 http://sqrcw.com/zhaopin/job_233460.html List<String> linkDetails =page.getHtml().links().regex("(^http://sqrcw.com\\/zhaopin\\/job_\\d*\\.html)").all(); for (String linkDetail: linkDetails) { System.out.println(linkDetail); } System.out.println("詳細頁面總共:"+linkDetails.size()); //放到page對象中,並取出,目的是為以后放進pipeline對象中,做持久化操作 page.putField("linkDetails",linkDetails); //放入到Page對象中 List<ResultItems> linkDetails_inPage = page.getResultItems().get("linkDetails"); System.out.println("page對象中取出的數據"); for (Integer j=0 ;j<linkDetails_inPage.size();j++){ System.out.println(linkDetails_inPage.get(j)); } } @Override public Site getSite() { return site; } public static void main(String[] args) { System.out.println("開始采集001"); //本類是實現 pageProcessor PageProcessor pageProcessor = new Sqrcw_comProcessor(); //從才網查找數據詳細頁面測試 Spider.create(pageProcessor).addUrl("http://sqrcw.com/job/") .addPipeline(new JsonFilePipeline("D:\\data\\webmagic")) .run(); System.out.println("采集結束0001"); } }