nodeJS實現簡易爬蟲

nodeJS實現簡易爬蟲

需求：使用nodeJS爬取昵圖網某個分類下的圖片並存入本地

• 運用nodeJS自帶系統模塊http、fs

示例代碼:

var http =require('http');
var fs =require('fs');

var curentPage=1; //當前圖片頁數
var maxcurentPage=5;//最大頁數
//獲取圖片地址
function getData(){
    let url = 'http://www.nipic.com/photo/xiandai/jiaotong/index.html?page='+curentPage
    http.get(url,(res)=>{
        var data = '';
        res.on('data',(a)=>{
            data+=a.toString();
        })
        res.on('end',()=>{
            let reg = /<img src="(.+?)" data-src="(.*?)"  alt="(.*?)" \/>/g
            let arr=[];
            while (reg.exec(data)){
                arr.push(reg.exec(data)[2]);
            }
            for(i in arr){
                (function(i){
                    setTimeout(()=>{
                        getImg(arr[i])
                    },500*i)
                })(i)
            }
            if (curentPage < maxcurentPage){
                curentPage++;
                arguments.callee();
            }
        })
    })
}
//圖片寫入img文件夾
function getImg(url){
    let u = url.replace(/\/pic\//,'/file/')
        .replace(/_4.jpg/,'_2.jpg');
    http.get(u,(res)=>{
        let name = new Date().getTime();
        let stream = fs.createWriteStream('./img/' + name + '.png');
        res.pipe(stream);
    })
}
getData();