使用Nodejs實現的小說爬蟲

//引入模塊
const http = require('http')
const fs = require('fs')
const cheerio = require('cheerio')
const iconv = require('iconv-lite')
//第一章url
const url = 'http://www.81zw.com/book/8634/745331.html'
//開始章節數
let i = 1
//最大獲取章節數
let num = 100

function main(url) {
    startRequest(url)
}

function startRequest(url) {
    http.get(url, res => {
        //定義空數組存放html
        const html = []
        res.on('data', (chunk) => {
            //把數據塊添加進數組
            html.push(chunk)
        })
        res.on('end', () => {
            //獲取數據完畢后，使用iconv-lite轉碼，decedo中為Buffer對象，Buffer.concat為數組
            const html1 = iconv.decode(Buffer.concat(html), 'gbk')
            //使用cheerio解析html，cheerio模塊的語法跟jQuery基本一樣
            const $ = cheerio.load(html1, {decodeEntities: false})
            //處理數據
            const title = $('.bookname h1').text()
            const arr = []
            const content = $("#content").html()
            //分析結構后分割html
            const contentArr = content.split('<br><br>')
            contentArr.forEach(elem => {
                //去除內容的兩端空格和&nbsp;
                const data = trim(elem.toString())
                arr.push(data)
            })
            const bookName = $(".con_top a").eq(2).text()
            //定義存入數據庫的對象
            const obj = {
                id: i,
                err: 0,
                bookName: bookName,
                title: title,
                content: arr
            }

            let url2 = url.split('/')[url.split('/').length - 2]
            const link = $(".bottem2 a").eq(2).attr('href')
            //獲取當前章節的下一章地址，遞歸調用fetchPage
            const nextLink = `http://www.81zw.com/book/${url2}/${link}`
            saveContent(obj, nextLink)
            console.log(`第${i + 1}章：${nextLink}`)
            i++
            if (i <= num) {
                setTimeout(() => {
                    main(nextLink)
                }, 100)
            }
        })
    })
}

function saveContent(obj, nextLink) {
    console.log(`${i}--${obj.title}`)
    //判斷書名文件夾是否存在，不存在則創建
    if (!fs.existsSync(`data/${obj.bookName}`)) {
        fs.mkdirSync(`data/${obj.bookName}`)
    }
    //寫入json文件
    fs.writeFile(`./data/${obj.bookName}/chapter${i}.json`, JSON.stringify(obj), 'utf-8', err => {
        if (err) throw err
    })
}

function trim(str) {
    return str.replace(/(^\s*)|(\s*$)/g, '').replace(/&nbsp;/g, '')
}

main(url)

生成文件