python爬蟲案例-爬取當當網數據

　　輸入關鍵字，爬取當當網中商品的基本數據，代碼如下：

# Author:K
import requests
from lxml import etree
from fake_useragent import UserAgent
import re
import csv


def get_page(key):
    for page in range(1,50):
        url = 'http://search.dangdang.com/?key=%s&act=input&page_index=%s' % (key,page)
        headers = {
            'User-Agent':UserAgent().random
        }
        response = requests.get(url = url,headers = headers)
        parse_page(response)
        print('page %s over!!!' % page)

def parse_page(response):
    tree = etree.HTML(response.text)
    li_list = tree.xpath('//ul[@class="bigimg"]/li')
    # print(len(li_list))  # 測試
    for li in li_list:
        data = []
        try:
            # 獲取書的標題,並添加到列表中
            title = li.xpath('./p[@class="name"]/a/@title')[0].strip()
            data.append(title)
            # 獲取商品鏈接,並添加到列表中
            commodity_url = li.xpath('./p[@class="name"]/a/@href')[0]
            data.append(commodity_url)
            # 獲取價格,並添加到列表中
            price = li.xpath('./p[@class="price"]/span[1]/text()')[0]
            data.append(price)
            # 獲取作者,並添加到列表中
            author = ''.join(li.xpath('./p[@class="search_book_author"]/span[1]//text()')).strip()
            data.append(author)
            # 獲取出版時間,並添加到列表中
            time = li.xpath('./p[@class="search_book_author"]/span[2]/text()')[0]
            pub_time = re.sub('/','',time).strip()
            data.append(pub_time)
            # 獲取評論數,並添加到列表中
            comment_count = li.xpath('./p[@class="search_star_line"]/a/text()')[0]
            # 獲取書本的簡介,並添加到列表中.由於有些書本沒有簡介，所以要用try
            commodity_detail = ''
            commodity_detail = li.xpath('./p[@class="detail"]/text()')[0]
            data.append(commodity_detail)
        except:
            pass
        save_data(data)

def save_data(data):
    writer.writerow(data)


def main():
    key = 'python'  # input('Please input key:')
    get_page(key)

fp = open('當當網.csv','w',encoding = 'utf-8-sig',newline = '')
writer = csv.writer(fp)
header = ['標題','鏈接','價格','作者','出版時間','評論數','簡介']
writer.writerow(header)
main()
fp.close()