python 爬取百度雲資源

 

pan1

 1 import urllib.request
 2 import re
 3 import random
 4 
 5 def get_source(key):
 6 
 7     print('請稍等，爬取中....')
 8     headers = [{'User-Agent':'Mozilla/5.0 (Windows NT 6.3 WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.8.1000 Chrome/30.0.1599.101 Safari/537.36'},{'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:45.0) Gecko/20100101 Firefox/45.0"},{'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 10.0; WOW64; Trident/7.0)'}]
 9 
10     header = random.choice(headers) # 隨機選取一個header
11     
12     keyword = key.encode('utf-8')
13     keyword = urllib.request.quote(keyword)   # 對關鍵詞進行編碼
14 
15     # 這里是關鍵，稱隨機搜索一個資源名，然后對其結果網址進行分析
16     url = "http://www.wangpansou.cn/s.php?wp=0&ty=gn&op=gn&q="+keyword+"&q="+keyword
17     req = urllib.request.Request(url, headers=header)
18 
19     html = urllib.request.urlopen(req)
20 
21     # 編碼類型
22     head_type = html.headers['Content-Type'].split('=')[-1]   
23 
24 
25     status = html.getcode()   # 獲取狀態碼，只有訪問成功了才繼續。
26     if status == 200:
27         html = html.read()
28         html = html.decode(head_type) #　根據網站編碼時行解碼
29 
30         #　正則匹配　
31         pattern = re.compile('<a href="(.+)"><div class="cse-search-result_paging_num " tabindex="\d{1,3}">\d{1,3}</div></a>')
32         content = pattern.findall(html)
33 
34         url_list = []
35         url_head = 'http://www.wangpansou.cn/'
36         for i in content:
37             i = url_head +i # 因為正匹配出來的只有一部分,所以把前面的一部分加上,開成完整的鏈接
38             if not i in url_list:   # 去掉重復的,網頁確實有兩分,所以去重.
39                 url_list.append(i)   # 得到所有 有搜索結果的頁面網址列表
40 
41     count = 1  # 計數用
42     for each_url in url_list:
43         header = random.choice(headers)    # 針對每個鏈接都隨機選擇一個'頭',防止被服務器封掉
44         request1 = urllib.request.Request(each_url, headers=header)
45         html2 = urllib.request.urlopen(request1)
46 
47         status = html2.getcode()   # 獲取狀態碼
48         if status == 200:
49             html2 = html2.read()
50             html2 = html2.decode(head_type)
51             pattern1 = re.compile('<a class=".+" href="(.+)" rel.+')
52             content1 = pattern1.findall(html2)
53             
54 
55             pattern2 = re.compile('<div id=".+" class="cse-search-result_content_item_mid">\s+(.+)')
56             content2 = pattern2.findall(html2)
57 
58         
59         for i in range(0,len(content2)):
60             print(str(count) + ':' + content2[i] + '\n' + content1[i])
61             print()
62             count += 1
63 
64     print('共搜索到%d個資源,已經全部爬取完畢！' % count)
65 
66 if __name__ == '__main__':
67     get_source(input('請輸入要搜索的資源名:'))

 

 """

說明：
a.本搜索實際是通過通過網盤搜這個網站進行的二次搜索，如果找資源也可以直接到網盤搜進行一頁一頁的搜索

本腳本唯一的優點是一次性將所有結果全部爬下來，不用一頁一頁的翻找。

b.代碼相當丑，但這也是對學習過程的記錄，先實現功能，再考慮代碼。
"""

 

略作優化，同時修正了兩個小bug

pan2

# coding = 'utf-8'
import urllib.request
import re
import random

def get_html(url, header):
    req = urllib.request.Request(url, headers=header)
    html = urllib.request.urlopen(req)

    head_type = html.headers['Content-Type'].split('=')[-1]
    status = html.getcode()

    return html, head_type, status   # 分別得到html, 編碼方式， 訪問狀態碼


headers = [{'User-Agent':'Mozilla/5.0 (Windows NT 6.3 WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.8.1000 Chrome/30.0.1599.101 Safari/537.36'},{'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:45.0) Gecko/20100101 Firefox/45.0"},{'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 10.0; WOW64; Trident/7.0)'}]


keyword =input('請輸入要搜索的資源名:')
keyword = urllib.request.quote(keyword.encode('utf-8'))
url = "http://www.wangpansou.cn/s.php?wp=0&ty=gn&op=gn&q="+keyword+"&q="+keyword
header = random.choice(headers)

f_html, f_head_type, f_status = get_html(url, header)

if f_status == 200:
    f_html = f_html.read()
    f_html = f_html.decode(f_head_type)

    pattern = re.compile('<a href="(.+)"><div class="cse-search-result_paging_num " tabindex="\d{1,3}">\d{1,3}</div></a>')
    content = pattern.findall(f_html)   # 得到所有有相關結果的頁面鏈接

    url_list = []
    url_head = 'http://www.wangpansou.cn/'
    for i in content:
        i = url_head +i # 因為正匹配出來的只有一部分,所以把前面的一部分加上,開成完整的鏈接
        if not i in url_list:   # 去掉重復的,網頁確實有兩分,所以去重.
            url_list.append(i)   # 得到所有 有搜索結果的頁面網址列表
    first_url = url_list[0][:-2] + '0'  # 加上第一頁
    url_list.insert(0,first_url)

count = 0
for each_url in url_list:
    header = random.choice(headers)
    s_html, s_head_type, s_status = get_html(each_url, header)

    if s_status == 200:
        s_html = s_html.read()
        s_html = s_html.decode(s_head_type)
        s_pattern = re.compile('<a class=".+" href="(.+)" rel.+')
        s_content = s_pattern.findall(s_html) # 分享的鏈接 

        t_pattern = re.compile('<div id=".+" class="cse-search-result_content_item_mid">\s+(.+)')
        t_content = t_pattern.findall(s_html) # 文件信息
    else:
        print('Website Error!')

    for i in range(0, len(s_content)):
        count += 1
        print(str(count) + ':' + t_content[i] + '\n' + s_content[i])
        print()
        


print('共搜索到%d個資源,已經全部爬取完畢！' % count)