Python3.6多線程爬蟲

Python版本 3.6

簡單寫一個爬蟲,在寫的過程熟悉Python語法,不得不說Python用起來真666;

　　代碼功能是訪問網站首頁將所有a標簽值作為文件夾,將當前網頁所有圖片下載對應文件夾中;其實還有很多很多需要修改和完善的地方 比如異常,多線程,遞歸等;以后有機會再說吧.歡迎拍磚

# -*- UTF-8 -*-
from urllib import request
from bs4 import BeautifulSoup
import os
import time, threading


exe_Count = 1
aList = []

def CallView(url, timeout, directoryPath,exe_count):
    try:
        listAvalue = []
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 UBrowser/6.1.2716.5 Safari/537.36",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
        }
        rep = request.Request(url, headers=headers)
        response = request.urlopen(rep, timeout=timeout)
        soup = BeautifulSoup(response)
        # 獲取a標簽href 屬性並寫入list
        for a in soup.find_all("a"):
            if a.string is None:
                continue
            if not a.attrs["href"].strip() in aList:
                aList.append(a.attrs["href"].strip())
                listAvalue.append([a.string.strip()[0:11], a.attrs["href"].strip()])
            else:
                continue
        # 創建不存在的目錄
        if not os.path.exists(directoryPath):
            os.mkdir(directoryPath)
        print("新目錄：" + directoryPath)
        # 開啟線程遞歸
        thread = threading.Thread(target=ForRequest, args=(listAvalue, timeout, directoryPath,exe_count))
        thread.start()
        listImgSrc = []
        # 獲取img標簽 並下載
        for img in soup.find_all("img"):
            try:
                imgSrc = img.attrs["src"]
                print(imgSrc)
                # 過濾重復src
                if not imgSrc in listImgSrc:
                    listImgSrc.append(imgSrc)
                    # 讀取圖片
                    rep = request.Request(imgSrc)
                    response = request.urlopen(rep, timeout=timeout)
                    # 寫入圖片
                    filepath = directoryPath + "/" + imgSrc.split('/')[len(imgSrc.split('/')) - 1]
                    with open(filepath, "wb") as o:
                        o.write(response.read())
            except:
                print("訪問圖片或者寫入本地Error")
    except request.HTTPError as e:
        print(e.code)
    except:
        print("CallView Error")


def ForRequest(listA, timeout, directoryPath,exe_count):
    print("當前已執行：" + str(exe_count) + " 次")
    #調用次數超過200跳出
    if  exe_count == 2:
        thread = threading.current_thread()
        raise SystemError("正在停止線程")
    else:
        exe_count = exe_count + 1

    for info in listA:
        directoryChildPath = directoryPath + "/" + info[0]
        if not os.path.exists(directoryChildPath):
            os.mkdir(directoryChildPath)
        CallView(info[1], timeout, directoryChildPath, exe_count)

try:
    print("爬蟲開始活動了")
    CallView("http://www.xxxxx.com", 5000, "D:/PythonTest/Img/素材公社",exe_Count);
    print("爬蟲正在偷偷活動,不要着急哦！")
except:
    print("Error")