python入門 訪問網站並將爬回來的數據保存為csv文件

一、用requeses庫的個體（）函數訪問必應主頁20次，打印返回狀態，text內容，並且計算text（）屬性和content屬性返回網頁內容的長度

代碼如下：

import requests
def getHTMLText(url):
    try: 
        for i in range(0,20):                   #訪問20次
            r = requests.get(url, timeout=30)
        r.raise_for_status() #如果狀態不是200，引發異常
        r.encoding = 'utf-8' #無論原來用什么編碼，都改成utf-8
        return r.status_code,r.text,r.content,len(r.text),len(r.content)  ##返回狀態，text和content內容，text（）和content（）網頁的長度
    except:
        return ""
url = "https://cn.bing.com/?toHttps=1&redig=731C98468AFA474D85AECB7DB98B95D9"
print(getHTMLText(url))

運行結果如下：顯示結果太多  這里只截一部分

 

二、爬取的2019年中國最好大學的排名（這里只顯示排名前十的學校了）並且把它保存為csv文件

代碼如下：

import requests
import csv
import os
import codecs
from bs4 import BeautifulSoup
allUniv = []
def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return ""
def fillUnivList(soup):
    data = soup.find_all('tr')
    for tr in data:
        ltd = tr.find_all('td')
        if len(ltd)==0:
            continue
        singleUniv = []
        for td in ltd:
            singleUniv.append(td.string)
        allUniv.append(singleUniv)
def printUnivList(num):
    print("{:^4}{:^10}{:^5}{:^8}{:^10}".format("排名","學校名稱","省市","總分","培養規模"))
    for i in range(num):
        u=allUniv[i]
        print("{:^4}{:^10}{:^5}{:^8}{:^10}".format(u[0],u[1],u[2],u[3],u[6]))

def writercsv(save_road,num,title):
    if os.path.isfile(save_road):
        with open(save_road,'a',newline='')as f:
            csv_write=csv.writer(f,dialect='excel')
            for i in range(num):
                u=allUniv[i]
                csv_write.writerow(u)
    else:
         with open(save_road,'w',newline='')as f:
            csv_write=csv.writer(f,dialect='excel')
            csv_write.writerow(title)
            for i in range(num):
                u=allUniv[i]
                csv_write.writerow(u)
 
title=["排名","學校名稱","省市","總分","生源質量","培養結果","科研規模","科研質量","頂尖成果","頂尖人才","科技服務","產學研究合作","成果轉化"]
save_road="F:\\python\csvData.csv"
def main():
    url = 'http://www.zuihaodaxue.cn/zuihaodaxuepaiming2019.html'
    html = getHTMLText(url)
    soup = BeautifulSoup(html, "html.parser")
    fillUnivList(soup)
    printUnivList(10)
    writercsv('F:\\python\csvData.csv',10,title)
main()

代碼顯示結果如下：

打開文件: