爬蟲初學——爬取中國大學排名並存為csv文件

鏈接：軟科中國最好大學排名2016

代碼：

# -*- coding: utf-8 -*-
"""
Created on Mon May 27 21:10:59 2019

@author: Benny
"""

import csv
import os
import requests
import pandas
from bs4 import BeautifulSoup
allUniv = []
def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return ""
def fillUnivList(soup):
    data = soup.find_all('tr')
    for tr in data:
        ltd = tr.find_all('td')
        if len(ltd)==0:
            continue
        singleUniv = []
        for td in ltd:
            singleUniv.append(td.string)
        allUniv.append(singleUniv)
def writercsv(save_road,num,title):
    if os.path.isfile(save_road):
        with open(save_road,'a',newline='')as f:
            csv_write=csv.writer(f,dialect='excel')
            for i in range(num):
                u=allUniv[i]
                csv_write.writerow(u)
    else:
         with open(save_road,'w',newline='')as f:
            csv_write=csv.writer(f,dialect='excel')
            csv_write.writerow(title)
            for i in range(num):
                u=allUniv[i]
                csv_write.writerow(u)
 
title=["排名","學校名稱","省市","總分","生源質量","培養結果","科研規模","科研質量","頂尖成果","頂尖人才","科技服務","產學研究合作","成果轉化"]
save_road="C:\\Users\\Benny\\Desktop\\Python\\Python練習\sqlit_test02.csv"
def main():
    url = 'http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html'
    html = getHTMLText(url)
    soup = BeautifulSoup(html, "html.parser")
    fillUnivList(soup)
    writercsv(save_road,10,title)
main()

文件截屏如下：（這里只是保存了前十名的數據，可以通過更改num來保存更多）