python3實踐-從網站獲取數據(Carbon Market Data-BJ) （pandas，bs4）

 

自己邊看邊實踐一些簡單的實際應用，下面的程序是從某個網站上獲取需要的數據。

在編寫的過程中，通過學習陸續了解到一些方法，發現Python真的是很便捷。

尤其是用pandas獲取網頁中的表格數據，真的是太方便了！！！

程序寫的可能並不好，但基本上實現了自己的需求。

希望有高手來指點下~~

 

Version 04 （Jan 12 2017）【對於獲取表格信息，推薦使用該方法】

# Code based on Python 3.x
# _*_ coding: utf-8 _*_
# __Author: "LEMON"

import pandas as pd

url2 = 'http://www.bjets.com.cn/article/jyxx/?'
links = []
for n in range(2, 40):
    # 頁面總數為39頁，需要自己先從網頁判斷，也可以從頁面抓取，后續可以完善
    link = url2 + str(n)
    links.append(link)
links.insert(0, url2)

df2 = pd.DataFrame()  # creates a new dataframe that's empty
for url in links:
    # 利用pandas獲取數據，需要安裝 html5lib模塊
    dfs = pd.read_html(url, header=0)
    for df in dfs:
        df2= df2.append(df, ignore_index= True)

# df2.to_excel('MktDataBJ.xlsx') # 將數據存儲在excel文件里
df2.to_csv('MktDataBJ-1.csv')  # 將數據存儲在csv文件里

 

Version 03 （Jan 12 2017）

# Code based on Python 3.x
# _*_ coding: utf-8 _*_
# __Author: "LEMON"

from bs4 import BeautifulSoup
import requests
import csv

url2 = 'http://www.bjets.com.cn/article/jyxx/?'
links = []
for n in range(2, 40):
    # 頁面總數為39頁，需要自己先從網頁判斷，也可以從頁面抓取，后續可以完善
    link = url2 + str(n)
    links.append(link)
links.insert(0, url2)

for url in links:
    rep = requests.get(url)
    # content = rep.text.encode(rep.encoding).decode('utf-8')
    # # 直接用requests時，中文內容需要轉碼

    soup = BeautifulSoup(rep.content, 'html.parser')

    # table = soup.table
    table = soup.find('table')  # 兩種方式都可以

    trs = table.find_all('tr')
    trs2 = trs[1:len(trs)]
    list1 = []
    for tr in trs2:
        td = tr.find_all('td')
        row = [i.text for i in td]
        list1.append(row)

    with open('MktDataBJ.csv', 'a', errors='ignore', newline='') as f:
        f_csv = csv.writer(f)
        f_csv.writerows(list1)

 

Version 02 （Jan 09 2017）

# Code based on Python 3.x
# _*_ coding: utf-8 _*_
# __Author: "LEMON"

from bs4 import BeautifulSoup
import requests
import csv

url2 = 'http://www.bjets.com.cn/article/jyxx/?'
links = []
for n in range(2, 40):
    # 頁面總數為39頁，需要自己先從網頁判斷，也可以從頁面抓取，后續可以完善
    link = url2 + str(n)
    links.append(link)
links.insert(0, url2)
# print(links)

for url in links:
    rep = requests.get(url)
    # content = rep.text.encode(rep.encoding).decode('utf-8')
    # # 直接用requests時，中文內容需要轉碼

    soup = BeautifulSoup(rep.content, 'html.parser')
    body = soup.body
    data = body.find('div', {'class': 'list_right'})

    quotes = data.find_all('tr')
    quotes1 = quotes[1:len(quotes)]

    list1 = []
    for x in quotes1:
        list2 = []
        for y in x.find_all('td'):
            list2.append(y.text)  # 每日的數據做一個單獨的list
        list1.append(list2)
    # print(list1)  # list1為每日數據的總列表
    with open('MktDataBJ.csv', 'a', errors='ignore', newline='') as f:
        f_csv = csv.writer(f)
        f_csv.writerows(list1)

 

Version 01 （Jan 08 2017）

# Code based on Python 3.x
# _*_ coding: utf-8 _*_
# __Author: "LEMON"

from bs4 import BeautifulSoup
import requests
import csv

urllink = 'http://www.bjets.com.cn/article/jyxx/?'
links = []
for n in range(2, 40):
    #頁面總數為39頁，需要自己先從網頁判斷，也可以從頁面抓取，后續可以完善
    link = urllink + str(n)
    links.append(link)
links.insert(0, urllink)
# print(links)

for url in links:

    rep = requests.get(url)
    # content = rep.text.encode(rep.encoding).decode('utf-8')
    # # 直接用requests時，中文內容需要轉碼

    soup = BeautifulSoup(rep.content, 'html.parser')

    # print(soup.prettify())
    # # prettify()

    body = soup.body
    data = body.find('div', {'class': 'list_right'})

    # table title
    titles = data.find_all('th')

    title = []
    for x in titles:
        title.append(x.text)
    # print(title)

    quotes = data.find_all('tr')
    quotes1 = quotes[1:len(quotes)]
    # print(quotes1)

    list1 = []
    for x in quotes1:
        for y in x.find_all('td'):
            list1.append(y.text)
    # print(list1)  # list為每日數據的總列表

    date = []
    volumes = []
    meanprice = []
    totalmoney = []

    for i in range(0, len(list1)):
        if i % 4 == 0:
            date.append(list1[i])
        elif i % 4 == 1:
            volumes.append(list1[i])
        elif i % 4 == 2:
            meanprice.append(list1[i])
        else:
            totalmoney.append(list1[i])

    # print(date)
    # print(volumes)
    # print(meanprice)
    # print(totalmoney)

    final = []
    for i in range(0, len(date)):
        temp = [date[i], volumes[i], meanprice[i], totalmoney[i]]
        final.append(temp)
    # print(final)
    with open('bj_carbon.csv', 'a', errors='ignore', newline='') as f:
        f_csv = csv.writer(f)
        f_csv.writerows(final)