一個爬取股票信息的爬蟲程序

　　給大家分享一個早前爬取東方財富網股票信息的爬蟲程序，回頭來看做了好多改進，特別是數據處理部分使用了heapd模塊，方便快捷一步到位...

# _*_ coding:utf-8 _*_

import requests,re,json,time,os
import heapq
from bs4 import BeautifulSoup

class GPINFO(object):
    """docstring for GPINFO"""
    def __init__(self):
        self.Url = 'http://quote.eastmoney.com/stocklist.html'
        self.BaseData = []
        self.Date = time.strftime('%Y%m%d')
        self.Record = 'basedata'+self.Date
        if os.path.exists(self.Record):
            print ('record exist...')
            self.BaseData = self.get_base_data_from_record()
        else:
            print ('fuck-get data again...')
            self.get_data()

    def write_record(self,text):
        with open(self.Record,'ab') as f:
            f.write((text+'\n').encode('utf-8'))

    def get_base_data_from_record(self):
        ll = []
        with open(self.Record,'rb') as f:
            json_l = f.readlines()
            for j in json_l:
                ll.append(json.loads(j.decode('utf-8')))
        return ll

    def get_data(self):
        #請求數據
        orihtml = requests.get(self.Url).content
        #創建 beautifulsoup 對象
        soup = BeautifulSoup(orihtml,'lxml')
        #采集每一個股票的信息
        count = 0
        for a in soup.find('div',class_='quotebody').find_all('a',{'target':'_blank'}):
            record_d = {}
            #代號
            num = a.get_text().split('(')[1].strip(')')
            if not (num.startswith('00') or num.startswith('60')):continue #只需要6*/0*
            record_d['num']=num
            #名稱
            name = a.get_text().split('(')[0]
            record_d['name']=name
            #詳情頁
            detail_url = a['href']
            record_d['detail_url']=detail_url

            cwzburl = detail_url
            #發送請求
            try:
                cwzbhtml = requests.get(cwzburl,timeout=30).content
            except Exception as e:
                print ('perhaps timeout:',e)
                continue
            #創建soup對象
            cwzbsoup = BeautifulSoup(cwzbhtml,'lxml')

            #財務指標列表 [浦發銀行，總市值    凈資產    凈利潤    市盈率    市凈率    毛利率    凈利率    ROE] roe:凈資產收益率
            try:
                cwzb_list = cwzbsoup.find('div',class_='cwzb').tbody.tr.get_text().split()
            except Exception as e:
                print ('error:',e)
                continue
            #去除退市股票
            if '-' not in cwzb_list:
                record_d['data']=cwzb_list
                self.BaseData.append(record_d)
                self.write_record(json.dumps(record_d))
                count=count+1
                print (len(self.BaseData))

def main():
    test = GPINFO()
    result = test.BaseData
    #[浦發銀行，總市值    凈資產    凈利潤    市盈率    市凈率    毛利率    凈利率    ROE] roe:凈資產收益率]
    top_10 = heapq.nlargest(10,result,key=lambda r:float(r['data'][7].strip('%')))
    for i in top_10:
        print(i['data'])

if __name__ == '__main__':
    main()

 

　　程序主函數部分是為了獲取凈利率前10名的股票信息,打印結果如下:

['綿石投資', '52.2億', '14.0億', '1.25億', '30.90', '3.73', '42.25%', '2047.04%', '9.27%']
['國投安信', '556億', '270億', '21.1億', '19.80', '2.12', '5.90%', '487.53%', '7.79%']
['川投能源', '379億', '202億', '28.0億', '10.16', '1.91', '37.01%', '402.64%', '14.58%']
['ST明科', '47.6億', '9.25億', '5.11千萬', '68.00', '5.14', '2.38%', '345.11%', '5.68%']
['華聯控股', '93.6億', '31.5億', '4.76億', '14.54', '3.74', '46.25%', '328.53%', '20.88%']
['上海九百', '68.2億', '12.3億', '1.61億', '31.67', '5.56', '54.00%', '297.99%', '13.21%']
['凱瑞德', '46.7億', '1.14億', '3.27千萬', '107.10', '40.94', '16.07%', '294.19%', '33.41%']
['魯信創投', '172億', '38.6億', '3.32億', '38.48', '4.64', '28.67%', '244.43%', '9.26%']
['博聞科技', '35.0億', '6.56億', '2.23千萬', '117.65', '5.36', '-16.07%', '215.27%', '3.41%']
['萬澤股份', '71.8億', '13.7億', '6.87千萬', '78.38', '5.29', '22.57%', '203.15%', '5.13%']