同花順概念板塊成分股數據分享

爬取了同花順上概念板塊的成分股

主要是發現了各個量化的網站沒有這個數據源，很多策略無法展開。這份數據應該挺多人需要的吧。把鏈接掛這里了，需要的闊以下載

http://pan.baidu.com/s/1eSGSS5W

數據有4列，分別是板塊代碼，板塊名字，成分股代碼以及對應的公司

源碼貼出來了，初步學習爬蟲，寫的十分難看。。。。

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Nov 17 19:41:44 2017

@author: Mr.ZeroW

同花順板塊成分股
"""

#首先不同板塊地址不同 http://q.10jqka.com.cn/gn/detail/order/desc/page/1/ajax/1/code/300018
#需要高出不同板塊頁數有多少，才能得出地址
import urllib.request
from lxml import etree
import pandas as pd
import time

#爬取板塊名稱以及代碼並且存在文件
with urllib.request.urlopen('http://q.10jqka.com.cn/gn/') as f:
    text = f.read().decode('gb2312')

html = etree.HTML(text)

gnbk = html.xpath('/html/body/div[2]/div[1]/div//div//div//a')
thsgnbk = []
for i in range(len(gnbk)):
    thsgnbk.append((gnbk[i].text))
    
#板塊代碼
bkcode = html.xpath('/html/body/div[2]/div[1]/div//div//div//a/@href')
bkcode = list(map(lambda x : x.split('/')[-2], bkcode))
data = {'Name': thsgnbk}

#存儲
gnbk = pd.DataFrame(data, index = bkcode)
gnbk.to_csv('gnbk.csv')

print('板塊名稱以及代碼已爬取，存儲文件名：gnbk.csv')
#導入板塊名稱和代碼
data = pd.read_csv('gnbk.csv')
#建立數據框，四列【板塊id, 板塊name, 成分股id, 成分股name】

bk_id = []
bk_name = []
s_id = []
s_name = []
iCount = 1
print('爬取開始！')
start = time.time()
for i in range(len(data)):

    bk_code = str(data.iloc[i, 0])
    name = str(data.iloc[i, 1])
    url = 'http://q.10jqka.com.cn/gn/detail/code/' + bk_code + '/'
    print('%d: %s' %(iCount, name))
    iCount += 1
    
    with urllib.request.urlopen(url) as f:
        text = f.read().decode('GBK', 'ignore')

    #得出板塊成分股有多少頁
    html = etree.HTML(text)
    
    result = html.xpath('//*[@id="m-page"]/span/text()')
    try:
        page = int(result[0].split('/')[-1])
        for j in range(page):
            page_n = str(j + 1)
            curl = 'http://q.10jqka.com.cn/gn/detail/order/desc/page/' + page_n+ '/ajax/1/code/' + bk_code
            with urllib.request.urlopen(curl) as f:
                text = f.read().decode('GBK')
            html = etree.HTML(text)
            #成分股代碼
            stock_code = html.xpath('/html/body/table/tbody/tr/td[2]/a/text()')
            #成分股名稱
            stock_name = html.xpath('/html/body/table/tbody/tr/td[3]/a/text()')
            s_id += stock_code
            s_name += stock_name
            bk_id.extend([bk_code]* len(stock_code))
            bk_name.extend([name]* len(stock_name))
            
    except IndexError as e:
        curl = url
        with urllib.request.urlopen(curl) as f:
            text = f.read().decode('GBK')
        html = etree.HTML(text)
        #成分股代碼
        stock_code = html.xpath('//*[@id="maincont"]/table/tbody/tr/td[2]/a/text()')
        #成分股名稱
        stock_name = html.xpath('//*[@id="maincont"]/table/tbody/tr/td[3]/a/text()')   
        s_id += stock_code
        s_name += stock_name
        bk_id.extend([bk_code]* len(stock_code))
        bk_name.extend([name]* len(stock_name))
        
    
data_dict = dict(BK_ID = bk_id, BK_NAME = bk_name, S_ID = s_id, S_NAME = s_name)
cdata = pd.DataFrame(data_dict)
cdata.to_csv('chengfengu.csv')
end = time.time()
print('爬取結束！！\n開始時間：%s\n結束時間：%s\n'%(time.ctime(end), time.ctime(start)))