代码：
主要是爬取行情中心的不同板块的股票数据：
import requests import json import re import pandas as pd # 存储相关信息
def getMessage(getCount): if getCount == 1: stockPlateDict = { '沪深京A股': 'hs_a_board', '上证A股': 'sh_a_board', '深证A股': 'sz_a_board', '北证A股': 'bj_a_board', '新股': 'newshares', '创业板': 'gem_board', '科创板': 'kcb_board', '沪股通': 'sh_hk_board', '深股通': 'sz_hk_board', 'B股': 'b_board', '上证AB股比价': 'ab_comparison_sh', '深证AB股比价': 'ab_comparison_sz', '风险警示板': 'st_board', '两网及退市': 'staq_net_board' } return stockPlateDict elif getCount == 2: webUrl = "https://23.push2.eastmoney.com/api/qt/clist/get?"
return webUrl elif getCount == 3: urlSuf = [ # 每个板块的网址后缀
"cb=jQuery1124011309990273440462_1650110291355&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23,m:0+t:81+s:2048&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1650110291377", "cb=jQuery1124011309990273440462_1650110291355&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:1+t:2,m:1+t:23&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1650110291388", "cb=jQuery1124011309990273440462_1650110291355&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:80&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1650110291392", "cb=jQuery1124011309990273440462_1650110291355&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:81+s:2048&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1650110291396", "cb=jQuery1124011309990273440462_1650110291355&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=m:0+f:8,m:1+f:8&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1650110291400", "cb=jQuery1124011309990273440462_1650110291355&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:80&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1650110291405", "cb=jQuery1124011309990273440462_1650110291355&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:1+t:23&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1650110291408", "cb=jQuery1124011309990273440462_1650110291351&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=b:BK0707&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1650110291412", "cb=jQuery1124011309990273440462_1650110291351&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=b:BK0804&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1650110291416", "cb=jQuery1124011309990273440462_1650110291351&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:7,m:1+t:3&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1650110291420", "cb=jQuery1124011309990273440462_1650110291351&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f199&fs=m:1+b:BK0498&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152,f201,f202,f203,f196,f197,f199,f195,f200&_=1650110291424", "cb=jQuery1124011309990273440462_1650110291351&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f199&fs=m:0+b:BK0498&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152,f201,f202,f203,f196,f197,f199,f195,f200&_=1650110291427", "cb=jQuery1124031292793882810255_1650114519858&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+f:4,m:1+f:4&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1650114519867", "cb=jQuery1124031292793882810255_1650114519858&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+s:3&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f33,f11,f62,f128,f136,f115,f152&_=1650114519872" ] return urlSuf elif getCount == 4: stockMessage = { 'f12': '股票代码', 'f14': '名称', 'f2': '最高', 'f3': '涨跌幅%', 'f4': '涨跌额', 'f5': '成交量(手)', 'f6': '成交额', 'f7': '振幅%', 'f8': '换手率%', 'f9': '市盈率(动态)', 'f10': '量比', 'f15': '最新价', 'f16': '最低价', 'f17': '今开', 'f18': '昨收', 'f23': '市净率', } return stockMessage # 得到板块名称
def getPlateName(): stockPlateDict = getMessage(1) plateName = [] # 板块名称
for i in stockPlateDict: plateName.append(i) return plateName # 得到板块uel列表
def getPlateUrlList(inputPlateName, pageCount): webUrl = getMessage(2) # 网站首地址
op = getSplitURL() # 分割后的网址
plateName = getPlateName() # 板块名称
pagesStart = [] # 存储分割后页码之前的后缀
pagesEnd = [] # 存储分割后页码之后的后缀
plateURL = [] # 板块URL
for i in range(0, len(op)): # 组装每一个板块
pagesStart.append(webUrl + op[i][0][0]) # 分割后页码之前的后缀
pagesEnd.append(op[i][0][1]) # 分割后页码之后的后缀
for i in range(0, len(plateName)): if plateName[i] == inputPlateName: for j in range(1, pageCount + 1): # 添加第i个板块的第j页
plateURL.append(pagesStart[i] + str(j) + pagesEnd[i]) return plateURL # 返回多页数的URL列表
# 处理url（不加页码的）
def getSplitURL(): urlSuf = getMessage(3) # 得到网址后缀
suf = [i.split("fields")[0] for i in urlSuf] # 分割掉无用的后缀
pattern = re.compile("(.*pn=)1(&.*)") # 分割页码
op = [re.findall(pattern, i) for i in suf] # 正则匹配分割页码
return op # 请求网页得到json数据
def requestURL(plateName, requestCount): # 请求已经处理好的url
urls = getPlateUrlList(plateName, requestCount) # 得到指定板块指定页数的url列表
resTxT = [] # 存储请求到的每一页数据
for i in range(0, requestCount): # 请求第i页url
res = requests.get(urls[i]) # text文本
doc = res.text # 正则表达式
pat = re.compile("({\"f1\".*?})") # 正则匹配
resTxT.append(re.findall(pat, doc)) return resTxT # 程序开始的选择项
def getChoose(choose): plateNmae = getPlateName() # 板块名称列表
for i in range(0, len(plateNmae)): if choose == i + 1: # 对应的选择返回对应的板块名称
return plateNmae[i] # 处理f信息
def getStockMsg(chPlateNmae, pageCount): text = requestURL(chPlateNmae, pageCount) # 请求到的f信息
dicMessage = [] # 二维列表，第一层的个数表示页码数，第二层是存储的对应页码的数据
for i in range(0, len(text)): # 遍历第i页
dicMessage.append([]) for j in range(0, len(text[i])): # 第i页的第j个元素
dicMessage[i].append(json.loads(text[i][j])) # 转换为字典
return dicMessage def getStockFF(): stockMessage = getMessage(4) # 拿到f信息字典
stockFF = [i for i in stockMessage] # f信息
return stockFF # 获取股票的信息
def getStockValue(dicMsg): # dicMsg # 二维列表，存储股票信息
stockFF = getStockFF() # 获取对应的f信息
stockVal = [] # 存储f信息对应的值
for i in range(0, len(dicMsg)): # 访问第i页
stockVal.append([]) for j in range(0, len(dicMsg[i])): # 访问第i页的第j个元素
stockVal[i].append([]) for k in stockFF: # 保存对应的f信息的值
stockVal[i][j].append(dicMsg[i][j].get(k)) return stockVal # 创建表格
def makeDataDrame(stockValueList): stockFF = getStockFF() # 得到f信息
stockMsg = getMessage(4) # 拿到股票title
columns = [] # 存储f信息对应的值
df = pd.DataFrame() for i in stockFF: columns.append(stockMsg.get(i)) # 添加值
stockList = [] for i in range(0, len(stockValueList)): # 遍历每一页
for j in range(0, len(stockValueList[i])): stockList.append(stockValueList[i][j]) # 使用一个新列表将三维转为二维
index = [i for i in range(1, len(stockList) + 1)] # 行数
df = pd.DataFrame(stockList, columns=columns, index=index) # 创建表格
return df # 保存
def toSave(df): df.to_excel('D:\df.xlsx') return True if __name__ == '__main__': plateNmae = getPlateName() print("----------------- 请选择板块 ---------------------") for i in range(0, len(plateNmae)): print("" + str(i + 1) + "、" + plateNmae[i]) print("----------------- 请选择以上板块-------------------------------") chPlateNmae = getChoose(int(input("请输入您的选择："))) pageCount = int(input("请输入您要爬取的页数：")) dicMsg = getStockMsg(chPlateNmae, pageCount) stockValueList = getStockValue(dicMsg) df = makeDataDrame(stockValueList) isOk = toSave(df) print(isOk)
保存结果：
