課程亮點
- 爬蟲的基本流程
- re正則表達式模塊的簡單使用
- requests模塊的使用
- 保存csv
- python 3.8 >>> 安裝包找木子老師領取
- pycharm 2021專業版 需要激活可以找木子老師領取
- pycharm 社區版 (免費) 沒有主題
專業版 (需要激活碼)
- requests >>> pip install requests (數據請求模塊)
- re
- json
- csv
- time
- 就要去分析 數據是從哪里來得
通過開發者工具進行分析
(找數據內容) - 發送請求 對於目標網址發送請求
- 獲取數據內容 網頁源代碼 response.text
import requests # 數據請求模塊 pip install requests import re # 正則表達式模塊 import json # 序列化與反序列化 import pprint # 格式化輸出模塊 import csv import time # 時間模塊
url = f'https://search.51job.com/list/010000%252C020000%252C030200%252C040000%252C180200,000000,0000,00,9,99,python,2,1.html' headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36' } # 發送請求 requests調用里面get請求方法 然后把 url 以及 headers 傳進去 response = requests.get(url=url, headers=headers) # <Response [200]> 返回響應對象 response 200狀態碼 表示請求成功
# re正則表達式 re正則表達式 可以直接提取字符串數據 # 使用re模塊里面findall()方法 html_data = re.findall('window.__SEARCH_RESULT__ = (.*?)</script>', response.text)[0] # 正則匹配出來的數據 是列表 [] json_data = json.loads(html_data)['engine_jds'] print(json_data) for index in json_data: # 等號左邊都是自定義變量 # pprint.pprint(index) title = index['job_title'] # 職位名字 company_name = index['company_name'] # 公司名字 money = index['providesalary_text'] # 薪資 job_welf = index['jobwelf'] # 福利 # job_info = index['attribute_text'] #基本信息 area = index['attribute_text'][0] # 城市 exp = index['attribute_text'][1] # 經驗 edu = index['attribute_text'][2] # 學歷 company_type = index['companyind_text'] # 公司類型 date = index['updatedate'] # 發布日期 href = index['job_href'] # 招聘詳情頁 # 復制一行 ctrl + D dit = { '職位名字': title, '公司名字': company_name, '薪資': money, '公司福利': job_welf, '地區': area, '經驗': exp, '學歷': edu, '公司類型': company_type, '發布日期': date, '詳情頁': href, } print(title, company_name, money, area, exp, edu, job_welf, company_type, date, href) csv_writer.writerow(dit)
f = open('數據.csv', mode='a', encoding='utf-8', newline='') csv_writer = csv.DictWriter(f, fieldnames=[ '職位名字', '公司名字', '薪資', '公司福利', '地區', '經驗', '學歷', '公司類型', '發布日期', '詳情頁', ]) csv_writer.writeheader() # 寫入表頭
import pandas as pd from pyecharts.charts import * from pyecharts import options as opts import matplotlib.pyplot as plt plt.rcParams['font.sans-serif']=['SimHei'] plt.rcParams['axes.unicode_minus']=False
boss = pd.read_csv('data.csv', engine='python', encoding='utf-8') boss
import numpy as np def shulie(first,end,step): x = [] for i in np.arange(first, end,step): x.append(i) return x list_1 = shulie(0,10,0.5) boss['top'].plot.hist(bins=30,figsize=(20,8),edgecolor="black") plt.xticks(list_1) plt.xlabel('萬/月') plt.ylabel('數量') plt.show()
c = ( Pie(init_opts=opts.InitOpts(width="1000px", height="600px", bg_color="#2c343c")) .add( series_name="經驗需求占比", data_pair=data_pair_1, rosetype="radius", radius="55%", center=["25%", "50%"], label_opts=opts.LabelOpts(is_show=False, position="center", color="rgba(255, 255, 255, 0.3)"), ) .add( series_name="學歷需求占比", data_pair=data_pair_2, radius="55%", center=["75%", "50%"], label_opts=opts.LabelOpts(is_show=False, position="center", color="rgba(255, 255, 255, 0.3)"), ) .set_series_opts( tooltip_opts=opts.TooltipOpts( trigger="item", formatter="{a} <br/>{b}: {c} ({d}%)" ), label_opts=opts.LabelOpts(color="rgba(255, 255, 255, 0.3)"), ) .set_global_opts( title_opts=opts.TitleOpts( title="經驗、學歷需求占比", pos_left="center", pos_top="20", title_textstyle_opts=opts.TextStyleOpts(color="#fff"), ), legend_opts=opts.LegendOpts(is_show=False), ) .set_colors(["#D53A35", "#334B5C", "#61A0A8", "#D48265", "#749F83"]) ) c.render_notebook()
from pyecharts.globals import SymbolType address_count = boss.groupby('地區').count()['公司名字'].sort_values() x = address_count.index.tolist() y = address_count.values.tolist() c = ( PictorialBar() .add_xaxis(x) .add_yaxis( "", y, label_opts=opts.LabelOpts(is_show=False), symbol_size=18, symbol_repeat="fixed", symbol_offset=[0, 0], is_symbol_clip=True, symbol=SymbolType.ROUND_RECT, ) .reversal_axis() .set_global_opts( title_opts=opts.TitleOpts(title="地區人員招聘數量"), xaxis_opts=opts.AxisOpts(is_show=False), yaxis_opts=opts.AxisOpts( axistick_opts=opts.AxisTickOpts(is_show=False), axisline_opts=opts.AxisLineOpts( linestyle_opts=opts.LineStyleOpts(opacity=0) ), ), ) ) c.render_notebook()
mean = boss.groupby('經驗')['工資平均'].mean().sort_values() x = mean.index.tolist() y = mean.values.tolist() c = ( Bar() .add_xaxis(x) .add_yaxis( "工作經驗", y, markpoint_opts=opts.MarkPointOpts( data=[opts.MarkPointItem(name="無需經驗", coord=[x[3], y[3]], value=y[3])] ) ) .set_global_opts(title_opts=opts.TitleOpts(title="不同工作經驗的平均薪資")) .set_series_opts(label_opts=opts.LabelOpts(is_show=False)) ) c.render_notebook()
mean = boss.groupby('學歷')['工資平均'].mean().sort_values() x = mean.index.tolist() y = mean.values.tolist() c = ( Bar() .add_xaxis(x) .add_yaxis( "學歷", y, markpoint_opts=opts.MarkPointOpts( data=[opts.MarkPointItem(name="學歷不限", coord=[x[1], y[1]], value=y[1])] ) ) .set_global_opts(title_opts=opts.TitleOpts(title="不同學歷的平均薪資")) .set_series_opts(label_opts=opts.LabelOpts(is_show=False)) ) c.render_notebook()
import jieba words = jieba.lcut(text) #通過遍歷words的方式,統計出每個詞出現的頻次 counts = {} for word in words: if len(word) == 1: continue else: counts[word] = counts.get(word,0) + 1 c = ( WordCloud() .add(series_name="熱點分析", data_pair=new, word_size_range=[6, 66]) .set_global_opts( title_opts=opts.TitleOpts( title="公司福利", title_textstyle_opts=opts.TextStyleOpts(font_size=23) ), tooltip_opts=opts.TooltipOpts(is_show=True), ) ) c.render_notebook()
