Python爬取前程無憂職位信息

一、選題背景

剛畢業往往會為自己不知道每個職位之間各種待遇的差異而迷茫,所以為了了解畢業后職位的待遇等方面做多種參考，貨比三家。

 

1.數據來源

前程無憂（https://www.51job.com/）

2.爬取內容

爬取內容包括職位名稱，公司名稱，地點，薪資，學歷要求，以及發布日期等。

 

二、實現爬取的步驟

1.代碼所需包

import urllib.request
import xlwt
import re
import urllib.parse
import time

 

 

2.進入前程無憂官網，搜索職位信息

 

3.打開開發者模式

 

4.模擬瀏覽器

header={
    'Host':'search.51job.com',
    'Upgrade-Insecure-Requests':'1',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}

 

 

5.為了實現爬取，我寫了一個能夠實現輸入想了解的職位就能爬取相關內容的函數

#page是頁數，item是輸入的字符串
def getfront(page,item):
#先把字符串轉成十六進制編碼      
     result = urllib.parse.quote(item)                    
     ur1 = result+',2,'+ str(page)+'.html'
     ur2 = 'https://search.51job.com/list/000000,000000,0000,00,9,99,'
     res = ur2+ur1                                                            #拼接網址
     a = urllib.request.urlopen(res)
# 讀取源代碼並轉為unicode
     html = a.read().decode('gbk')          
     return html

def getInformation(html):
    #匹配換行符
    reg = re.compile(r'class="t1 ">.*? <a target="_blank" title="(.*?)" href="(.*?)".*? <span class="t2"><a target="_blank" title="(.*?)" href="(.*?)".*?<span class="t3">(.*?)</span>.*?<span class="t4">(.*?)</span>.*?<span class="t5">(.*?)</span>.*?',re.S)
    items=re.findall(reg,html)
    return items

除了爬取基本信息外，還把職位超鏈接后的網址，以及公司超鏈接的網址爬取下來了。

 

6.把爬取的信息以Excel文件形式儲存起來，比較清晰直觀。

#新建表格空間
excel1 = xlwt.Workbook()
# 設置單元格格式
sheet1 = excel1.add_sheet('Job', cell_overwrite_ok=True)

sheet1.write(0, 0, '序號')

sheet1.write(0, 1, '職位')

sheet1.write(0, 2, '公司名稱')

sheet1.write(0, 3, '公司地點')

sheet1.write(0, 4, '公司性質')

sheet1.write(0, 5, '薪資')

sheet1.write(0, 6, '學歷要求')

sheet1.write(0, 7, '工作經驗')

sheet1.write(0, 8, '公司規模')

sheet1.write(0, 9, '公司類型')

sheet1.write(0, 10,'公司福利')

sheet1.write(0, 11,'發布時間')

 

爬取代碼如下

number = 1
item = input()

for j in range(1,1000):
    try:
        print("正在爬取第"+str(j)+"頁數據...")
#調用獲取網頁原碼
        html = getfront(j,item)      

        for i in getInformation(html):
            try:
#職位網址
                url1 = i[1]          
                res1 = urllib.request.urlopen(url1).read().decode('gbk')
                company = re.findall(re.compile(r'<div class="com_tag">.*?<p class="at" title="(.*?)"><span class="i_flag">.*?<p class="at" title="(.*?)">.*?<p class="at" title="(.*?)">.*?',re.S),res1)

                job_need = re.findall(re.compile(r'<p class="msg ltype".*?>.*?&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;(.*?)&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;(.*?)&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;.*?</p>',re.S),res1)

                welfare = re.findall(re.compile(r'<span class="sp4">(.*?)</span>',re.S),res1)
                print(i[0],i[2],i[4],i[5],company[0][0],job_need[2]
[0],job_need[1][0],company[0][1],company[0][2],welfare,i[6])
                sheet1.write(number,0,number)

                sheet1.write(number,1,i[0])

                sheet1.write(number,2,i[2])

                sheet1.write(number,3,i[4])

                sheet1.write(number,4,company[0][0])

                sheet1.write(number,5,i[5])

                sheet1.write(number,6,job_need[1][0])

                sheet1.write(number,7,job_need[2][0])

                sheet1.write(number,8,company[0][1])

                sheet1.write(number,9,company[0][2])

                sheet1.write(number,10,("  ".join(str(i) for i in welfare)))

                sheet1.write(number,11,i[6])

                number+=1
                excel1.save("51job.xls")
#休息間隔，避免爬取海量數據時被誤判為攻擊，IP遭到封禁
                time.sleep(0.3) 
            except:
                pass
    except:
        pass

 

結果如下：

 

三、數據清洗與處理

1.先打開文件

#coding:utf-8
import pandas as pd
import re

#除此之外還要安裝xlrd包

data = pd.read_excel(r'51job.xls',sheet_name='Job')
result = pd.DataFrame(data)

 

清洗思路：

1、出現有空值的信息，直接刪除整行

a = result.dropna(axis=0,how='any')
#輸出全部行，不省略
pd.set_option('display.max_rows',None) 

 

2.職位出錯（爬取職位與預想職位無關）

1 b = u'數據'
2 number = 1
3 li = a['職位']
4 for i in range(0,len(li)):
5     try:
6         if b in li[i]:
7             #print(number,li[i])
8             number+=1
9         else:
10             a = a.drop(i,axis=0)
11     except:
12         pass

 

3.其他地方出現的信息錯位，比如在學歷里出現 ‘招多少人’

b2= u'人'
li2 = a['學歷要求']
for i in range(0,len(li2)):
    try:
        if b2 in li2[i]:
            #print(number,li2[i])
            number+=1
            a = a.drop(i,axis=0)
    except:
        pass

 

 

4.轉換薪資單位不一致

b3 =u'萬/年'
b4 =u'千/月'
li3 = a['薪資']

#注釋部分的print都是為了調試用的

for i in range(0,len(li3)):
    try:
        if b3 in li3[i]:
            x = re.findall(r'\d*\.?\d+',li3[i])
            #print(x)

#轉換成浮點型並保留兩位小數
            min_ = format(float(x[0])/12,'.2f')              
            max_ = format(float(x[1])/12,'.2f')
            li3[i][1] = min_+'-'+max_+u'萬/月'
        if b4 in li3[i]:
            x = re.findall(r'\d*\.?\d+',li3[i])
            #print(x)

            #input()
            min_ = format(float(x[0])/10,'.2f')
            max_ = format(float(x[1])/10,'.2f')
            li3[i][1] = str(min_+'-'+max_+'萬/月')
        print(i,li3[i])

    except:
        pass

 

清洗完成后保存到新的Excel文件里。

a.to_excel('51job2.xlsx', sheet_name='Job', index=False)

 

四、數據可視化

經過可視化處理能使數據更加直觀，更有利於分析 甚至可以說可視化是數據挖掘最重要的內容。

1.查看需要的包

# -*- coding: utf-8 -*-
import pandas as pd
import re
from pyecharts import Funnel,Pie,Geo
import matplotlib.pyplot as plt

 

2.打開文件

file = pd.read_excel(r'51job2.xls',sheet_name='Job')
f = pd.DataFrame(file)
pd.set_option('display.max_rows',None)

3.創建多個列表來單獨存放薪資，工作經驗，學歷要求，公司地點等信息

add = f['公司地點']
sly = f['薪資']
edu = f['學歷要求']
exp = f['工作經驗']
address =[]
salary = []
education = []
experience = []
for i in range(0,len(f)):
    try:
        a = add[i].split('-')
        address.append(a[0])
        #print(address[i])
        s = re.findall(r'\d*\.?\d+',sly[i])
        s1= float(s[0])
        s2 =float(s[1])
        salary.append([s1,s2])
        #print(salary[i])
        education.append(edu[i])
        #print(education[i])
        experience.append(exp[i])
        #print(experience[i])
    except:
       pass

4.工作經驗—薪資圖 與 學歷—薪資圖

#定義存放最低薪資的列表
min_s=[]
#定義存放最高薪資的列表
max_s=[]
for i in range(0,len(experience)):
    min_s.append(salary[i][0])
    max_s.append(salary[i][0])

my_df = pd.DataFrame({'experience':experience, 'min_salay' : min_s, 
#關聯工作經驗與薪資
'max_salay' : max_s})
data1 = my_df.groupby('experience').mean()['min_salay'].plot(kind='line')
plt.show()

my_df2 = pd.DataFrame({'education':education, 'min_salay' : min_s, 
#關聯學歷與薪資
'max_salay' : max_s})
data2 = my_df2.groupby('education').mean()['min_salay'].plot(kind='line')
plt.show()

5.學歷要求圓環圖

def get_edu(list):
    education2 = {}
    for i in set(list):
        education2[i] = list.count(i)
    return education2
dir1 = get_edu(education)

# print(dir1)

attr= dir1.keys()
value = dir1.values()
pie = Pie("學歷要求")
pie.add("", attr, value, center=[50, 50], is_random=False, radius=[30, 75], rosetype='radius',
        is_legend_show=False, is_label_show=True,legend_orient='vertical')
pie.render('學歷要求玫瑰圖.html')

 

6.大數據城市需求地理位置分布圖

def get_address(list):
    address2 = {}
    for i in set(list):
        address2[i] = list.count(i)

    address2.pop('異地招聘')

    #address2.pop('山東')
    #address2.pop('怒江')
    #address2.pop('池州')

    return address2

dir2 = get_address(address)

#print(dir2)

geo = Geo("大數據人才需求分布圖", title_color="#2E2E2E",
          title_text_size=24,title_top=20,title_pos="center", width=1300,height=600)

attr2 = dir2.keys()
value2 = dir2.values()

geo.add("",attr2, value2, type="effectScatter", is_random=True, visual_range=[0, 1000], maptype='china',symbol_size=8, effect_scale=5, is_visualmap=True)

geo.render('大數據城市需求分布圖.html')

 

 

7.工作經驗要求漏斗圖

def get_experience(list):
    experience2 = {}
    for i in set(list):

         experience2[i] = list.count(i)

    return experience2

dir3 = get_experience(experience)

#print(dir3)

attr3= dir3.keys()
value3 = dir3.values()
funnel = Funnel("工作經驗漏斗圖",title_pos='center')

funnel.add("", attr3, value3,is_label_show=True,label_pos="inside", label_text_color="#fff",legend_orient='vertical',legend_pos='left')

funnel.render('工作經驗要求漏斗圖.html')

 

 

完整代碼：

import urllib.request
import xlwt
import re
import urllib.parse
import time

header={
     'Host':'search.51job.com',

     'Upgrade-Insecure-Requests':'1',
     'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
 }

#page是頁數，item是輸入的字符串
def getfront(page,item):
#先把字符串轉成十六進制編碼      
      result = urllib.parse.quote(item) 
                   
      ur1 = result+',2,'+ str(page)+'.html'
      ur2 = 'https://search.51job.com/list/000000,000000,0000,00,9,99,'
#拼接網址
      res = ur2+ur1 
      a = urllib.request.urlopen(res)
# 讀取源代碼並轉為unicode
      html = a.read().decode('gbk')          
      return html

 def getInformation(html):
     #匹配換行符
     reg = re.compile(r'class="t1 ">.*? <a target="_blank" title="(.*?)" href="(.*?)".*? <span class="t2"><a target="_blank" title="(.*?)" href="(.*?)".*?<span class="t3">(.*?)</span>.*?<span class="t4">(.*?)</span>.*?<span class="t5">(.*?)</span>.*?',re.S)
     items=re.findall(reg,html)
     return items

#新建表格空間
 excel1 = xlwt.Workbook()
# 設置單元格格式
 sheet1 = excel1.add_sheet('Job', cell_overwrite_ok=True)
 5 
 sheet1.write(0, 0, '序號')
 
 sheet1.write(0, 1, '職位')
 
 sheet1.write(0, 2, '公司名稱')
 
 sheet1.write(0, 3, '公司地點')
 
 sheet1.write(0, 4, '公司性質')
 
 sheet1.write(0, 5, '薪資')
 
 sheet1.write(0, 6, '學歷要求')
 
 sheet1.write(0, 7, '工作經驗')
 
 sheet1.write(0, 8, '公司規模')
 
 sheet1.write(0, 9, '公司類型')
 
 sheet1.write(0, 10,'公司福利')
 
 sheet1.write(0, 11,'發布時間')

 number = 1
 item = input()
 
 for j in range(1,1000):
     try:
         print("正在爬取第"+str(j)+"頁數據...")
#調用獲取網頁原碼 
        html = getfront(j,item)      
 
         for i in getInformation(html):
             try:
#職位網址
                 url1 = i[1]          
                 res1 = urllib.request.urlopen(url1).read().decode('gbk')
                 company = re.findall(re.compile(r'<div class="com_tag">.*?<p class="at" title="(.*?)"><span class="i_flag">.*?<p class="at" title="(.*?)">.*?<p class="at" title="(.*?)">.*?',re.S),res1)
 
                 job_need = re.findall(re.compile(r'<p class="msg ltype".*?>.*?&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;(.*?)&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;(.*?)&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;.*?</p>',re.S),res1)
 
                welfare = re.findall(re.compile(r'<span class="sp4">(.*?)</span>',re.S),res1)
                 print(i[0],i[2],i[4],i[5],company[0][0],job_need[2]
 [0],job_need[1][0],company[0][1],company[0][2],welfare,i[6])
                 sheet1.write(number,0,number)
 
                 sheet1.write(number,1,i[0])

                 sheet1.write(number,2,i[2])
 
                 sheet1.write(number,3,i[4])
 
                 sheet1.write(number,4,company[0][0])
 
                 sheet1.write(number,5,i[5])
 
                 sheet1.write(number,6,job_need[1][0])
 
                 sheet1.write(number,7,job_need[2][0])
 
                 sheet1.write(number,8,company[0][1])
 
                 sheet1.write(number,9,company[0][2])
 
                 sheet1.write(number,10,("  ".join(str(i) for i in welfare)))
 
                 sheet1.write(number,11,i[6])
 
                 number+=1
                 excel1.save("51job.xls")
#休息間隔，避免爬取海量數據時被誤判為攻擊，IP遭到封禁
                 time.sleep(0.3) 
             except:
                 pass
     except:
        pass

#coding:utf-8
 import pandas as pd
 import re
 
#除此之外還要安裝xlrd包
 
 data = pd.read_excel(r'51job.xls',sheet_name='Job')
 result = pd.DataFrame(data)

 a = result.dropna(axis=0,how='any')
#輸出全部行，不省略
 pd.set_option('display.max_rows',None) 

 b = u'數據'
 number = 1
 li = a['職位']
 for i in range(0,len(li)):

     try:
         if b in li[i]:
             #print(number,li[i])
             number+=1
         else:
             a = a.drop(i,axis=0)
     except:
         pass

 b2= u'人'
 li2 = a['學歷要求']
 for i in range(0,len(li2)):

     try:
         if b2 in li2[i]:
             #print(number,li2[i])
             number+=1
             a = a.drop(i,axis=0)

     except:
         pass

 b3 =u'萬/年'
 b4 =u'千/月'
 li3 = a['薪資']
 
 #注釋部分的print都是為了調試用的
 
 for i in range(0,len(li3)):
     try:
         if b3 in li3[i]:
             x = re.findall(r'\d*\.?\d+',li3[i])
             #print(x)
 
 #轉換成浮點型並保留兩位小數
             min_ = format(float(x[0])/12,'.2f')              
             max_ = format(float(x[1])/12,'.2f')
             li3[i][1] = min_+'-'+max_+u'萬/月'

         if b4 in li3[i]:
             x = re.findall(r'\d*\.?\d+',li3[i])
             #print(x)
 
             #input()
             min_ = format(float(x[0])/10,'.2f')
             max_ = format(float(x[1])/10,'.2f')
             li3[i][1] = str(min_+'-'+max_+'萬/月')
         print(i,li3[i])
 
     except:
         pass

 a.to_excel('51job2.xlsx', sheet_name='Job', index=False)

 # -*- coding: utf-8 -*-
 import pandas as pd
 import re

 from pyecharts import Funnel,Pie,Geo
 import matplotlib.pyplot as plt

 file = pd.read_excel(r'51job2.xls',sheet_name='Job')
 f = pd.DataFrame(file)

 pd.set_option('display.max_rows',None)

 add = f['公司地點']

 sly = f['薪資']

 edu = f['學歷要求']

 exp = f['工作經驗']

 address =[]

 salary = []

 education = []

 experience = []

 for i in range(0,len(f)):
     try:
         a = add[i].split('-')
         address.append(a[0])

         #print(address[i])
         s = re.findall(r'\d*\.?\d+',sly[i])
         s1= float(s[0])
         s2 =float(s[1])
         salary.append([s1,s2])

         #print(salary[i])
         education.append(edu[i])

         #print(education[i])
         experience.append(exp[i])

         #print(experience[i])
     except:
        pass

 #定義存放最低薪資的列表
 min_s=[]
 #定義存放最高薪資的列表
 max_s=[]
 for i in range(0,len(experience)):
     min_s.append(salary[i][0])
     max_s.append(salary[i][0])
 
 my_df = pd.DataFrame({'experience':experience, 'min_salay' : min_s, 
 #關聯工作經驗與薪資
 'max_salay' : max_s})
 data1 = my_df.groupby('experience').mean()['min_salay'].plot(kind='line')
 plt.show()
 
 my_df2 = pd.DataFrame({'education':education, 'min_salay' : min_s, 
#關聯學歷與薪資
 'max_salay' : max_s})
 data2 = my_df2.groupby('education').mean()['min_salay'].plot(kind='line')
 plt.show()

 def get_edu(list):
     education2 = {}
     for i in set(list):
         education2[i] = list.count(i)
     return education2
 dir1 = get_edu(education)
 
# print(dir1)
 
 attr= dir1.keys()
 value = dir1.values()
 pie = Pie("學歷要求")
 pie.add("", attr, value, center=[50, 50], is_random=False, radius=[30, 75], rosetype='radius',
         is_legend_show=False, is_label_show=True,legend_orient='vertical')
 pie.render('學歷要求玫瑰圖.html')

 def get_address(list):
     address2 = {}
     for i in set(list):
         address2[i] = list.count(i)
 
     address2.pop('異地招聘')
 
     #address2.pop('山東')
     #address2.pop('怒江')
     #address2.pop('池州')
 
     return address2
 
 dir2 = get_address(address)
 
 #print(dir2)
 
 geo = Geo("大數據人才需求分布圖", title_color="#2E2E2E",
           title_text_size=24,title_top=20,title_pos="center", width=1300,height=600)
 
 attr2 = dir2.keys()
 value2 = dir2.values()
 
 geo.add("",attr2, value2, type="effectScatter", is_random=True, visual_range=[0, 1000], maptype='china',symbol_size=8, effect_scale=5, is_visualmap=True)
 
 geo.render('大數據城市需求分布圖.html')

 def get_experience(list):
     experience2 = {}
     for i in set(list):
 
          experience2[i] = list.count(i)
 
     return experience2
 
 dir3 = get_experience(experience)
 
#print(dir3)
 
 attr3= dir3.keys()
 value3 = dir3.values()
 funnel = Funnel("工作經驗漏斗圖",title_pos='center')
 
 funnel.add("", attr3, value3,is_label_show=True,label_pos="inside", label_text_color="#fff",legend_orient='vertical',legend_pos='left')
 
 funnel.render('工作經驗要求漏斗圖.html')

 

五、總結

本次主題的爬蟲因基礎薄弱進行的時間較久，但結果還是好的。通過Execll文件和可視化分析可以清晰直觀的了解到應聘職位的各種要求，

基本達到了想要的結果。但是pyecharts里面的圖還有很多種，還是要繼續慢慢發掘，加強自己的專業知識。