爬取前程無憂官網 搜索大數據職位信息

1.主題式網絡爬蟲名稱：爬取前程無憂官網  搜索大數據職位信息
2.主題式網絡爬蟲爬取的內容與數據特征分析：爬取前程無憂官網  搜索大數據職位信息
3.主題式網絡爬蟲設計方案概述（包括實現思路與技術難點）

思路：通過按f12查找頁面的源代碼，找到所需代碼處在的標簽，通過爬蟲處理將所需代碼保存到excel文件內，再進行清洗，分析以及數據可視化的處理。

二、主題頁面的結構特征分析
1.主題頁面的結構與特征分析

 

 

 

 

2.Htmls頁面解析

 

 三、網絡爬蟲程序設計

1.數據爬取與采集

import urllib.request
import xlwt
import re
import urllib.parse
import time
header={
    'Host':'search.51job.com',
    'Upgrade-Insecure-Requests':'1',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
def getfront(page,item):       #page是頁數，item是輸入的字符串，見后文
     result = urllib.parse.quote(item)                    #先把字符串轉成十六進制編碼
     ur1 = result+',2,'+ str(page)+'.html'
     ur2 = 'https://search.51job.com/list/000000,000000,0000,00,9,99,'
     res = ur2+ur1                                                            #拼接網址
     a = urllib.request.urlopen(res)
     html = a.read().decode('gbk')          # 讀取源代碼並轉為unicode
     return html
def getInformation(html):
    reg = re.compile(r'class="t1 ">.*? <a target="_blank" title="(.*?)" href="(.*?)".*? <span class="t2"><a target="_blank" title="(.*?)" href="(.*?)".*?<span class="t3">(.*?)</span>.*?<span class="t4">(.*?)</span>.*?<span class="t5">(.*?)</span>.*?',re.S)#匹配換行符
    items=re.findall(reg,html)
    return items
#新建表格空間
excel1 = xlwt.Workbook()
# 設置單元格格式
sheet1 = excel1.add_sheet('Job', cell_overwrite_ok=True)
sheet1.write(0, 0, '序號')
sheet1.write(0, 1, '職位')
sheet1.write(0, 2, '公司名稱')
sheet1.write(0, 3, '公司地點')
sheet1.write(0, 4, '公司性質')
sheet1.write(0, 5, '薪資')
sheet1.write(0, 6, '學歷要求')
sheet1.write(0, 7, '工作經驗')
sheet1.write(0, 8, '公司規模')
sheet1.write(0, 9, '公司類型')
sheet1.write(0, 10,'公司福利')
sheet1.write(0, 11,'發布時間')
number = 1
item = input()
for j in range(1,10000):   #頁數自己隨便改
    try:
        print("正在爬取第"+str(j)+"頁數據...")
        html = getfront(j,item)      #調用獲取網頁原碼
        for i in getInformation(html):
            try:
                url1 = i[1]          #職位網址
                res1 = urllib.request.urlopen(url1).read().decode('gbk')
                company = re.findall(re.compile(r'<div class="com_tag">.*?<p class="at" title="(.*?)"><span class="i_flag">.*?<p class="at" title="(.*?)">.*?<p class="at" title="(.*?)">.*?',re.S),res1)
                job_need = re.findall(re.compile(r'<p class="msg ltype".*?>.*?&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;(.*?)&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;(.*?)&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;.*?</p>',re.S),res1)
                welfare = re.findall(re.compile(r'<span class="sp4">(.*?)</span>',re.S),res1)
                print(i[0],i[2],i[4],i[5],company[0][0],job_need[2][0],job_need[1][0],company[0][1],company[0][2],welfare,i[6])
                sheet1.write(number,0,number)
                sheet1.write(number,1,i[0])
                sheet1.write(number,2,i[2])
                sheet1.write(number,3,i[4])
                sheet1.write(number,4,company[0][0])
                sheet1.write(number,5,i[5])
                sheet1.write(number,6,job_need[1][0])
                sheet1.write(number,7,job_need[2][0])
                sheet1.write(number,8,company[0][1])
                sheet1.write(number,9,company[0][2])
                sheet1.write(number,10,("  ".join(str(i) for i in welfare)))
                sheet1.write(number,11,i[6])
                number+=1
                excel1.save("51job.xls")
                time.sleep(0.3) #休息間隔，避免爬取海量數據時被誤判為攻擊，IP遭到封禁
            except:
                pass
    except:
        pass

 

 

 

 

2.對數據進行清洗和處理

#coding:utf-8
import pandas as pd
import re
#除此之外還要安裝xlrd包

data = pd.read_excel(r'51job.xls',sheet_name='Job')
result = pd.DataFrame(data)
a = result.dropna(axis=0,how='any')
pd.set_option('display.max_rows',None)     #輸出全部行，不省略
b = u'數據'
number = 1
li = a['職位']
for i in range(0,len(li)):
    try:
        if b in li[i]:
            #print(number,li[i])
            number+=1
        else:
            a = a.drop(i,axis=0)
    except:
        pass
b2= u'人'
li2 = a['學歷要求']
for i in range(0,len(li2)):
    try:
        if b2 in li2[i]:
            #print(number,li2[i])
            number+=1
            a = a.drop(i,axis=0)
    except:
        pass
b3 =u'萬/年'
b4 =u'千/月'
li3 = a['薪資']
#注釋部分的print都是為了調試用的
for i in range(0,len(li3)):
    try:
        if b3 in li3[i]:
            x = re.findall(r'\d*\.?\d+',li3[i])
            #print(x)
            min_ = format(float(x[0])/12,'.2f')              #轉換成浮點型並保留兩位小數
            max_ = format(float(x[1])/12,'.2f')
            li3[i][1] = min_+'-'+max_+u'萬/月'
        if b4 in li3[i]:
            x = re.findall(r'\d*\.?\d+',li3[i])
            #print(x)
            #input()
            min_ = format(float(x[0])/10,'.2f')
            max_ = format(float(x[1])/10,'.2f')
            li3[i][1] = str(min_+'-'+max_+'萬/月')
        print(i,li3[i])

    except:
        pass
a.to_excel('51job2.xls', sheet_name='Job', index=False)

 

 

 

3.數據分析與可視化（例如：數據柱形圖、直方圖、散點圖、盒圖、分布圖）

# -*- coding: utf-8 -*-
import pandas as pd
import re
from pyecharts import Funnel,Pie,Geo
import matplotlib.pyplot as plt
file = pd.read_excel(r'51job2.xls',sheet_name='Job')
f = pd.DataFrame(file)
pd.set_option('display.max_rows',None)
add = f['公司地點']
sly = f['薪資']
edu = f['學歷要求']
exp = f['工作經驗']
address =[]
salary = []
education = []
experience = []
for i in range(0,len(f)):
    try:
        a = add[i].split('-')
        address.append(a[0])
        #print(address[i])
        s = re.findall(r'\d*\.?\d+',sly[i])
        s1= float(s[0])
        s2 =float(s[1])
        salary.append([s1,s2])
        #print(salary[i])
        education.append(edu[i])
        #print(education[i])
        experience.append(exp[i])
        #print(experience[i])
    except:
       pass
min_s=[]                            #定義存放最低薪資的列表
max_s=[]                            #定義存放最高薪資的列表
for i in range(0,len(experience)):
    min_s.append(salary[i][0])
    max_s.append(salary[i][0])

my_df = pd.DataFrame({'experience':experience, 'min_salay' : min_s, 'max_salay' : max_s})                #關聯工作經驗與薪資
data1 = my_df.groupby('experience').mean()['min_salay'].plot(kind='line')
plt.show()
my_df2 = pd.DataFrame({'education':education, 'min_salay' : min_s, 'max_salay' : max_s})                #關聯學歷與薪資
data2 = my_df2.groupby('education').mean()['min_salay'].plot(kind='line')
plt.show()
def get_edu(list):
    education2 = {}
    for i in set(list):
        education2[i] = list.count(i)
    return education2
dir1 = get_edu(education)
# print(dir1)

attr= dir1.keys()
value = dir1.values()
pie = Pie("學歷要求")
pie.add("", attr, value, center=[50, 50], is_random=False, radius=[30, 75], rosetype='radius',
        is_legend_show=False, is_label_show=True,legend_orient='vertical')
pie.render('學歷要求玫瑰圖.html')
def get_address(list):
    address2 = {}
    for i in set(list):
        address2[i] = list.count(i)
    address2.pop('異地招聘')
    # 有些地名可能不合法或者地圖包里沒有可以自行刪除，之前以下名稱都會報錯，現在好像更新了
    #address2.pop('山東')
    #address2.pop('怒江')
    #address2.pop('池州')
    return address2
dir2 = get_address(address)
#print(dir2)

geo = Geo("大數據人才需求分布圖", title_color="#2E2E2E",
          title_text_size=24,title_top=20,title_pos="center", width=1300,height=600)
attr2 = dir2.keys()
value2 = dir2.values()
geo.add("",attr2, value2, type="effectScatter", is_random=True, visual_range=[0, 1000], maptype='china',symbol_size=8, effect_scale=5, is_visualmap=True)
geo.render('大數據城市需求分布圖.html')
def get_experience(list):
    experience2 = {}
    for i in set(list):
         experience2[i] = list.count(i)
    return experience2
dir3 = get_experience(experience)
#print(dir3)

attr3= dir3.keys()
value3 = dir3.values()
funnel = Funnel("工作經驗漏斗圖",title_pos='center')
funnel.add("", attr3, value3,is_label_show=True,label_pos="inside", label_text_color="#fff",legend_orient='vertical',legend_pos='left')
funnel.render('工作經驗要求漏斗圖.html')

 

 

 

 

 

 

 

 

 

 

5.根據數據之間的關系，分析兩個變量之間的相關系數，畫出散點圖，並建立變量之間的回歸方程（一元或多元）（10分）。

X = df.score
Y = df.Numbers  
def func(params, x):
        a, b, c = params
        return a*x*x+b*x+c
def error(params,x,y):
     return func(params,x)-y    
def main(a,b,c):
    p0 = [0,0,0]
    Para=leastsq(error,p0,args=(X,Y))
    a,b,c=Para[0]    
    print("a=",a,"b=",b,"c=",c) 
    plt.scatter(X,Y,color="green",label=u"評分分布",linewidth=2)
    x=np.linspace(0,30,20)
    y=a*x*x+b*x+c
    plt.plot(x,y,color="red",label=u"回歸方程直線",linewidth=2)     
    plt.title("大數據職位信息關系圖")
    plt.legend()
    plt.grid()
    plt.show()
main()
#一元二次回歸方程

 

7.將以上各部分的代碼匯總，附上完整程序代碼

# -*- coding:utf-8 -*-
import urllib.request
import xlwt
import re
import urllib.parse
import time
header={
    'Host':'search.51job.com',
    'Upgrade-Insecure-Requests':'1',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
def getfront(page,item):       #page是頁數，item是輸入的字符串
     result = urllib.parse.quote(item)                    #先把字符串轉成十六進制編碼
     ur1 = result+',2,'+ str(page)+'.html'
     ur2 = 'https://search.51job.com/list/000000,000000,0000,00,9,99,'
     res = ur2+ur1                                                            #拼接網址
     a = urllib.request.urlopen(res)
     html = a.read().decode('gbk')          # 讀取源代碼並轉為unicode
     return html
def getInformation(html):
    reg = re.compile(r'class="t1 ">.*? <a target="_blank" title="(.*?)" href="(.*?)".*? <span class="t2"><a target="_blank" title="(.*?)" href="(.*?)".*?<span class="t3">(.*?)</span>.*?<span class="t4">(.*?)</span>.*?<span class="t5">(.*?)</span>.*?',re.S)#匹配換行符
    items=re.findall(reg,html)
    return items
#新建表格空間
excel1 = xlwt.Workbook()
# 設置單元格格式
sheet1 = excel1.add_sheet('Job', cell_overwrite_ok=True)
sheet1.write(0, 0, '序號')
sheet1.write(0, 1, '職位')
sheet1.write(0, 2, '公司名稱')
sheet1.write(0, 3, '公司地點')
sheet1.write(0, 4, '公司性質')
sheet1.write(0, 5, '薪資')
sheet1.write(0, 6, '學歷要求')
sheet1.write(0, 7, '工作經驗')
sheet1.write(0, 8, '公司規模')
sheet1.write(0, 9, '公司類型')
sheet1.write(0, 10,'公司福利')
sheet1.write(0, 11,'發布時間')
number = 1
item = input()
for j in range(1,10000):   #頁數自己隨便改
    try:
        print("正在爬取第"+str(j)+"頁數據...")
        html = getfront(j,item)      #調用獲取網頁原碼
        for i in getInformation(html):
            try:
                url1 = i[1]          #職位網址
                res1 = urllib.request.urlopen(url1).read().decode('gbk')
                company = re.findall(re.compile(r'<div class="com_tag">.*?<p class="at" title="(.*?)"><span class="i_flag">.*?<p class="at" title="(.*?)">.*?<p class="at" title="(.*?)">.*?',re.S),res1)
                job_need = re.findall(re.compile(r'<p class="msg ltype".*?>.*?&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;(.*?)&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;(.*?)&nbsp;&nbsp;<span>|</span>&nbsp;&nbsp;.*?</p>',re.S),res1)
                welfare = re.findall(re.compile(r'<span class="sp4">(.*?)</span>',re.S),res1)
                print(i[0],i[2],i[4],i[5],company[0][0],job_need[2][0],job_need[1][0],company[0][1],company[0][2],welfare,i[6])
                sheet1.write(number,0,number)
                sheet1.write(number,1,i[0])
                sheet1.write(number,2,i[2])
                sheet1.write(number,3,i[4])
                sheet1.write(number,4,company[0][0])
                sheet1.write(number,5,i[5])
                sheet1.write(number,6,job_need[2][0])
                sheet1.write(number,7,job_need[1][0])
                sheet1.write(number,8,company[0][1])
                sheet1.write(number,9,company[0][2])
                sheet1.write(number,10,("  ".join(str(i) for i in welfare)))
                sheet1.write(number,11,i[6])
                number+=1
                excel1.save("51job.xls")
                time.sleep(0.3) #休息間隔，避免爬取海量數據時被誤判為攻擊，IP遭到封禁
            except:
                pass
    except:
        pass
#coding:utf-8
import pandas as pd
import re

data = pd.read_excel(r'51job.xls',sheet_name='Job')
result = pd.DataFrame(data)

a = result.dropna(axis=0,how='any')
pd.set_option('display.max_rows',None)     #輸出全部行，不省略

b = u'數據'
number = 1
li = a['職位']
for i in range(0,len(li)):
    try:
        if b in li[i]:
            #print(number,li[i])
            number+=1
        else:
            a = a.drop(i,axis=0)  #刪除整行
    except:
        pass

b2 = '人'
li2 = a['學歷要求']
for i in range(0,len(li2)):
    try:
        if b2 in li2[i]:
            # print(number,li2[i])
            number += 1
            a = a.drop(i, axis=0)
    except:
        pass

b3 =u'萬/年'
b4 =u'千/月'
li3 = a['薪資']
#注釋部分的print都是為了調試用的
for i in range(0,len(li3)):
    try:
        if b3 in li3[i]:
            x = re.findall(r'\d*\.?\d+',li3[i])
            #print(x)
            min_ = format(float(x[0])/12,'.2f')              #轉換成浮點型並保留兩位小數
            max_ = format(float(x[1])/12,'.2f')
            li3[i][1] = min_+'-'+max_+u'萬/月'
        if b4 in li3[i]:
            x = re.findall(r'\d*\.?\d+',li3[i])
            #print(x)
            #input()
            min_ = format(float(x[0])/10,'.2f')
            max_ = format(float(x[1])/10,'.2f')
            li3[i][1] = str(min_+'-'+max_+'萬/月')
        print(i,li3[i])

    except:
        pass
a.to_excel('51job2.xls', sheet_name='Job', index=False)
#############################################################################################
import pandas as pd
import re
from pyecharts import Funnel,Pie,Geo
import matplotlib.pyplot as plt

file = pd.read_excel(r'51job2.xls',sheet_name='Job')
f = pd.DataFrame(file)
pd.set_option('display.max_rows',None)

add = f['公司地點']
sly = f['薪資']
edu = f['學歷要求']
exp = f['工作經驗']
address =[]
salary = []
education = []
experience = []
for i in range(0,len(f)):
    try:
        a = add[i].split('-')
        address.append(a[0])
        #print(address[i])
        s = re.findall(r'\d*\.?\d+',sly[i])
        s1= float(s[0])
        s2 =float(s[1])
        salary.append([s1,s2])
        #print(salary[i])
        education.append(edu[i])
        #print(education[i])
        experience.append(exp[i])
        #print(experience[i])
    except:
       pass

min_s=[]                            #定義存放最低薪資的列表
max_s=[]                            #定義存放最高薪資的列表
for i in range(0,len(experience)):
    min_s.append(salary[i][0])
    max_s.append(salary[i][0])
#matplotlib模塊如果顯示不了中文字符串可以用以下代碼。
plt.rcParams['font.sans-serif'] = ['KaiTi'] # 指定默認字體
plt.rcParams['axes.unicode_minus'] = False # 解決保存圖像是負號'-'顯示為方塊的問題

my_df = pd.DataFrame({'experience':experience, 'min_salay' : min_s, 'max_salay' : max_s})                #關聯工作經驗與薪資
data1 = my_df.groupby('experience').mean()['min_salay'].plot(kind='line')
plt.show()
my_df2 = pd.DataFrame({'education':education, 'min_salay' : min_s, 'max_salay' : max_s})                #關聯學歷與薪資
data2 = my_df2.groupby('education').mean()['min_salay'].plot(kind='line')
plt.show()

def get_edu(list):
    education2 = {}
    for i in set(list):
        education2[i] = list.count(i)
    return education2
dir1 = get_edu(education)
# print(dir1)

attr= dir1.keys()
value = dir1.values()
pie = Pie("學歷要求")
pie.add("", attr, value, center=[50, 50], is_random=False, radius=[30, 75], rosetype='radius',
        is_legend_show=False, is_label_show=True,legend_orient='vertical')
pie.render('學歷要求玫瑰圖.html')

def get_address(list):
    address2 = {}
    for i in set(list):
        address2[i] = list.count(i)
    address2.pop('異地招聘')
    # 有些地名可能不合法或者地圖包里沒有可以自行刪除，之前以下名稱都會報錯，現在好像更新了
    #address2.pop('山東')
    #address2.pop('怒江')
    #address2.pop('池州')
    return address2
dir2 = get_address(address)
#print(dir2)

geo = Geo("大數據人才需求分布圖", title_color="#2E2E2E",
          title_text_size=24,title_top=20,title_pos="center", width=1300,height=600)
attr2 = dir2.keys()
value2 = dir2.values()
geo.add("",attr2, value2, type="effectScatter", is_random=True, visual_range=[0, 1000], maptype='china',symbol_size=8, effect_scale=5, is_visualmap=True)
geo.render('大數據城市需求分布圖.html')

def get_experience(list):
    experience2 = {}
    for i in set(list):
         experience2[i] = list.count(i)
    return experience2
dir3 = get_experience(experience)
#print(dir3)

attr3= dir3.keys()
value3 = dir3.values()
funnel = Funnel("工作經驗漏斗圖",title_pos='center')
funnel.add("", attr3, value3,is_label_show=True,label_pos="inside", label_text_color="#fff",legend_orient='vertical',legend_pos='left')
funnel.render('工作經驗要求漏斗圖.html')
X = df.score
Y = df.Numbers  
def func(params, x):
        a, b, c = params
        return a*x*x+b*x+c
def error(params,x,y):
     return func(params,x)-y    
def main(a,b,c):
    p0 = [0,0,0]
    Para=leastsq(error,p0,args=(X,Y))
    a,b,c=Para[0]    
    print("a=",a,"b=",b,"c=",c) 
    plt.scatter(X,Y,color="green",label=u"評分分布",linewidth=2)
    x=np.linspace(0,30,20)
    y=a*x*x+b*x+c
    plt.plot(x,y,color="red",label=u"回歸方程直線",linewidth=2)     
    plt.title("大數據職位信息關系圖")
    plt.legend()
    plt.grid()
    plt.show()
main()
#一元二次回歸方程

1.經過對主題數據的分析與可視化，可以得到哪些結論？

經過對主題數據的分析與可視化可以更直觀的了解數據
2.對本次程序設計任務完成的情況做一個簡單的小結。

通過此次作業了解到了對於函數熟悉應用重要性

對代碼不斷的修改，對於python有進一步的認識，我明白了數據的分析與可視化，掌握了不少庫的使用，加深了對python的熱愛通過這次做題任務，我在完成的過程中遇到了很多的困難，讓我得到了許多收獲，也讓我充分的認識到了自己的不足之處！