Python爬蟲+可視化教學：爬取分析寵物貓咪交易數據

前言

各位，七夕快到了，想好要送什么禮物了嗎？

昨天有朋友私信我，問我能用Python分析下網上小貓咪的數據，是想要送一只給女朋友，當做禮物。

Python從零基礎入門到實戰系統教程、源碼、視頻

網上的數據太多、太雜，而且我也不知道哪個網站的數據比較好。所以，只能找到一個貓咪交易網站的數據來分析了

地址：

http://www.maomijiaoyi.com/

 

 

 

爬蟲部分

請求數據

import requests

url = f'http://www.maomijiaoyi.com/index.php?/chanpinliebiao_c_2_1--24.html'
headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'
}
response = requests.get(url=url, headers=headers)
print(response.text)

 

解析數據

# 把獲取到的 html 字符串數據轉換成 selector 對象 這樣調用
selector = parsel.Selector(response.text)
# css 選擇器只要是根據標簽屬性內容提取數據 編程永遠不看過程 只要結果
href = selector.css('.content:nth-child(1) a::attr(href)').getall()
areas = selector.css('.content:nth-child(1) .area .color_333::text').getall()
areas = [i.strip() for i in areas] # 列表推導式

 

提取標簽數據

for index in zip(href, areas):
    # http://www.maomijiaoyi.com/index.php?/chanpinxiangqing_224383.html
    index_url = 'http://www.maomijiaoyi.com' + index[0]
    response_1 = requests.get(url=index_url, headers=headers)
    selector_1 = parsel.Selector(response_1.text)
    area = index[1]
    # getall 取所有 get 取一個
    title = selector_1.css('.detail_text .title::text').get().strip()
    shop = selector_1.css('.dinming::text').get().strip()  # 店名
    price = selector_1.css('.info1 div:nth-child(1) span.red.size_24::text').get()  # 價格
    views = selector_1.css('.info1 div:nth-child(1) span:nth-child(4)::text').get()  # 瀏覽次數
    # replace() 替換
    promise = selector_1.css('.info1 div:nth-child(2) span::text').get().replace('賣家承諾: ', '')  # 瀏覽次數
    num = selector_1.css('.info2 div:nth-child(1) div.red::text').get()  # 在售只數
    age = selector_1.css('.info2 div:nth-child(2) div.red::text').get()  # 年齡
    kind = selector_1.css('.info2 div:nth-child(3) div.red::text').get()  # 品種
    prevention = selector_1.css('.info2 div:nth-child(4) div.red::text').get()  # 預防
    person = selector_1.css('div.detail_text .user_info div:nth-child(1) .c333::text').get()  # 聯系人
    phone = selector_1.css('div.detail_text .user_info div:nth-child(2) .c333::text').get()  # 聯系方式
    postage = selector_1.css('div.detail_text .user_info div:nth-child(3) .c333::text').get().strip()  # 包郵
    purebred = selector_1.css(
        '.xinxi_neirong div:nth-child(1) .item_neirong div:nth-child(1) .c333::text').get().strip()  # 是否純種
    sex = selector_1.css(
        '.xinxi_neirong div:nth-child(1) .item_neirong div:nth-child(4) .c333::text').get().strip()  # 貓咪性別
    video = selector_1.css(
        '.xinxi_neirong div:nth-child(2) .item_neirong div:nth-child(4) .c333::text').get().strip()  # 能否視頻
    worming = selector_1.css(
        '.xinxi_neirong div:nth-child(2) .item_neirong div:nth-child(2) .c333::text').get().strip()  # 是否驅蟲
    dit = {
        '地區': area,
        '店名': shop,
        '標題': title,
        '價格': price,
        '瀏覽次數': views,
        '賣家承諾': promise,
        '在售只數': num,
        '年齡': age,
        '品種': kind,
        '預防': prevention,
        '聯系人': person,
        '聯系方式': phone,
        '異地運費': postage,
        '是否純種': purebred,
        '貓咪性別': sex,
        '驅蟲情況': worming,
        '能否視頻': video,
        '詳情頁': index_url,
    }

 

保存數據

import csv # 內置模塊

f = open('貓咪1.csv', mode='a', encoding='utf-8', newline='')
csv_writer = csv.DictWriter(f, fieldnames=['地區', '店名', '標題', '價格', '瀏覽次數', '賣家承諾', '在售只數',
                                           '年齡', '品種', '預防', '聯系人', '聯系方式', '異地運費', '是否純種',
                                           '貓咪性別', '驅蟲情況', '能否視頻', '詳情頁'])
csv_writer.writeheader() # 寫入表頭
csv_writer.writerow(dit)
print(title, area, shop, price, views, promise, num, age,
      kind, prevention, person, phone, postage, purebred, sex, video, worming, index_url, sep=' | ')    

 

得到數據

 

 

數據可視化部分

詞雲圖

from pyecharts import options as opts
from pyecharts.charts import WordCloud
from pyecharts.globals import SymbolType
from pyecharts.globals import ThemeType


words = [(i,1) for i in cat_info['品種'].unique()]
c = (
    WordCloud(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
    .add("", words,shape=SymbolType.DIAMOND)
    .set_global_opts(title_opts=opts.TitleOpts(title=""))
)
c.render_notebook()

 

 

 

交易品種占比圖

from pyecharts import options as opts
from pyecharts.charts import TreeMap

pingzhong = cat_info['品種'].value_counts().reset_index()
data = [{'value':i[1],'name':i[0]} for i in zip(list(pingzhong['index']),list(pingzhong['品種']))]

c = (
    TreeMap(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
    .add("", data)
    .set_global_opts(title_opts=opts.TitleOpts(title=""))
    .set_series_opts(label_opts=opts.LabelOpts(position="inside"))
)

c.render_notebook()

 

 

 

均價占比圖

from pyecharts import options as opts
from pyecharts.charts import PictorialBar
from pyecharts.globals import SymbolType

location = list(price['品種'])
values = list(price['價格'])

c = (
    PictorialBar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
    .add_xaxis(location)
    .add_yaxis(
        "",
        values,
        label_opts=opts.LabelOpts(is_show=False),
        symbol_size=18,
        symbol_repeat="fixed",
        symbol_offset=[0, 0],
        is_symbol_clip=True,
        symbol=SymbolType.ROUND_RECT,
    )
    .reversal_axis()
    .set_global_opts(
        title_opts=opts.TitleOpts(title="均價排名"),
        xaxis_opts=opts.AxisOpts(is_show=False),
        yaxis_opts=opts.AxisOpts(
            axistick_opts=opts.AxisTickOpts(is_show=False),
            axisline_opts=opts.AxisLineOpts(
                linestyle_opts=opts.LineStyleOpts(opacity=0),
            
            ),
        ),
    )
    .set_series_opts(
        label_opts=opts.LabelOpts(position='insideRight')
    )
)

c.render_notebook()

 

 

 

貓齡柱狀圖

from pyecharts import options as opts
from pyecharts.charts import Bar
from pyecharts.faker import Faker

x = ['1-3個月','3-6個月','6-9個月','9-12個月','1年以上']
y = [69343,115288,18239,4139,5]

c = (
    Bar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
    .add_xaxis(x)
    .add_yaxis('', y)
    .set_global_opts(title_opts=opts.TitleOpts(title="貓齡分布"))
)

c.render_notebook()