一、选题背景
由于现在的音乐版权问题，很多音乐分布在各个平台的音乐播放器，而版权问题也使很多人非常的困扰，从而找不到音乐的资源。因此为帮助使用网易云的伙伴们，更好的找到各个平台的资源，听到更多自己喜欢的歌。
二、网络爬虫设计方案
网络爬虫名称：“网易云音乐歌单”
内容与数据分析特征：该爬虫主要获取性能榜的数据进行分析。
三、主题页面的结构特征分析
四、网络爬虫程序设计
1.数据爬取与采集
from bs4 import BeautifulSoup import requests import time headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36' } for i in range(0, 1330, 35): print(i) time.sleep(2) url = 'https://music.163.com/discover/playlist/?cat=欧美&order=hot&limit=35&offset=' + str(i) response = requests.get(url=url, headers=headers) html = response.text soup = BeautifulSoup(html, 'html.parser') # 获取包含歌单详情页网址的标签 ids = soup.select('.dec a') # 获取包含歌单索引页信息的标签 lis = soup.select('#m-pl-container li') print(len(lis)) for j in range(len(lis)): # 获取歌单详情页地址 url = ids[j]['href'] # 获取歌单标题 title = ids[j]['title'] # 获取歌单播放量 play = lis[j].select('.nb')[0].get_text() # 获取歌单贡献者名字 user = lis[j].select('p')[1].select('a')[0].get_text() # 输出歌单索引页信息 print(url, title, play, user) # 将信息写入CSV文件中 with open('playlist.csv', 'a+', encoding='utf-8-sig') as f: f.write(url + ',' + title + ',' + play + ',' + user + ' ')
from bs4 import BeautifulSoup import pandas as pd import requests import time df = pd.read_csv('playlist.csv', header=None, error_bad_lines=False, names=['url', 'title', 'play', 'user']) headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36' } for i in df['url']: time.sleep(2) url = 'https://music.163.com' + i response = requests.get(url=url, headers=headers) html = response.text soup = BeautifulSoup(html, 'html.parser') # 获取歌单标题 title = soup.select('h2')[0].get_text().replace(',', '，') # 获取标签 tags = [] tags_message = soup.select('.u-tag i') for p in tags_message: tags.append(p.get_text()) # 对标签进行格式化 if len(tags) > 1: tag = '-'.join(tags) else: tag = tags[0] # 获取歌单介绍 if soup.select('#album-desc-more'): text = soup.select('#album-desc-more')[0].get_text().replace(' ', '').replace(',', '，') else: text = '无' # 获取歌单收藏量 collection = soup.select('#content-operation i')[1].get_text().replace('(', '').replace(')', '') # 歌单播放量 play = soup.select('.s-fc6')[0].get_text() # 歌单内歌曲数 songs = soup.select('#playlist-track-count')[0].get_text() # 歌单评论数 comments = soup.select('#cnt_comment_count')[0].get_text() # 输出歌单详情页信息 print(title, tag, text, collection, play, songs, comments) # 将详情页信息写入CSV文件中 with open('music_message.csv', 'a+', encoding='utf-8-sig') as f: f.write(title + ',' + tag + ',' + text + ',' + collection + ',' + play + ',' + songs + ',' + comments + ' ') # 获取歌单内歌曲名称 li = soup.select('.f-hide li a') for j in li: with open('music_name.csv', 'a+', encoding='utf-8-sig') as f: f.write(j.get_text() + ' ')
2.数据可视化
import numpy as np import pandas as pd import matplotlib.pyplot as plt df = pd.read_csv('music_message_4.csv', header=None) # 对播放数取对数 dom = [] for i in df[4]: dom.append(np.log(i)) df['collection'] = dom # 设置图片显示属性,字体及大小 plt.rcParams['font.sans-serif'] = ['STXihei'] plt.rcParams['font.size'] = 12 plt.rcParams['axes.unicode_minus'] = False # 设置图片显示属性 fig = plt.figure(figsize=(16, 8), dpi=80) ax = plt.subplot(1, 1, 1) ax.patch.set_color('white') # 设置坐标轴属性 lines = plt.gca() # 设置坐标轴颜色 lines.spines['right'].set_color('none') lines.spines['top'].set_color('none') lines.spines['left'].set_color((64/255, 64/255, 64/255)) lines.spines['bottom'].set_color((64/255, 64/255, 64/255)) lines.xaxis.set_ticks_position('none') lines.yaxis.set_ticks_position('none') # 绘制直方图,设置直方图颜色 ax.hist(df['collection'], bins=30, alpha=0.7, color=(255/255, 153/255, 0/255)) ax.set_title('华语歌单播放数量分布情况', fontsize=20) # 显示图片 plt.show()
import pandas as pd import matplotlib.pyplot as plt df = pd.read_csv('music_message_3.csv', header=None, names=['title'], encoding='utf-8-sig') # 数据聚合分组 place_message = df.groupby(['title']) place_com = place_message['title'].agg(['count']) place_com.reset_index(inplace=True) place_com_last = place_com.sort_index() dom = place_com_last.sort_values('count', ascending=False)[0:10] # 设置显示数据 names = [i for i in dom.title] names.reverse() nums = [i for i in dom['count']] nums.reverse() data = pd.Series(nums, index=names) # 设置图片显示属性,字体及大小 plt.rcParams['font.sans-serif'] = ['Microsoft YaHei'] plt.rcParams['font.size'] = 10 plt.rcParams['axes.unicode_minus'] = False # 设置图片显示属性 fig = plt.figure(figsize=(16, 8), dpi=80) ax = plt.subplot(1, 1, 1) ax.patch.set_color('white') # 设置坐标轴属性 lines = plt.gca() # 设置坐标轴颜色 lines.spines['right'].set_color('none') lines.spines['top'].set_color('none') lines.spines['left'].set_color((64/255, 64/255, 64/255)) lines.spines['bottom'].set_color((64/255, 64/255, 64/255)) # 设置坐标轴刻度 lines.xaxis.set_ticks_position('none') lines.yaxis.set_ticks_position('none') # 绘制柱状图,设置柱状图颜色 data.plot.barh(ax=ax, width=0.7, alpha=0.7, color=(16/255, 152/255, 168/255)) # 添加标题,设置字体大小 ax.set_title('网易云音乐华语歌单歌曲 TOP10', fontsize=18, fontweight='light') # 添加歌曲出现次数文本 for x, y in enumerate(data.values): plt.text(y+3.5, x-0.12, '%s' % y, ha='center') # 显示图片 plt.show()
import numpy as np import pandas as pd import matplotlib.pyplot as plt df = pd.read_csv('music_message_4.csv', header=None) # 对收藏数取对数 dom = [] for i in df[3]: dom.append(np.log(int(i.replace('万', '0000')))) df['collection'] = dom # 设置图片显示属性,字体及大小 plt.rcParams['font.sans-serif'] = ['STXihei'] plt.rcParams['font.size'] = 12 plt.rcParams['axes.unicode_minus'] = False # 设置图片显示属性 fig = plt.figure(figsize=(16, 8), dpi=80) ax = plt.subplot(1, 1, 1) ax.patch.set_color('white') # 设置坐标轴属性 lines = plt.gca() # 设置坐标轴颜色 lines.spines['right'].set_color('none') lines.spines['top'].set_color('none') lines.spines['left'].set_color((64/255, 64/255, 64/255)) lines.spines['bottom'].set_color((64/255, 64/255, 64/255)) lines.xaxis.set_ticks_position('none') lines.yaxis.set_ticks_position('none') # 绘制直方图,设置直方图颜色 ax.hist(df['collection'], bins=30, alpha=0.7, color=(21/255, 47/255, 71/255)) ax.set_title('华语歌单收藏数量分布情况', fontsize=20) # 显示图片 plt.show()
import squarify import pandas as pd import matplotlib.pyplot as plt df = pd.read_csv('music_message_4.csv', header=None) # 处理标签信息 tags = [] dom2 = [] for i in df[1]: c = i.split('-') for j in c: if j not in tags: tags.append(j) else: continue for item in tags: num = 0 for i in df[1]: type2 = i.split('-') for j in range(len(type2)): if type2[j] == item: num += 1 else: continue dom2.append(num) # 数据创建 data = {'tags': tags, 'num': dom2} frame = pd.DataFrame(data) df1 = frame.sort_values(by='num', ascending=False) name = df1['tags'][:10] income = df1['num'][:10] # 绘图details colors = ['#993333', '#CC9966', '#333333', '#663366', '#003366', '#009966', '#FF6600', '#FF0033', '#009999', '#333366'] plot = squarify.plot(sizes=income, label=name, color=colors, alpha=1, value=income, edgecolor='white', linewidth=1.5) # 设置图片显示属性,字体及大小 plt.rcParams['font.sans-serif'] = ['Microsoft YaHei'] plt.rcParams['font.size'] = 8 plt.rcParams['axes.unicode_minus'] = False # 设置标签大小为1 plt.rc('font', size=6) # 设置标题大小 plot.set_title('网易云音乐华语歌单标签图', fontsize=13, fontweight='light') # 除坐标轴 plt.axis('off') # 除上边框和右边框刻度 plt.tick_params(top=False, right=False) # 图形展示 plt.show()
五、总结
网易云音乐的使用还是非常火爆的，以上是对网易云爬虫的一次愉快的探索之旅~