clear_data.py
#!/usr/bin/env python # -*- coding: utf-8 -*- import requests from docx import Document from docx.shared import Inches,Pt from docx.oxml.ns import qn key_word =['小孩','兒童','兒子','女兒','小孩子','娃','嬉戲','親子','玩具','寶寶','寶貝'] childer =[] with open('code_dict.txt','r',encoding='utf-8')as f: content = f.readlines() for one_line in content: dict = eval(one_line) for w in key_word: if w in dict['comment']: childer.append(dict) break print(len(childer)) print((childer[1:10])) doc =Document() for i in childer: print(i) name = i.get('name') comment = i.get('comment') time = i.get('time') star = i.get('star') pic = i.get('pic') pen = doc.add_paragraph() ph = pen.paragraph_format ph.line_spacing = Pt(22) pensize1 = pen.add_run('用戶：'+name+'\n') pensize = pen.add_run('評分：'+str(star)+'\n') pensize2 = pen.add_run('時間：'+time+'\n') pensize3 = pen.add_run('評論：'+comment+'\n') pensize.font.name = '宋體' pensize._element.rPr.rFonts.set(qn('w:eastAsia'), '宋體') pensize.font.size = Pt(15) pensize.bold=True pensize1.font.name = '宋體' pensize1._element.rPr.rFonts.set(qn('w:eastAsia'), '宋體') pensize1.font.size = Pt(15) pensize2.font.name = '宋體' pensize2._element.rPr.rFonts.set(qn('w:eastAsia'), '宋體') pensize2.font.size = Pt(15) pensize3.font.name = '宋體' pensize3._element.rPr.rFonts.set(qn('w:eastAsia'), '宋體') pensize3.font.size = Pt(15) if pic: for p in pic: req = requests.get(p) with open('capth.png','wb')as f: f.write(req.content) doc.add_picture('capth.png', width=Inches(2.5)) doc.save('dianping.docx')
dazhong.py
#!/usr/bin/env python # -*- coding: utf-8 -*- import datetime import random import time import re # from selenium.webdriver.chrome.options import Options # from selenium import webdriver from lxml import etree import requests class DianpingComment: font_size = 14 start_y = 23 def __init__(self, shop_id, cookies, delay=7, handle_ban=False): self.shop_id = shop_id self._delay = delay self._cookies = self._format_cookies(cookies) self._css_headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36', } self._default_headers = { 'Connection': 'keep-alive', 'Host': 'www.dianping.com', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36', } self._cur_request_url = 'http://www.dianping.com/shop/{}/review_all/p1'.format(shop_id) if handle_ban: print('不想寫跳過驗證了') # self._browser = self._init_browser() # self._handle_ban() def run(self): self._css_link = self._get_css_link(self._cur_request_url) self._font_dict = self._get_font_dict(self._css_link) self._get_conment_page() def _delay_func(self): delay_time = random.randint((self._delay - 2) * 10, (self._delay + 2) * 10) * 0.1 print('睡一會',delay_time) time.sleep(delay_time) # def _init_browser(self): # """ # 初始化游覽器 # """ # chrome_options = Options() # chrome_options.add_argument('--headless') # chrome_options.add_argument('--disable-gpu') # browser = webdriver.Chrome(chrome_options=chrome_options) # browser.get(self._cur_request_url) # for name, value in self._cookies.items(): # browser.add_cookie({'name': name, 'value': value}) # browser.refresh() # return browser # def _handle_ban(self): # """ # 爬取速度過快，出現異常時處理驗證 # """ # try: # self._browser.refresh() # time.sleep(1) # button = self._browser.find_element_by_id('yodaBox') # move_x_offset = self._browser.find_element_by_id('yodaBoxWrapper').size['width'] # webdriver.ActionChains(self._browser).drag_and_drop_by_offset( # button, move_x_offset, 0).perform() # except: # pass def _format_cookies(self, cookies): cookies = {cookie.split('=')[0]: cookie.split('=')[1] for cookie in cookies.replace(' ', '').split(';')} return cookies def _get_conment_page(self): # 獲得評論內容 """ 請求評論頁，並將<span></span>樣式替換成文字 """ while self._cur_request_url: self._delay_func() print('[{now_time}] {msg}'.format(now_time=datetime.datetime.now(), msg=self._cur_request_url)) res = requests.get(self._cur_request_url, headers=self._default_headers, cookies=self._cookies) html = res.text class_set = set() for span in re.findall(r'<span class="([a-zA-Z0-9]{5,6})"></span>', html): class_set.add(span) for class_name in class_set: html = re.sub('<span class="%s"></span>' % class_name, self._font_dict[class_name], html) doc = etree.HTML(html) self._parse_comment_page(doc) try: self._default_headers['Referer'] = self._cur_request_url next_page_url = 'http://www.dianping.com' + doc.xpath('.//a[@class="NextPage"]/@href')[0] except IndexError: next_page_url = None self._cur_request_url = next_page_url def _data_pipeline(self, data): """ 處理數據 """ print(data) def _parse_comment_page(self, doc): """ 解析評論頁並提取數據 """ for li in doc.xpath('//*[@class="reviews-items"]/ul/li'): name = li.xpath('.//a[@class="name"]/text()')[0].strip('\n\r \t') try: star = li.xpath('.//span[contains(./@class, "sml-str")]/@class')[0] star = re.findall(r'sml-rank-stars sml-str(.*?) star', star)[0] except IndexError: star = 0 time = li.xpath('.//span[@class="time"]/text()')[0].strip('\n\r \t') pics =[] if li.xpath('.//*[@class="review-pictures"]/ul/li'): for pic in li.xpath('.//*[@class="review-pictures"]/ul/li'): print(pic.xpath('.//a/@href')) pics.append(pic.xpath('.//a/img/@data-big')[0]) comment = ''.join(li.xpath('.//div[@class="review-words Hide"]/text()')).strip('\n\r \t') if not comment: comment = ''.join(li.xpath('.//div[@class="review-words"]/text()')).strip('\n\r \t') data = { 'name': name, 'comment': comment, 'star': star, 'pic':pics, 'time': time, } self._data_pipeline(data) def _get_css_link(self, url): """ 請求評論首頁，獲取css樣式文件 """ res = requests.get(url, headers=self._default_headers, cookies=self._cookies) html = res.text # print(html) # css_link = re.search(r'<link re.*?css.*?href="(.*?svgtextcss.*?)">', html) css_link = re.findall(r'<link rel="stylesheet" type="text/css" href="//s3plus.meituan.net/v1/(.*?)">', html) assert css_link css_link = 'http://s3plus.meituan.net/v1/' + css_link[0] return css_link def _get_font_dict(self, url): """ 獲取css樣式對應文字的字典 """ res = requests.get(url, headers=self._css_headers) html = res.text background_image_link = re.findall(r'background-image: url\((.*?)\);', html) print('帶有svg的鏈接',background_image_link) assert background_image_link background_image_link = 'http:' + background_image_link[1] html = re.sub(r'span.*?\}', '', html) group_offset_list = re.findall(r'\.([a-zA-Z0-9]{5,6}).*?round:(.*?)px (.*?)px;', html) # css中的類 print('css中class對應坐標',group_offset_list) font_dict_by_offset = self._get_font_dict_by_offset(background_image_link) # svg得到這里面圖片對應成字典 print('解析svg成字典',font_dict_by_offset) font_dict = {} for class_name, x_offset, y_offset in group_offset_list: y_offset = y_offset.replace('.0', '') x_offset = x_offset.replace('.0', '') # print(y_offset,x_offset) if font_dict_by_offset.get(int(y_offset)): font_dict[class_name] = font_dict_by_offset[int(y_offset)][int(x_offset)] return font_dict def _get_font_dict_by_offset(self, url): """ 獲取坐標偏移的文字字典, 會有最少兩種形式的svg文件（目前只遇到兩種） """ res = requests.get(url, headers=self._css_headers) html = res.text font_dict = {} # print(html) y_list = re.findall(r'd="M0 (\d+?) ', html) if y_list: font_list = re.findall(r'<textPath .*?>(.*?)<', html) for i, string in enumerate(font_list): y_offset = self.start_y - int(y_list[i]) sub_font_dict = {} for j, font in enumerate(string): x_offset = -j * self.font_size sub_font_dict[x_offset] = font font_dict[y_offset] = sub_font_dict else: font_list = re.findall(r'<text.*?y="(.*?)">(.*?)<', html) for y, string in font_list: y_offset = self.start_y - int(y) sub_font_dict = {} for j, font in enumerate(string): x_offset = -j * self.font_size sub_font_dict[x_offset] = font font_dict[y_offset] = sub_font_dict return font_dict if __name__ == "__main__": pass
demo.py
#!/usr/bin/env python # -*- coding: utf-8 -*- from dazhong import DianpingComment COOKIES = '_lxsdk_cuid=1699b152d90c8-04b0ee8b481697-541f3415-1fa400-1699b152d91c8; _lxsdk=1699b152d90c8-04b0ee8b481697-541f3415-1fa400-1699b152d91c8; _hc.v=992d8c67-a9b0-ee61-c6cf-ed9b42cfe11f.1553085051; _thirdu.c=136cbfec8b174105c45f6628ce431df6; ctu=cc29f77c02b4556c6a1db1c67c5c10e084f7f63d00208c59788c11a4845348aa; cy=160; cye=zhengzhou; thirdtoken=e0dfd5bf-3cc9-482c-a559-ecb5a5408581; dper=13f0e16d38f4829e80270687b88c4ce8d36d333a6f525bc6be3dec9bbc60b1d7f44f8b47a413dc1c18f3ef5fed921594f3c5161e72d50fed52f3006625babe559507c56bb8b77d1f9dd95d104ffb3cdba1c49805e34df17c99e3ba781183b850; ll=7fd06e815b796be3df069dec7836c3df; ua=aJay13; ctu=a5f067d1428ce75e417e53634b352a7767a63503c85b2d59c0c70ae996add3e701d656899061b0eddfa568430b723553; _lxsdk_s=1699df6ef73-4f6-781-d9c%7C%7C719' class Customer(DianpingComment): def _data_pipeline(self, data): print(data) with open('code_dict.txt','a+',encoding='utf-8')as f: f.write(str(data)+'\n') if __name__ == "__main__": dianping = Customer('1726435', cookies=COOKIES) dianping.run()