登录+爬取淘宝商品信息+字典用json存储

任务1:利用cookie可以免去登录的烦恼(验证码)

'''
	只需要有登录后的cookie,就可以绕过验证码
	登录后的cookie可以通过Selenium用第三方(微博)进行登录,不需要进行淘宝的滑动验证码
'''


import requests
from urllib.parse import urlencode


headers = {
	'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36',
	# 登录后的cookie
	'cookie': 'xxx',
}

params = {
	'q': 'iphone',
	'imgfile':'',
	'commend': 'all',
	'ssid': 's5-e',
	'search_type': 'item',
	'sourceId': 'tb.index',
	'spm': 'a21bo.2017.201856-taobao-item.2',
	'ie': 'utf8',
	'initiative_id': 'tbindexz_20170306',
}

url = 'https://s.taobao.com/search?' + urlencode(params)
s = requests.Session()
response = s.get(url,headers=headers,verify=False).text
print(response)

　

任务2:爬取淘宝商品信息

 

from selenium import webdriver
# 通用选择
from selenium.webdriver.common.by import By

from selenium.webdriver.common.keys import Keys

from selenium.webdriver import ActionChains

from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# 超时
from selenium.common.exceptions import TimeoutException

from lxml import etree

import json
import random

import time

browser = webdriver.Chrome()

browser.set_window_size(1400, 900)

'''
爬到第19页的时候淘宝会弹窗,熟悉的滑动解锁= =,以后解决了这个滑动在继续吧
总体思路没问题
'''

def taobao_login():
	'''
		淘宝的滑动验证码过不去,使用第三方登录
	'''
	'''
	# 需要先登录
	login = browser.find_element(By.ID,'J_Quick2Static').click()
	username = browser.find_element(By.CSS_SELECTOR,'#TPL_username_1')
	username.send_keys('XXX')
	password = browser.find_element(By.CSS_SELECTOR,'#TPL_password_1')
	password.send_keys('XXX')
	button = browser.find_element(By.ID, 'J_SubmitStatic').click()
	ActionChains(browser).move_by_offset(random.randint(10, 60), random.randint(10, 60)).perform()
	# 判断大小
	input2 = browser.find_element(By.ID,'nc_1__scale_text')
	print(input2.size)
	# 滑动验证码
	action = ActionChains(browser)
	source = browser.find_element(By.ID,'nc_1_n1z')
	# 按住不放
	action.click_and_hold(source).perform()
	# 需要滑动的坐标
	action.move_by_offset(298,0)
	# 释放鼠标
	action.release().perform()
	'''


def weibo_login():
	'''
	账号密码输入后有滑动验证码,滑动成功也无法登录,考虑绕过去,采用第三方登录

	通过微博账号登录
	'''
	weibo_button = browser.find_element(By.CSS_SELECTOR, '.weibo-login')
	weibo_button.click()
	# 网速有点慢 需要先注册一个微博账号且绑定一个淘宝账号,真麻烦= =
	# 多登录几次,微博就会弹出当前网络超时，请稍后再试(600002),需要等一段时间才能进去
	time.sleep(2)
	username = browser.find_element(By.NAME, 'username')
	username.send_keys('账号')
	time.sleep(1)
	password = browser.find_element(By.NAME, 'password')
	password.send_keys('密码')
	browser.find_element(By.CSS_SELECTOR, '.W_btn_g').click()


def index_page(page):
	print('正在抓取第', page, '页')
	try:
		if page > 1:
			# 等待直到页码输入框出现
			input_box = wait.until(
				EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager div.form > input')))
			# 等待确定按钮可以被点击
			submit = wait.until(
				EC.element_to_be_clickable((By.CSS_SELECTOR, '#mainsrp-pager div.form > span.btn.J_Submit')))
			input_box.clear()
			input_box.send_keys(page)
			submit.click()
			# time.sleep(2)
		# 等待直到跳转的页码等于高亮显示的页码,说明跳转成功
		wait.until(
			EC.text_to_be_present_in_element((By.CSS_SELECTOR, '#mainsrp-pager li.item.active > span'), str(page)))
		# 等待直到所有商品加载出来
		wait.until(
			EC.presence_of_element_located((By.CSS_SELECTOR, '.m-itemlist .items .item')))
		# 爬取详细信息
		# time.sleep(3)
		get_products()
	except TimeoutException:
		# 超时就重试一次
		index_page(page)


def get_products():
	html = etree.HTML(browser.page_source)
	items = html.xpath('//div[@class="m-itemlist"]//div[@class="items"]/div')
	for item in items:
		product = {}
		product['image'] = item.xpath('.//img/@data-src')
		product['price'] = item.xpath('.//strong/text()')
		product['title'] = item.xpath('.//img[@class="J_ItemPic img"]/@alt')
		product['shop'] = item.xpath('.//a[@class="shopname J_MouseEneterLeave J_ShopInfo"]/span[2]/text()')
		product['location'] = item.xpath('.//div[@class="location"]/text()')
		print(product)
		result = json.dumps(product, ensure_ascii=False) + ',\n'
		with open('product.json', 'ab') as f:
			f.write(result.encode('utf-8'))


# print(browser.get_cookies())
# print(browser.page_source)


wait = WebDriverWait(browser, 10)
MAX_PACE = 100


def main():
	try:
		browser.get('http://www.taobao.com')
		print(browser.window_handles)
		input = browser.find_element(By.ID, 'q')
		# 搜索的信息
		input.send_keys('iphone')
		input.send_keys(Keys.ENTER)
		browser.find_element(By.ID, 'J_Quick2Static').click()
		time.sleep(1)
		# 处理登录
		weibo_login()
	except:
		main()
	for i in range(1, MAX_PACE + 1):
		index_page(i)


if __name__ == '__main__':
	main()

　　

总结:

　　1.学会利用cookie绕过验证码

　　2.学会从第三方进入需要爬取的网站

　　3.淘宝的滑动解锁(真的麻烦)

　　4.通过selenium模拟点击,爬取网站,虽然只爬了20页,但是思路没问题

淘宝必杀:滑动三连

 

　　

 

小知识:

import json
a = {"name":"123","age":123}
text = json.dumps(a,ensure_ascii=False) + ",\n"

with open('1234.json','wb') as  f:
	f.write(text.encode('utf-8'))