亞馬遜的網址構造很簡單，幾乎算是靜態的網頁，花費3小時完美收工，不要在意細節！
在python3下利用xpath就可以完美解決
xpath的使用方法請見：
python之lxml(xpath)
入口圖界面為：
抓取操作為：
抓取的效果圖如下：
圖片：
excel：
1 ''' 2 .======. 3 | INRI | 4 | | 5 | | 6 .========' '========. 7 | _ xxxx _ | 8 | /_;-.__ / _\ _.-;_\ | 9 | `-._`'`_/'`.-' | 10 '========.`\ /`========' 11 | | / | 12 |/-.( | 13 |\_._\ | 14 | \ \`;| 15 | > |/| 16 | / // | 17 | |// | 18 | \(\ | 19 | `` | 20 | | 21 | | 22 | | 23 | | 24 .======. 25 …………………………………………………………………………………… 26 27 ！！！！！ 28 \\ - - // 29 (-● ●-) 30 \ (_) / 31 \ u / 32 ┏oOOo-━━━━━━━━┓ 33 ┃ ┃ 34 ┃ 耶穌保佑！ ┃ 35 ┃ 永無BUG！！！┃ 36 ┃ ┃ 37 ┗━━━━━━━━-oOOo┛ 38 39 …………………………………………………………………………………… 40 41 _oo0oo_ 42 088888880 43 88" . "88 44 (| -_- |) 45 0\ = /0 46 ___/'---'\___ 47 .' \\\\| |// '. 48 / \\\\||| : |||// \\ 49 /_ ||||| -:- |||||- \\ 50 | | \\\\\\ - /// | | 51 | \_| ''\---/'' |_/ | 52 \ .-\__ '-' __/-. / 53 ___'. .' /--.--\ '. .'___ 54 ."" '< '.___\_<|>_/___.' >' "". 55 | | : '- \'.;'\ _ /';.'/ - ' : | | 56 \ \ '_. \_ __\ /__ _/ .-' / / 57 ====='-.____'.___ \_____/___.-'____.-'===== 58 '=---=' 59 60 61 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 62 佛祖保佑 永無BUG 63 64 65 66 67 ┌─┐ ┌─┐ 68 ┌──┘ ┴───────┘ ┴──┐ 69 │ │ 70 │ ─── │ 71 │ ─┬┘ └┬─ │ 72 │ │ 73 │ ─┴─ │ 74 │ │ 75 └───┐ ┌───┘ 76 │ │ 77 │ │ 78 │ │ 79 │ └──────────────┐ 80 │ │ 81 │ ├─┐ 82 │ ┌─┘ 83 │ │ 84 └─┐ ┐ ┌───────┬──┐ ┌──┘ 85 │ ─┤ ─┤ │ ─┤ ─┤ 86 └──┴──┘ └──┴──┘ 87 神獸保佑 88 代碼無BUG! 89 ''' 90 # !/usr/bin/python3.4 91 # -*- coding: utf-8 -*- 92 93 # 前排燒香 94 # 永無BUG 95 96 import requests 97 import time 98 import random 99 import xlsxwriter 100 from lxml import etree 101 import urllib.parse 102 import urllib.request 103 104 def geturl(url): 105 # 制作頭部 106 header = { 107 'User-Agent': 'Mozilla/5.0 (iPad; U; CPU OS 4_3_4 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8K2 Safari/6533.18.5', 108 'Referer': 'https://www.amazon.cn/', 109 'Host': 'www.amazon.cn', 110 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 111 'Accept-Encoding': 'gzip, deflate, br', 112 'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3', 113 'Connection': 'keep-alive' 114 } 115 # get參數 116 res = requests.get(url=url, headers=header) 117 # ('UTF-8')('unicode_escape')('gbk','ignore') 118 resdata = res.content 119 return resdata 120 121 def getimg(url): 122 # 制作頭部 123 header = { 124 'User-Agent': 'Mozilla/5.0 (iPad; U; CPU OS 4_3_4 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8K2 Safari/6533.18.5', 125 'Referer': 'https://www.amazon.cn/', 126 'Host': 'www.amazon.cn', 127 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 128 'Accept-Encoding': 'gzip, deflate, br', 129 'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3', 130 'Connection': 'keep-alive' 131 } 132 # get參數 133 res = requests.get(url=url, headers=header,stream=True) 134 # ('UTF-8')('unicode_escape')('gbk','ignore') 135 resdata = res.iter_content(chunk_size=1024) 136 for chunk in resdata: 137 if chunk: 138 return chunk 139 140 def begin(): 141 taoyanbai = ''' 142 ----------------------------------------- 143 | 歡迎使用亞馬遜爬取系統 | 144 | 時間：2016年9月20日 | 145 | 出品：TTyb | 146 | 微信/QQ：420439007 | 147 ----------------------------------------- 148 ''' 149 print(taoyanbai) 150 151 152 def timetochina(longtime, formats='{}天{}小時{}分鍾{}秒'): 153 day = 0 154 hour = 0 155 minutue = 0 156 second = 0 157 try: 158 if longtime > 60: 159 second = longtime % 60 160 minutue = longtime // 60 161 else: 162 second = longtime 163 if minutue > 60: 164 hour = minutue // 60 165 minutue = minutue % 60 166 if hour > 24: 167 day = hour // 24 168 hour = hour % 24 169 return formats.format(day, hour, minutue, second) 170 except: 171 raise Exception('時間非法') 172 173 174 if __name__ == '__main__': 175 176 begin() 177 178 keyword = input("請輸入關鍵詞：") 179 try: 180 sort = int(input("相關度排序請按0，人氣排序請按1，上架時間排序請按2，價格低到高排序請按3，價格高到低請按4，用戶評分排序請按5（默認相關度排序）：")) 181 if sort > 5 or sort <= 0: 182 sort = "" 183 elif sort == 1: 184 sort = "popularity-rank" 185 elif sort == 2: 186 sort = "date-desc-rank" 187 elif sort == 3: 188 sort = "price-asc-rank" 189 elif sort == 4: 190 sort = "price-desc-rank" 191 elif sort == 5: 192 sort = "review-rank" 193 except: 194 sort = "" 195 try: 196 pages = int(input("請輸入抓取頁數（默認5頁）：")) 197 except: 198 pages = 5 199 200 a = time.clock() 201 202 # 轉成字符串 203 # %y 兩位數的年份表示（00 - 99） 204 # %Y 四位數的年份表示（000 - 9999） 205 # %m 月份（01 - 12） 206 # %d 月內中的一天（0 - 31） 207 # %H 24小時制小時數（0 - 23） 208 # %I 12小時制小時數（01 - 12） 209 # %M 分鍾數（00 = 59） 210 # %S 秒（00 - 59） 211 today = time.strftime('%Y%m%d%H%M', time.localtime()) 212 # 創建一個Excel文件 213 workbook = xlsxwriter.Workbook('../excel/' + today + '.xlsx') 214 # 創建一個工作表 215 worksheet = workbook.add_worksheet() 216 217 # 第一行參數 218 first = ['商品名稱', '品牌', '詳情頁網址', '原價格', '星級', '圖片','圖片網址'] 219 220 # 寫入excel計數行 221 count = 1 222 223 # 下載圖片計數 224 num = 0 225 226 # 構造時間戳 227 nowtime = int(time.time()) 228 229 for page in range(0,pages): 230 231 urldata = { 232 'page':page, 233 'sort':sort, 234 'keywords':keyword, 235 'ie':'UTF-8', 236 'qid':str(nowtime) 237 } 238 urldata = urllib.parse.urlencode(urldata) 239 url = "https://www.amazon.cn/s/ref=nb_sb_noss_1?__mk_zh_CN=亞馬遜網站&" + urldata 240 241 html = geturl(url).decode('Utf-8', 'ignore') 242 #file = open("../data/html.txt","wb") 243 #file.write(html) 244 #file.close() 245 246 #file = open("../data/html.txt","rb") 247 #html = file.read().decode('Utf-8', 'ignore') 248 #print(html) 249 250 # xpath解析需要的東西 251 contents = etree.HTML(html) 252 253 # 找到商品名稱 254 titles = contents.xpath('//a[@class="a-link-normal s-access-detail-page a-text-normal"]/@title') 255 arr_title = [] 256 for title in titles: 257 arr_title.append(title) 258 259 # 找到品牌 260 brands = contents.xpath('//div[@class="a-row a-spacing-mini"][1]/div/span/text()') 261 arr_brand = [] 262 for brand in brands: 263 if "更多購買選擇" in brand: 264 pass 265 else: 266 arr_brand.append(brand) 267 268 # 找到詳情頁網址 269 detailurls = contents.xpath('//a[@class="a-link-normal s-access-detail-page a-text-normal"]/@href') 270 arr_detailurl = [] 271 for detailurl in detailurls: 272 arr_detailurl.append(urllib.parse.unquote(detailurl)) 273 #print(detailurl) 274 #print(len(arr_detailurl)) 275 276 # 得到原價格 277 # 這里是忽略了新品價格、非新品價格 278 prices = contents.xpath('//div[@class="a-row a-spacing-none"][1]/a/span[1]/text()') 279 arr_price = [] 280 for price in prices: 281 arr_price.append(price) 282 283 # 得到星級 284 grades = contents.xpath('//span[@class="a-icon-alt"]/text()') 285 arr_grade = [] 286 for grade in grades: 287 if "平均" in grade: 288 arr_grade.append(grade) 289 #print(grade) 290 else: 291 pass 292 if arr_grade: 293 arr_grade.pop() 294 #print(len(arr_grades)) 295 296 # 得到圖片 297 imgurls = contents.xpath('//a[@class="a-link-normal a-text-normal"]/img/@src') 298 arr_img = [] 299 300 for imgurl in imgurls: 301 file = open("../jpg/" + str(num) + ".jpg","wb") 302 pic = urllib.request.urlopen(imgurl) 303 file.write(pic.read()) 304 file.close() 305 # 每一次下載都暫停1-3秒 306 imgtime = random.randint(1, 3) 307 print("下載圖片暫停" + str(imgtime) + "秒") 308 time.sleep(imgtime) 309 arr_img.append(imgurl) 310 num = num + 1 311 #print(imgurl) 312 #print(len(arr_img)) 313 314 # 寫入excel 315 # 寫入第一行 316 for i in range(0, len(first)): 317 worksheet.write(0, i, first[i]) 318 319 # 寫入其他數據 320 for j in range(0,len(arr_title)): 321 worksheet.write(count,0,arr_title[j]) 322 worksheet.write(count, 1, arr_brand[j]) 323 worksheet.write(count, 2, arr_detailurl[j]) 324 try: 325 worksheet.write(count, 3, arr_price[j]) 326 except Exception as err: 327 print(err) 328 worksheet.write(count, 3, "") 329 worksheet.write(count, 4, arr_grade[j]) 330 worksheet.insert_image(count, 5, "../jpg/" + str(count - 1) + ".jpg") 331 worksheet.write(count, 6, arr_img[j]) 332 count = count + 1 333 334 # 每一次下載都暫停5-10秒 335 loadtime = random.randint(5, 10) 336 print("抓取網頁暫停" + str(loadtime) + "秒") 337 time.sleep(loadtime) 338 339 workbook.close() 340 b = time.clock() 341 print('運行時間：' + timetochina(b - a)) 342 input('請關閉窗口')