from bs4 import BeautifulSoup
import requests
import time
import re
url = 'http://search.qyer.com/hotel/89580_4.html'
urls = ['http://search.qyer.com/hotel/89580_{}.html'.format(str(i)) for i in range(1,10)] # 最多157頁
infos = []
# print(urls)
# 批量爬取數據
def getAUrl(urls):
data_number = 0
for url in urls:
getAttractions(url)
print('--------------{}-----------------'.format(len(infos)),sep='\n')
# 爬取當頁面數據
def getAttractions(url,data = None):
web_data = requests.get(url)
time.sleep(2)
soup = BeautifulSoup(web_data.text,'lxml')
# print(soup)
hotel_names = soup.select('ul.shHotelList.clearfix > li > h2 > a')
hotel_images = soup.select('span[class="pic"] > a > img')
hotel_points = soup.select('span[class="points"]')
hotel_introduces = soup.select('p[class="comment"]')
hotel_prices = soup.select('p[class="seemore"] > span > em')
if data == None:
for name,image,point,introduce,price in \
zip(hotel_names,hotel_images,hotel_points,hotel_introduces,hotel_prices):
data = {
'name':name.get_text().replace('\r\n','').strip(),
'image':image.get('src'),
'point':re.findall(r'-?\d+\.?\d*e?-?\d*?', point.get_text())[0],
'introduce':introduce.get_text().replace('\r\n','').strip(),
'price':int(price.get_text())
}
# print(data)
infos.append(data)
# 根據價格從高到低進行排序
def getInfosByPrice(infos = infos):
infos = sorted(infos, key=lambda info: info['price'], reverse=True)
for info in infos:
print(info['price'], info['name'])
# getAttractions(url)
爬取的網站鏈接
遇到的問題及解決辦法
1.【轉載】Python: 去掉字符串開頭、結尾或者中間不想要的字符
①Strip()方法用於刪除開始或結尾的字符。lstrip()|rstirp()分別從左右執行刪除操作。默認情況下會刪除空白或者換行符，也可以指定其他字符。
②如果想處理中間的空格，需要求助其他技術 ，比如replace(),或者正則表達式
③strip()和其他迭代結合，從文件中讀取多行數據，使用生成器表達式
④更高階的strip
可能需要使用translate()方法
2. 【轉載】Python:object of type 'Response' has no len()，如何解決？