1、最容易出現的問題是爬取到的url大多為相對路徑，如果直接將爬取到的url進行二次爬取就會出現以下報錯：
raise ValueError('Missing scheme in request url: %s' % self._url)，該錯誤的意思是request的url為無效鏈接。
解決方法：將相對路徑轉換為絕對路徑
1 # -*- coding: utf-8 -*- 2 import scrapy 3 from pyquery import PyQuery 4 from ..items import City10Item 5 from scrapy.http import Request 6 from scrapy.utils.response import get_base_url 7 from urllib.parse import urljoin 8 9 10 class CitySpiderTest1Spider(scrapy.Spider): 11 name = 'city_spider_test1' 12 allowed_domains = ['58.com'] 13 start_urls = ['https://cq.58.com/chuzu/'] 14 15 def parse(self, response): 16 jpy=PyQuery(response.text) 17 li=jpy('body > div.mainbox > div > div.content > div.listBox > ul > li').items() 18 for i in li: 19 item=City10Item() 20 item['name']=i('div.des > h2 > a').text() 21 item['url']=i('div.des > h2 > a').attr('href') #爬取url，此時為相對路徑 22 item['price']=i('div.listliright > div.money > b').text() 23 base_url = get_base_url(response) #獲取相對路徑前級地址 24 abs_url=urljoin(base_url,item['url']) #轉換為絕對路徑 25 26 27 if item['url']: 28 yield Request(abs_url, 29 callback=self.detail_parse, #回調詳情頁函數 30 meta={'item':item}, #將參數傳遞給meta 31 priority=10, 32 dont_filter=True #強制不過濾 33 34 ) 35 36 def detail_parse(self,response): 37 jpy = PyQuery(response.text) 38 item = response.meta['item'] #接收item 39 item['introduce_item'] = jpy('body > div.main-wrap > div.house-detail-desc > div.main-detail-info.fl > div.house-word-introduce.f16.c_555 > ul > li:nth-child(1) > span.a2').text() #提取房屋亮點 40 item['address'] = jpy('body > div.main-wrap > div.house-basic-info > div.house-basic-right.fr > div.house-basic-desc > div.house-desc-item.fl.c_333 > ul > li:nth-child(6) > span.dz').text() #房屋詳情地址 41 item['phone_number'] = jpy('body > div.main-wrap > div.house-basic-info > div.house-basic-right.fr > div.house-fraud-tip > div.house-chat-phone > span').text() #電話號碼 42 yield item