Python-爬蟲-針對有frame框架的頁面


有的頁面會使用frame 框架,使用Selenium + PhantomJS 后並不會加載iframe 框架中的網頁內容。iframe 框架相當於在頁面中又加載了一個頁面,需要使用Selenium 的 switch_to.frame() 方法加載(官網給的方法是switch_to_frame(),但是IDE提醒使用前面的方法替代該方法)。

比如:

        driver.switch_to.frame('g_iframe')

        html = driver.page_source

然后結合BeautifulSoup獲取網頁中信息。

這次我們爬取http://music.163.com/#/artist/album?id=101988&limit=120&offset=0頁面中的專輯信息,比如,圖片、網址及專輯名字。

"""
http://music.163.com/#/artist/album?id=101988&limit=120&offset=0
爬取上述網址中的專輯信息
"""

from selenium import webdriver
from urllib.request import urlretrieve
import os
from bs4 import BeautifulSoup

class DownloadInfo():

    def __init__(self):
        self.url = 'http://music.163.com/#/artist/album?id=101988&limit=120&offset=0'
        self.basePath = os.path.dirname(__file__)

    def makedir(self, name):
        path = os.path.join(self.basePath, name)
        isExist = os.path.exists(path)
        if not isExist:
            os.makedirs(path)
            print('The file is created now.')
        else:
            print('The file existed.')
        #切換到該目錄下
        os.chdir(path)
        return path

    def connect(self, url):
        driver = webdriver.PhantomJS()
        driver.get(url)
        print('success')
        return driver

    def getFileNames(self, path):
        pic_names = os.listdir(path)
        return pic_names

    def getInfo(self):
        driver = self.connect(self.url)
        driver.switch_to.frame('g_iframe')
        path = self.makedir('Infos')
        pic_names = self.getFileNames(path)
        imgs = driver.find_elements_by_xpath("//div[@class='u-cover u-cover-alb3']/img")
        titles = driver.find_elements_by_xpath("//li/p[@class='dec dec-1 f-thide2 f-pre']/a")
        dates = driver.find_elements_by_xpath("//span[@class='s-fc3']")
        count = 0
        for img in imgs:
            album_name = titles[count].text
            count += 1
            photo_name = album_name.replace('/', '') + '.jpg'
            print(photo_name)
            if photo_name in pic_names:
                print('圖片已下載。')
            else:
                urlretrieve(img.get_attribute('src'), photo_name)
        for title in titles:
            print(title.text)
        for date in dates:
            print(date.text)


"""
    def getInfo(self):
        driver = self.connect(self.url)
        driver.switch_to.frame('g_iframe')
        html = driver.page_source
        path = self.makedir('Infos')
        pic_names  = self.getFileNames(path)
        all_li = BeautifulSoup(html, 'lxml').find(id='m-song-module').find_all('li')
        for li in all_li:
            album_img = li.find('img')['src']
            album_name = li.find('p', class_='dec')['title']
            album_date = li.find('span', class_='s-fc3').get_text()
            print(album_img)
            print(album_name)
            print(album_date)
            photo_name = album_name.replace('/', '') + '.jpg'
            if photo_name in pic_names:
                print('圖片已下載。')
            else:
                urlretrieve(album_img, photo_name)
"""



if __name__ == '__main__':
    obj = DownloadInfo()
    obj.getInfo()
View Code

 


免責聲明!

本站轉載的文章為個人學習借鑒使用,本站對版權不負任何法律責任。如果侵犯了您的隱私權益,請聯系本站郵箱yoyou2525@163.com刪除。



 
粵ICP備18138465號   © 2018-2025 CODEPRJ.COM