Selenium FF WebDriver 遍歷所有鏈接（另類爬蟲）

請看這個頁面，我想要找到某個公告的內容，必須一個一個打開鏈接，尼瑪好多啊。

於是，我機智的使用selenium打開每一個鏈接，然后把公告內容寫入txt

那需要做一下步奏

1.依次打開一個公告

2.切換focus到新窗口，找到公告內容，寫到txt

3.關閉該窗口

4.切換到主窗口

5.當前頁面遍歷完，點擊下一頁

6.重復步奏1

 

由於下一頁是一個很好用的flag，就可以當做循環條件，因為最后一頁沒有下一頁的element

接下來要找到相關的的xpath

列表數目： count(//tr/td/a[starts-with(@href,'article_show.asp?ID=') and @title!='' ])
列表：      //tr/td/a[starts-with(@href,'article_show.asp?ID=') and @title!='' ]
下一頁：   //div/a[text()='下一頁']

selenium WebDriver測試網頁時,點擊target=”_blank”的鏈接，在打開新頁面，切換到新窗口的

這要使用

String currentWindow = driver.getWindowHandle();//獲取當前窗口句柄
Set<String> handles = driver.getWindowHandles();//獲取所有窗口句

WebDriver window = driver.switchTo().window(it.next());//切換到新窗口

driver.switchTo().window(currentWindow);//回到原來頁面

driver=driver.switchTo().window(driver.getWindowHandle()); //把下一頁變成當前driver

currentWindow = driver.getWindowHandle();
            //get all windows
            Set<String> handles= driver.getWindowHandles();
            for (String s : handles)
            {
                //current page is don't close
                if (s.equals(currentWindow))
                    continue;
                else 
                {
                    window =driver.switchTo().window(s);
                                }
                             window .close() ;
                        }
driver.switchTo().window(currentWindow);

具體代碼

 

package com.packt.webdriver.chapter3;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;

public class TraversalAllLinks {
    private static String currentWindow;

    public static void main(String[] args) {
    
 
        WebDriver driver=DriverFactory.getFirefoxDriver();
        driver.get("http://www.lhgtj.gov.cn/article.asp?ClassID=86&page=1");
    
        driver.manage().window().maximize();
        driver.manage().timeouts().implicitlyWait(60, TimeUnit.SECONDS);
        driver.manage().timeouts().pageLoadTimeout(60, TimeUnit.SECONDS);
        WebElement nextPage=driver.findElement(By.xpath("//tr/td/a[@title='下一頁']"));
        while(nextPage.isDisplayed())
        {    

        List<WebElement> links=driver.findElements(By.xpath("//tr/td/a[starts-with(@href,'article_show.asp?ID=') and @title!='' ]"));
        
        for(WebElement link:links)
        {
            WebDriver window;
            System.out.println(link.getText());
            try {
                writeToTXT(link.getText());
            } catch (IOException e1) {
                // TODO Auto-generated catch block
                e1.printStackTrace();
            }
            link.click();
            currentWindow = driver.getWindowHandle();
            //get all windows
            Set<String> handles= driver.getWindowHandles();
            for (String s : handles)
            {
                //current page is don't close
                if (s.equals(currentWindow))
                    continue;
                else 
                {
                    window =driver.switchTo().window(s);
                    window.manage().window().maximize();
                    window.manage().timeouts().implicitlyWait(60, TimeUnit.SECONDS);
                    window.manage().timeouts().pageLoadTimeout(60, TimeUnit.SECONDS);
                    //get all tables
                    List<WebElement> tbs=window.findElements(By.xpath("//tbody/tr/td/p"));
                    for(WebElement tb:tbs)
                    {
                        System.out.println(tb.getText());
                        try {
                            writeToTXT(tb.getText()+"\n");
                        } catch (IOException e) {
                            // TODO Auto-generated catch block
                            e.printStackTrace();
                        }
                     
                    }
                    //close the table window
                    window .close() ;
                }
            //swich to current window
            driver.switchTo().window(currentWindow);
          }    
            
        }
        // click next page
        nextPage.click();
        //set next page to current page
        driver=driver.switchTo().window(driver.getWindowHandle());
        driver.manage().window().maximize();
        driver.manage().timeouts().implicitlyWait(60, TimeUnit.SECONDS);
        driver.manage().timeouts().pageLoadTimeout(60, TimeUnit.SECONDS);
        nextPage=driver.findElement(By.xpath("//tr/td/a[@title='下一頁']"));
        
        }

        
    }
    //write logs
    public static void  writeToTXT(String message) throws IOException
    {
        BufferedWriter bf = null;
        try {
            //set true ,avoid 
            bf = new BufferedWriter(new FileWriter("report.txt", true));
            bf.write(message);
            bf.flush();
           
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        finally
        {
             bf.close();
        }
    
    }

}

DriverFactory

public static WebDriver getFirefoxDriver()
    {
        try
        {
            WindowsUtils.tryToKillByName("firefox.exe");
        }
        catch(Exception e)
        {
            System.out.println("can not find firefox process");
        }
        File file=new File("d:\\firebug-2.0.4-fx.xpi");
        FirefoxProfile profile = new FirefoxProfile();
 
 
        try {
            profile.addExtension(file);
            profile.setPreference("extensions.firebug.currentVersion", "2.0.4");
            profile.setPreference("extensions.firebug.allPagesActivation", "on");
        } catch (IOException e3) {
            // TODO Auto-generated catch block
            e3.printStackTrace();
        }
     
        WebDriver driver = new FirefoxDriver(profile);
        return driver;
        
    }