用C#+Selenium+ChromeDriver 爬取網頁，完美模擬真實的用戶瀏覽行為

背景

 Selenium是一個用於Web應用程序測試的工具。Selenium測試直接運行在瀏覽器中，就像真正的用戶在操作一樣。而對於爬蟲來說，使用Selenium操控瀏覽器來爬取網上的數據那么肯定是爬蟲中的殺手武器。這里，我將介紹selenium + 谷歌瀏覽器的一般使用。

需求

在平常的爬蟲開發中，有時候網頁是一堆js堆起來的代碼，涉及很多異步計算，如果是普通的http 控制台請求，那么得到的源文件是一堆js ，需要自己在去組裝數據，很費力；但是采用Selenium+ChromeDriver可以達到所見即所得的完美效果。

實現方式

項目結構：為了方便使用，用的winform程序，附nuget包

 

 

 以下是form1.cs的代碼，這里就只放關鍵方法代碼了。需要安裝最新的chrome瀏覽器+代碼中使用的chromedriver是 v2.9.248315

   private void crawlingWebFunc()
        {
            SetText("\r\n開始嘗試...");
            List<testfold> surls = new List<testfold>();
            string path = System.Environment.CurrentDirectory + "\\圖片url\\";
            DirectoryInfo root = new DirectoryInfo(path);
            DirectoryInfo[] dics = root.GetDirectories();
            foreach (var itemdic in dics)
            {
                string txt = "";
                StreamReader sr = new StreamReader(itemdic.FullName + "\\data.txt");
                while (!sr.EndOfStream)
                {
                    string str = sr.ReadLine();
                    txt += str;// + "\n";
                }
                sr.Close();
                surls.Add(new testfold() { key = itemdic.FullName, picurl = txt });
            }

            ChromeDriverService service = ChromeDriverService.CreateDefaultService(System.Environment.CurrentDirectory);
            //  service.HideCommandPromptWindow = true;

            ChromeOptions options = new ChromeOptions();
            options.AddArguments("--test-type", "--ignore-certificate-errors");
            options.AddArgument("enable-automation");
            //   options.AddArgument("headless");
            //  options.AddArguments("--proxy-server=http://user:[e-mail]:8080");

            using (IWebDriver driver = new OpenQA.Selenium.Chrome.ChromeDriver(service, options, TimeSpan.FromSeconds(120)))
            {
                driver.Url = "https://www.1688.com/";
                Thread.Sleep(200);
                try
                {
                    int a = 1;
                    foreach (var itemsurls in surls)
                    {
                        SetText("\r\n第" + a.ToString() + "個");
                        driver.Navigate().GoToUrl(itemsurls.picurl);
                        //登錄
                        if (driver.Url.Contains("login.1688.com"))
                        {
                            SetText("\r\n需要登錄，開始嘗試...");
                            trylogin(driver); //嘗試登錄完成
                                              //再試試
                            driver.Navigate().GoToUrl("https://s.1688.com/youyuan/index.htm?tab=imageSearch&imageType=oss&imageAddress=cbuimgsearch/eWXC7XHHPN1607529600000&spm=");

                            if (driver.Url.Contains("login.1688.com"))
                            {
                                //沒辦法退出
                                SetText("\r\n退出，換ip重試...");
                                return;
                            }
                        }

                        //鼠標放上去的內容因為頁面自帶只能顯示一個的原因 沒辦法做到全部顯示 然后在下載 只能是其他方式下載
                        //  var elements = document.getElementsByClassName('hover-container');
                        //  Array.prototype.forEach.call(elements, function(element) {
                        //  element.style.display = "block";
                        //   console.log(element);
                        //  });

                        //   IJavaScriptExecutor js = (IJavaScriptExecutor)driver;

                        //    var sss = js.ExecuteScript(" var elements = document.getElementsByClassName('hover-container');  Array.prototype.forEach.call(elements, function(element) {  console.log(element); element.setAttribute(\"class\", \"測試title\");  element.style.display = \"block\";  console.log(element); });");

                        Thread.Sleep(500);
                        var responseModel = Write(itemsurls.key, driver.PageSource, Pagetypeenum.列表);
                        Thread.Sleep(500);
                        int i = 1;
                        foreach (var offer in responseModel?.data?.offerList ?? new List<OfferItemModel>())
                        {
                            driver.Navigate().GoToUrl(offer.information.detailUrl);
                            string responseDatadetail = driver.PageSource;
                            Write(itemsurls.key, driver.PageSource, Pagetypeenum.詳情);
                            SetText("\r\n第" + a.ToString() + "-" + i.ToString() + "個");
                            Thread.Sleep(500);
                            i++;
                        }
                    }
                }
                catch (Exception ex)
                {
                    CloseChromeDriver(driver);
                    throw;
                }
            }
        }

 #region 異常  退出chromedriver

        [DllImport("user32.dll", EntryPoint = "FindWindow")]
        private extern static IntPtr FindWindow(string lpClassName, string lpWindowName);

        [DllImport("user32.dll", EntryPoint = "SendMessage")]
        public static extern int SendMessage(IntPtr hWnd, int Msg, int wParam, int lParam);

        public const int SW_HIDE = 0;
        public const int SW_SHOW = 5;

        [DllImport("user32.dll", EntryPoint = "ShowWindow")]
        public static extern int ShowWindow(IntPtr hwnd, int nCmdShow);

        /// <summary>
        /// 獲取窗口句柄
        /// </summary>
        /// <returns></returns>
        public IntPtr GetWindowHandle()
        {
            string name = (Environment.CurrentDirectory + "\\chromedriver.exe");
            IntPtr hwd = FindWindow(null, name);
            return hwd;
        }

        /// <summary>
        /// 關閉chromedriver窗口
        /// </summary>
        public void CloseWindow()
        {
            try
            {
                IntPtr hwd = GetWindowHandle();
                SendMessage(hwd, 0x10, 0, 0);
            }
            catch { }
        }

        /// <summary>
        /// 退出chromedriver
        /// </summary>
        /// <param name="driver"></param>
        public void CloseChromeDriver(IWebDriver driver)
        {
            try
            {
                driver.Quit();
                driver.Dispose();
            }
            catch { }
            CloseWindow();
        }

        #endregion 異常  退出chromedriver

 

 

 

 

總結

說一下思路:

1.跳轉到指定的網頁driver.Navigate().GoToUrl

2.確定數據源，從driver.PageSource讀取數據

3.對html數據進行解析