c#爬取筆趣閣小說（附源碼）

我的代碼小白復制也能實現效果

目標網站：https://www.biqugeu.net/

進入網站后我們搜索小說名稱

 

打開f12可以看到第一個調用的接口很明顯是我們剛剛搜索的接口，然后我們打開當前頁面的源代碼

 

 可以看出源代碼的這個地方對應的是頁面查詢到的第一個，我們就爬查詢到的第一個，點擊href鏈接進入下一個頁面

 

 然后我們就進入到了上面圖片的頁面，很明顯這些就是我們要爬取的小說章節，我們先打開頁面源代碼，如下

 

 到這，我們就應該找小說的第一章節開始爬取，第一個dt里面很明顯對應着頁面的最新章節，我們得從第二個dt開始第一個dd開始爬取，我們先點擊鏈接進入

 

 這個頁面應該就是我們真正要爬取的小說內容了，我們打開頁面源代碼

 

 

到這我們可以看到我們需要爬取的東西都在這，小說題目，章節名稱，章節內容以及下一章節的鏈接，然后我們開始寫代碼進行爬取。
以下是效果圖以及手機看的效果

 

 

 

 

 

 

 以下為完整代碼

using System;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;

namespace ConsoleApp3
{
    class Program
    {
        static void Main(string[] args)
        {
            string searchbook = "https://www.biqugeu.net/searchbook.php?keyword=<<bookname>>";
            string searchurl = null;
            string searchcontent = null;
            string baseurl = "https://www.biqugeu.net/";
            string nextChapter = null;
            string html = null;
            string bookname = null;
            string bookTitle = null;
            string ChapterContent;
            string regex1 = "<h1>(?<bookname>.*?)</h1>";
            string regex2 = "<a href=\"/.*?\" target=\"_top\" class=\"pre\">上一章</a> &larr; <a href=\"/.*?/\" target=\"_top\" title=\"\" class=\"back\">章節列表</a> &rarr; <a href=\"(?<nextChapter>.*?)\" target=\"_top\" class=\"next\"";
            string regex3 = "booktitle = \"(?<booktitle>.*?)\";";
            string regex4 = "(?<data>.*?)<br/><br/>";
            string regex5 = "<div class=\"image\">\\s*<a href=\"/(?<bookurl>.*?)\"";
            string regex6 = "<dt>.*?</dt><dd><ahref=\"/(?<bookfirst>.*?)\">.*?</a></dd>";

            Console.WriteLine("請輸入需要爬取的小說！");

            string novelName = Console.ReadLine();
            try
            {
                searchurl = searchbook.Replace("<<bookname>>", novelName);
                HttpWebRequest req1 = (HttpWebRequest)WebRequest.Create(searchurl);
                req1.Method = "GET";
                req1.Accept = "text/html";
                req1.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36";
                HttpWebResponse res1 = (HttpWebResponse)req1.GetResponse();
                using (StreamReader reader = new StreamReader(res1.GetResponseStream()))
                {
                    html = reader.ReadToEnd();
                    if (!string.IsNullOrEmpty(html))
                    {
                        //Console.WriteLine(html);
                        html = html.Replace("\n", "").Replace("\t", "").Replace("\r", "");
                        searchcontent = Regex.Match(html, regex5).Groups["bookurl"].ToString();
                        if (searchcontent == "")
                        {
                            Console.WriteLine("沒有找到該小說！");
                        }
                        searchurl = baseurl + searchcontent;
                    }
                }
            }
            catch (WebException we)
            {
                Console.WriteLine(we.Message);
            }
            try
            {
                HttpWebRequest req1 = (HttpWebRequest)WebRequest.Create(searchurl);
                req1.Method = "GET";
                req1.Accept = "text/html";
                req1.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36";
                HttpWebResponse res1 = (HttpWebResponse)req1.GetResponse();
                using (StreamReader reader = new StreamReader(res1.GetResponseStream()))
                {
                    html = reader.ReadToEnd();
                    if (!string.IsNullOrEmpty(html))
                    {
                        //Console.WriteLine(html);
                        html = html.Replace("\n", "").Replace("\t", "").Replace("\r", "").Replace(" ","");
                        searchcontent = Regex.Matches(html, regex6)[1].Groups["bookfirst"].ToString();
                        searchurl = baseurl + searchcontent;
                    }
                }
            }
            catch (Exception)
            {

                throw;
            }

            do
            {
            restart: try
                {
                    HttpWebRequest req = (HttpWebRequest)WebRequest.Create(searchurl);
                    req.Method = "GET";
                    req.Accept = "text/html";
                    req.AllowAutoRedirect = true;
                    req.Headers.Add("Encoding", Encoding.UTF8.ToString());
                    req.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)";
                    HttpWebResponse res = (HttpWebResponse)req.GetResponse();
                    using (StreamReader reader = new StreamReader(res.GetResponseStream()))
                    {
                        html = reader.ReadToEnd();
                        if (!string.IsNullOrEmpty(html))
                        {
                            ChapterContent = "";
                            //獲取下一章
                            nextChapter = Regex.Match(html, regex2).Groups["nextChapter"].ToString();
                            searchurl = baseurl + nextChapter;

                            //獲取章節名
                            bookname = Regex.Match(html, regex1).Groups["bookname"].ToString();
                            ChapterContent += "\r\n";
                            ChapterContent += bookname;
                            ChapterContent += "\r\n";
                            //獲取書名
                            bookTitle = Regex.Match(html, regex3).Groups["booktitle"].ToString();
                            //獲取內容
                            MatchCollection match = Regex.Matches(html, regex4);
                            foreach (Match item in match)
                            {
                                string book = Regex.Match(item.Value, regex4).Groups["data"].ToString().Trim();
                                ChapterContent += book;
                            }
                            Console.WriteLine(bookname + "-------下載完畢！");
                            AddBookToTXT(ChapterContent, bookTitle);
                        }

                    }
                }
                catch (WebException we)
                {
                    //Console.WriteLine(we.Message);
                    Console.WriteLine("遠程主機強迫關閉了一個現有的連接,重新爬取當前章節。。。");
                    goto restart;
                }
            } while (nextChapter.Contains("html"));//當下一章鏈接沒有跳轉時結束
        }

        /// <summary>
        /// 將內容保存到txt文件
        /// </summary>
        /// <param name="logstring">內容</param>
        /// <param name="pathName">書名</param>
        public static void AddBookToTXT(string logstring, string pathName)
        {
            string path = AppDomain.CurrentDomain.BaseDirectory + pathName + ".txt";
            if (!System.IO.File.Exists(path))
            {
                FileStream stream = System.IO.File.Create(path);
                stream.Close();
                stream.Dispose();
            }
            using (StreamWriter writer = new StreamWriter(path, true))
            {
                writer.WriteLine(logstring);
            }
        }
    }
}