C#簡單的爬蟲，爬博客園首頁文章標題

運行效果如圖：

 

 

 代碼如下：

using System;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;

namespace ConsoleApplication2
{
    class Program
    {
        static void Main(string[] args)
        {
            string host = "https://www.cnblogs.com/?";  //url后面必須加一個?不然程序請求都是404，不知道是什么原因
            int num = 0; //統計當前為第幾個文章的標題
            int pagSize = 100; //爬取的最大頁數 10表示爬首頁前10頁的標題

            //標題標簽樣例：<a class="titlelnk" href="https://www.xxxxx.html" target="_blank">【設計模式】簡單工廠模式 Simple Factory Pattern</a>
            string pater = "<a class=\"titlelnk\" href=\"(.*?)\" target=\"_blank\">(.*?)</a>";  //()為C#要捕捉的內容,括號里面的".*?"表示匹配任意內容（因為url的地址是不確定的）
            Regex regex = new Regex(pater);

            for (int i = 1; i < pagSize; i++)
            {
                //首頁完整鏈接為https://www.cnblogs.com/#p2  #p后面的數字代表當前頁
                string url = host + "#p" + i;
                var html = GetHtmlString(url);
                if (!string.IsNullOrEmpty(html))
                {
                    //標題標簽<a class="titlelnk" href="https://www.xxxxx.html" target="_blank">【設計模式】簡單工廠模式 Simple Factory Pattern</a>
                    //正則匹配標題的標簽，再提取其中的名稱和url
                    foreach (Match ma in regex.Matches(html))
                    {
                        Match match = Regex.Match(ma.Value, pater);
                        string title = match.Groups[2].Value;
                        string titlelnk = match.Groups[1].Value;
                        Console.WriteLine($"-------------------------------第{ ++num }個標題------------------------");
                        Console.WriteLine(title + "Url:" + titlelnk);
                        Console.WriteLine("--------------------------------------------------------------------------");
                        File.AppendAllText(@"d:\cnblog.txt", title + "     " + titlelnk + "\r\n");
                    }
                }
            }
            Console.WriteLine("結束一共爬了" + num + "個標題");
            Console.ReadKey();
        }

        /// <summary>
        /// 請求url
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetHtmlString(string url)
        {
            try
            {
                WebRequest request = WebRequest.Create(url);
                Stream stream = request.GetResponse().GetResponseStream();
                request.Timeout = 3000;
                using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                {
                    return reader.ReadToEnd();
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
                return null;
            }
        }

    }
}