C#简单的爬虫，爬博客园首页文章标题

运行效果如图：

 

 

 代码如下：

using System;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;

namespace ConsoleApplication2
{
    class Program
    {
        static void Main(string[] args)
        {
            string host = "https://www.cnblogs.com/?";  //url后面必须加一个?不然程序请求都是404，不知道是什么原因
            int num = 0; //统计当前为第几个文章的标题
            int pagSize = 100; //爬取的最大页数 10表示爬首页前10页的标题

            //标题标签样例：<a class="titlelnk" href="https://www.xxxxx.html" target="_blank">【设计模式】简单工厂模式 Simple Factory Pattern</a>
            string pater = "<a class=\"titlelnk\" href=\"(.*?)\" target=\"_blank\">(.*?)</a>";  //()为C#要捕捉的内容,括号里面的".*?"表示匹配任意内容（因为url的地址是不确定的）
            Regex regex = new Regex(pater);

            for (int i = 1; i < pagSize; i++)
            {
                //首页完整链接为https://www.cnblogs.com/#p2  #p后面的数字代表当前页
                string url = host + "#p" + i;
                var html = GetHtmlString(url);
                if (!string.IsNullOrEmpty(html))
                {
                    //标题标签<a class="titlelnk" href="https://www.xxxxx.html" target="_blank">【设计模式】简单工厂模式 Simple Factory Pattern</a>
                    //正则匹配标题的标签，再提取其中的名称和url
                    foreach (Match ma in regex.Matches(html))
                    {
                        Match match = Regex.Match(ma.Value, pater);
                        string title = match.Groups[2].Value;
                        string titlelnk = match.Groups[1].Value;
                        Console.WriteLine($"-------------------------------第{ ++num }个标题------------------------");
                        Console.WriteLine(title + "Url:" + titlelnk);
                        Console.WriteLine("--------------------------------------------------------------------------");
                        File.AppendAllText(@"d:\cnblog.txt", title + "     " + titlelnk + "\r\n");
                    }
                }
            }
            Console.WriteLine("结束一共爬了" + num + "个标题");
            Console.ReadKey();
        }

        /// <summary>
        /// 请求url
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetHtmlString(string url)
        {
            try
            {
                WebRequest request = WebRequest.Create(url);
                Stream stream = request.GetResponse().GetResponseStream();
                request.Timeout = 3000;
                using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                {
                    return reader.ReadToEnd();
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
                return null;
            }
        }

    }
}