C#寫一個簡單爬蟲

最近研究C#的爬蟲寫法，搞了半天，才在網上很多的寫法中整理出了一個簡單的demo（本人菜鳥，大神勿噴）。一是為了自己記錄一下以免日后用到，二是為了供需要朋友參考。

廢話不多說，上代碼

using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace Crawler
{
    class Program
    {
        static void Main(string[] args)
        {

            //WebProxy proxyObject = new WebProxy(IP, PORT);//這里我是用的代理。

            //向指定地址發送請求
            HttpWebRequest HttpWReq = (HttpWebRequest)WebRequest.Create("http://news.baidu.com/");
            //HttpWReq.Proxy = proxyObject;
            HttpWReq.Timeout = 10000;
            HttpWebResponse HttpWResp = (HttpWebResponse)HttpWReq.GetResponse();
            StreamReader sr = new StreamReader(HttpWResp.GetResponseStream(), Encoding.GetEncoding("UTF-8"));
            HtmlDocument doc = new HtmlDocument();
            doc.Load(sr);
            HtmlNodeCollection ulNodes = doc.DocumentNode.SelectSingleNode("//div[@id='pane-news']").SelectNodes("ul");
            if (ulNodes != null && ulNodes.Count > 0)
            {
                for (int i = 0; i < ulNodes.Count; i++)
                {
                    HtmlNodeCollection liNodes = ulNodes[i].SelectNodes("li");
                    for (int j = 0; j < liNodes.Count; j++)
                    {
                        string title = liNodes[j].SelectSingleNode("a").InnerHtml.Trim();
                        string href = liNodes[j].SelectSingleNode("a").GetAttributeValue("href", "").Trim();
                        Console.WriteLine("新聞標題：" + title + ",鏈接：" + href);
                    }
                }
            }
            Console.ReadLine();
            sr.Close();
            HttpWResp.Close();
            HttpWReq.Abort();
        }
    }
}

其中解析html的寫法用到了XPath的語法，大家可以自行百度下，比較簡單。