利用正則表達式獲取博客園隨筆（一）

　　晚上起先和朋友們跑步去了，然后回來之后洗了個澡，打開VS新建項目發現都會彈出一個問題

然后就去找萬能的度娘了，http://bbs.csdn.net/topics/390514964?page=1#post-395015041

25樓真相，卸載掉那2個補丁就可以了，不過在卸載第一個補丁的時候你需要停止他指出的那個服務。

　　我當初剛開始接觸正則是去年公司主管讓我去學，然后發了個網址給我：http://www.cnblogs.com/ie421/archive/2008/07/23/1249896.html

看完后收益頗大，下面就開始正題。

　　之所以要獲取博客園的內容是因為博客園造就了我，而大家也都是在博客園里相識，所以我們就以博客園為例子。

　　下面上傳的這個是當初主管給我的一個類，大家可以參考參考，我今天的內容用到了里面的GetString()這個方法。在運行之前要引用System.Web

using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text;
using System.Web;

namespace CnblogsSearch
{
    public class HttpClient
    {
        #region fields
        private bool keepContext;
        private string defaultLanguage = "zh-CN";
        private Encoding defaultEncoding = Encoding.UTF8;
        private string accept = "*/*";
        private string userAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
        private HttpVerb verb = HttpVerb.GET;
        private HttpClientContext context;
        private readonly List<HttpUploadingFile> files = new List<HttpUploadingFile>();
        private readonly Dictionary<string, string> postingData = new Dictionary<string, string>();
        private string url;
        private WebHeaderCollection responseHeaders;
        private int startPoint;
        private int endPoint;
        public bool boundaryed;
        private string encodingType = "utf-8";
        private int timeOut = 10000;

        #endregion

        #region events
        public event EventHandler<StatusUpdateEventArgs> StatusUpdate;

        private void OnStatusUpdate(StatusUpdateEventArgs e)
        {
            EventHandler<StatusUpdateEventArgs> temp = StatusUpdate;

            if (temp != null)
                temp(this, e);
        }
        #endregion

        #region properties

        public string EncodingType
        {
            get
            {
                return encodingType;
            }
            set
            {
                encodingType = value;
            }
        }

        /// <summary>
        /// 是否啟用gzip壓縮傳輸
        /// </summary>
        public bool IsGzip { get; set; }

        /// <summary>
        /// 是否在數據流中編碼
        /// </summary>
        public bool encodeMemory { get; set; }
        /// <summary>
        /// 是否自動在不同的請求間保留Cookie, Referer
        /// </summary>
        public bool KeepContext
        {
            get { return keepContext; }
            set { keepContext = value; }
        }
        public CookieContainer cookie;
        /// <summary>
        /// 期望的回應的語言
        /// </summary>
        public string DefaultLanguage
        {
            get { return defaultLanguage; }
            set { defaultLanguage = value; }
        }

        /// <summary>
        /// GetString()如果不能從HTTP頭或Meta標簽中獲取編碼信息,則使用此編碼來獲取字符串
        /// </summary>
        public Encoding DefaultEncoding
        {
            get { return defaultEncoding; }
            set { defaultEncoding = value; }
        }

        public int TimeOut
        {
            get
            {
                return timeOut;
            }
            set
            {
                timeOut = value;
            }
        }
        /// <summary>
        /// 指示發出Get請求還是Post請求
        /// </summary>
        public HttpVerb Verb
        {
            get { return verb; }
            set { verb = value; }
        }

        /// <summary>
        /// 要上傳的文件.如果不為空則自動轉為Post請求
        /// </summary>
        public List<HttpUploadingFile> Files
        {
            get { return files; }
        }

        public List<RepeatPostData> repeatPostData
        {
            get;
            set;
        }

        /// <summary>
        /// 要發送的Form表單信息
        /// </summary>
        public Dictionary<string, string> PostingData
        {

            get { return postingData; }
        }

        /// <summary>
        /// 獲取或設置請求資源的地址
        /// </summary>
        public string Url
        {
            get { return url; }
            set { url = value; }
        }

        /// <summary>
        /// 用於在獲取回應后,暫時記錄回應的HTTP頭
        /// </summary>
        public WebHeaderCollection ResponseHeaders
        {
            get { return responseHeaders; }
        }

        /// <summary>
        /// 獲取或設置期望的資源類型
        /// </summary>
        public string Accept
        {
            get { return accept; }
            set { accept = value; }
        }

        /// <summary>
        /// 獲取或設置請求中的Http頭User-Agent的值
        /// </summary>
        public string UserAgent
        {
            get { return userAgent; }
            set { userAgent = value; }
        }

        /// <summary>
        /// 獲取或設置Cookie及Referer
        /// </summary>
        public HttpClientContext Context
        {
            get { return context; }
            set { context = value; }
        }

        /// <summary>
        /// 獲取或設置獲取內容的起始點,用於斷點續傳,多線程下載等
        /// </summary>
        public int StartPoint
        {
            get { return startPoint; }
            set { startPoint = value; }
        }

        /// <summary>
        /// 獲取或設置獲取內容的結束點,用於斷點續傳,多下程下載等.
        /// 如果為0,表示獲取資源從StartPoint開始的剩余內容
        /// </summary>
        public int EndPoint
        {
            get { return endPoint; }
            set { endPoint = value; }
        }



        #endregion

        #region constructors
        /// <summary>
        /// 構造新的HttpClient實例
        /// </summary>
        public HttpClient()
            : this(null)
        {
        }

        /// <summary>
        /// 構造新的HttpClient實例
        /// </summary>
        /// <param name="url">要獲取的資源的地址</param>
        public HttpClient(string url)
            : this(url, null)
        {
        }

        /// <summary>
        /// 構造新的HttpClient實例
        /// </summary>
        /// <param name="url">要獲取的資源的地址</param>
        /// <param name="context">Cookie及Referer</param>
        public HttpClient(string url, HttpClientContext context)
            : this(url, context, false)
        {
        }

        /// <summary>
        /// 構造新的HttpClient實例
        /// </summary>
        /// <param name="url">要獲取的資源的地址</param>
        /// <param name="context">Cookie及Referer</param>
        /// <param name="keepContext">是否自動在不同的請求間保留Cookie, Referer</param>
        public HttpClient(string url, HttpClientContext context, bool keepContext)
        {
            this.url = url;
            this.context = context;
            this.keepContext = keepContext;
            if (this.context == null)
                this.context = new HttpClientContext();
            cookie = new CookieContainer();
        }
        #endregion

        #region AttachFile
        /// <summary>
        /// 在請求中添加要上傳的文件
        /// </summary>
        /// <param name="fileName">要上傳的文件路徑</param>
        /// <param name="fieldName">文件字段的名稱(相當於&lt;input type=file name=fieldName&gt;)里的fieldName)</param>
        public void AttachFile(string fileName, string fieldName)
        {
            HttpUploadingFile file = new HttpUploadingFile(fileName, fieldName);
            files.Add(file);
        }

        /// <summary>
        /// 在請求中添加要上傳的文件
        /// </summary>
        /// <param name="data">要上傳的文件內容</param>
        /// <param name="fileName">文件名</param>
        /// <param name="fieldName">文件字段的名稱(相當於&lt;input type=file name=fieldName&gt;)里的fieldName)</param>
        public void AttachFile(byte[] data, string fileName, string fieldName)
        {
            HttpUploadingFile file = new HttpUploadingFile(data, fileName, fieldName);
            files.Add(file);
        }
        #endregion

        /// <summary>
        /// 清空PostingData, Files, StartPoint, EndPoint, ResponseHeaders, 並把Verb設置為Get.
        /// 在發出一個包含上述信息的請求后,必須調用此方法或手工設置相應屬性以使下一次請求不會受到影響.
        /// </summary>
        public void Reset()
        {
            verb = HttpVerb.GET;
            files.Clear();
            postingData.Clear();
            responseHeaders = null;
            startPoint = 0;
            endPoint = 0;
            IsGzip = false;
            if (repeatPostData != null) repeatPostData.Clear();
        }
        public string ip;
        private IPEndPoint BindIPEndPointCallback(ServicePoint servicePoint, IPEndPoint remoteEndPoint, int retryCount)
        {
            return new IPEndPoint(IPAddress.Parse(ip), 0);
        }

        public string cookieStr = "";

        private HttpWebRequest CreateRequest()
        {
            HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);

            req.CookieContainer = cookie;
            //req.Headers.Add("Accept-Language", defaultLanguage);
            req.Accept = accept;
            req.UserAgent = userAgent;
            req.KeepAlive = true;
            req.AllowAutoRedirect = true;
            req.Timeout = TimeOut;



            if (IsGzip)
            {
                req.Headers.Add("Accept-Encoding", "gzip");
            }

            if (ip != null)
            {
                req.ServicePoint.BindIPEndPointDelegate = new BindIPEndPoint(BindIPEndPointCallback);
            }
            if (context.Cookies != null)
                req.CookieContainer.Add(context.Cookies);
            if (!string.IsNullOrEmpty(context.Referer))
                req.Referer = context.Referer;

            if (verb == HttpVerb.HEAD)
            {
                req.Method = "HEAD";
                return req;
            }

            if (postingData.Count > 0 || files.Count > 0)
                verb = HttpVerb.POST;
            if (cookieStr != "") req.Headers.Add("Cookie", cookieStr);
            if (verb == HttpVerb.POST)
            {
                req.Method = "POST";

                MemoryStream memoryStream = new MemoryStream();

                StreamWriter writer;
                if (encodeMemory)
                {
                    writer = new StreamWriter(memoryStream, Encoding.GetEncoding(EncodingType));
                }
                else
                    writer = new StreamWriter(memoryStream);

                if (files.Count > 0 || boundaryed)
                {
                    string newLine = "\r\n";
                    string boundary = Guid.NewGuid().ToString().Replace("-", "");
                    req.ContentType = "multipart/form-data; boundary=" + boundary;

                    foreach (string key in postingData.Keys)
                    {
                        writer.Write("--" + boundary + newLine);
                        writer.Write("Content-Disposition: form-data; name=\"{0}\"{1}{1}", key, newLine);
                        writer.Write(postingData[key] + newLine);
                    }



                    foreach (HttpUploadingFile file in files)
                    {
                        writer.Write("--" + boundary + newLine);
                        writer.Write(
                            "Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"{2}",
                            file.FieldName,
                            file.FileName,
                            newLine
                            );
                        writer.Write("Content-Type: image/jpeg" + newLine + newLine);
                        writer.Flush();
                        memoryStream.Write(file.Data, 0, file.Data.Length);
                        writer.Write(newLine);
                        writer.Write("--" + boundary + "--" + newLine);
                    }

                }
                else
                {
                    req.ContentType = "application/x-www-form-urlencoded";
                    StringBuilder sb = new StringBuilder();
                    foreach (string key in postingData.Keys)
                    {
                        sb.AppendFormat("{0}={1}&",HttpUtility.UrlEncode(key, Encoding.GetEncoding(EncodingType)), HttpUtility.UrlEncode(postingData[key], Encoding.GetEncoding(EncodingType)));
                    }

                    if (repeatPostData != null)
                    {
                        foreach (var item in repeatPostData)
                        {
                            sb.AppendFormat("{0}={1}&", HttpUtility.UrlEncode(item.key, Encoding.GetEncoding(EncodingType)), HttpUtility.UrlEncode(item.value, Encoding.GetEncoding(EncodingType)));
                        }
                    }

                    if (sb.Length > 0)
                        sb.Length--;
                    writer.Write(sb.ToString());
                }

                writer.Flush();

                using (Stream stream = req.GetRequestStream())
                {
                    memoryStream.WriteTo(stream);
                }
            }

            if (startPoint != 0 && endPoint != 0)
                req.AddRange(startPoint, endPoint);
            else if (startPoint != 0 && endPoint == 0)
                req.AddRange(startPoint);

            return req;
        }

        /// <summary>
        /// 發出一次新的請求,並返回獲得的回應
        /// 調用此方法永遠不會觸發StatusUpdate事件.
        /// </summary>
        /// <returns>相應的HttpWebResponse</returns>
        public HttpWebResponse GetResponse()
        {

            HttpWebRequest req = CreateRequest();
            HttpWebResponse res = null;
            try
            {
                res = (HttpWebResponse)req.GetResponse();


                responseHeaders = res.Headers;
                if (keepContext)
                {
                    context.Cookies = res.Cookies;
                    context.Referer = url;
                    cookie.Add(context.Cookies);
                }
            }
            catch (Exception)
            { throw; }
            return res;

        }

        /// <summary>
        /// 發出一次新的請求,並返回回應內容的流
        /// 調用此方法永遠不會觸發StatusUpdate事件.
        /// </summary>
        /// <returns>包含回應主體內容的流</returns>
        public Stream GetStream()
        {
            return GetResponse().GetResponseStream();
        }
        public string responseURL;
        /// <summary>
        /// 發出一次新的請求,並以字節數組形式返回回應的內容
        /// 調用此方法會觸發StatusUpdate事件
        /// </summary>
        /// <returns>包含回應主體內容的字節數組</returns>
        public byte[] GetBytes()
        {
            byte[] result = new byte[] { 0, 1 };
            try
            {
                HttpWebResponse res = GetResponse();
                int length = (int)res.ContentLength;
                responseURL = res.ResponseUri.AbsoluteUri;
                MemoryStream memoryStream = new MemoryStream();
                byte[] buffer = new byte[0x100];
                Stream rs = res.GetResponseStream();
                for (int i = rs.Read(buffer, 0, buffer.Length); i > 0; i = rs.Read(buffer, 0, buffer.Length))
                {
                    memoryStream.Write(buffer, 0, i);
                    OnStatusUpdate(new StatusUpdateEventArgs((int)memoryStream.Length, length));
                }
                rs.Close();
                result = memoryStream.ToArray();
            }
            catch (Exception)
            {
                throw;
            }

            return result;
        }

        /// <summary>
        /// 發出一次新的請求,以Http頭,或Html Meta標簽,或DefaultEncoding指示的編碼信息對回應主體解碼
        /// 調用此方法會觸發StatusUpdate事件
        /// </summary>
        /// <returns>解碼后的字符串</returns>
        public string GetString()
        {
            byte[] data = GetBytes();
            if (responseHeaders.AllKeys.Contains<string>("Content-Encoding") && responseHeaders["Content-Encoding"].Contains("gzip"))
            {
                //Console.WriteLine(responseHeaders["Content-Encoding"].ToString());
                data = GZipDecompress(data);
            }

            string encodingName = GetEncodingFromHeaders();

            if (encodingName == null)
                encodingName = GetEncodingFromBody(data);

            Encoding encoding;
            if (encodingName == null)
                encoding = defaultEncoding;
            else
            {
                try
                {
                    encoding = Encoding.GetEncoding(encodingName);
                }
                catch (ArgumentException)
                {
                    encoding = defaultEncoding;
                }
            }
            return encoding.GetString(data);
        }

        /// <summary>
        /// 發出一次新的請求,對回應的主體內容以指定的編碼進行解碼
        /// 調用此方法會觸發StatusUpdate事件
        /// </summary>
        /// <param name="encoding">指定的編碼</param>
        /// <returns>解碼后的字符串</returns>
        public string GetString(Encoding encoding)
        {
            byte[] data = GetBytes();
            return encoding.GetString(data);
        }

        /// <summary>
        /// GZip解壓函數
        /// </summary>
        /// <param name="data"></param>
        /// <returns></returns>
        private byte[] GZipDecompress(byte[] data)
        {
            using (MemoryStream stream = new MemoryStream())
            {
                using (GZipStream gZipStream = new GZipStream(new MemoryStream(data), CompressionMode.Decompress))
                {
                    byte[] bytes = new byte[40960];
                    int n;
                    while ((n = gZipStream.Read(bytes, 0, bytes.Length)) != 0)
                    {
                        stream.Write(bytes, 0, n);
                    }
                    gZipStream.Close();
                }

                return stream.ToArray();
            }
        }

        private string GetEncodingFromHeaders()
        {
            string encoding = null;
            try
            {
                string contentType = responseHeaders["Content-Type"];
                if (contentType != null)
                {
                    int i = contentType.IndexOf("charset=");
                    if (i != -1)
                    {
                        encoding = EncodingType = contentType.Substring(i + 8);
                    }
                }
            }
            catch (Exception)
            { }
            return encoding;
        }

        private string GetEncodingFromBody(byte[] data)
        {
            //string encodingName = null;
            string dataAsAscii = Encoding.ASCII.GetString(data);
            if (dataAsAscii != null)
            {
                int i = dataAsAscii.IndexOf("charset=");
                if (i != -1)
                {
                    int j = dataAsAscii.IndexOf("\"", i);
                    if (j != -1)
                    {
                        int k = i + 8;
                        EncodingType = dataAsAscii.Substring(k, (j - k) + 1);
                        char[] chArray = new char[2] { '>', '"' };
                        EncodingType = EncodingType.TrimEnd(chArray);
                    }
                }
            }
            return EncodingType;
        }

        /// <summary>
        /// 發出一次新的Head請求,獲取資源的長度
        /// 此請求會忽略PostingData, Files, StartPoint, EndPoint, Verb
        /// </summary>
        /// <returns>返回的資源長度</returns>
        public int HeadContentLength()
        {
            Reset();
            HttpVerb lastVerb = verb;
            verb = HttpVerb.HEAD;
            using (HttpWebResponse res = GetResponse())
            {
                verb = lastVerb;
                return (int)res.ContentLength;
            }
        }

        /// <summary>
        /// 發出一次新的請求,把回應的主體內容保存到文件
        /// 調用此方法會觸發StatusUpdate事件
        /// 如果指定的文件存在,它會被覆蓋
        /// </summary>
        /// <param name="fileName">要保存的文件路徑</param>
        public void SaveAsFile(string fileName)
        {
            SaveAsFile(fileName, FileExistsAction.Overwrite);
        }

        /// <summary>
        /// 發出一次新的請求,把回應的主體內容保存到文件
        /// 調用此方法會觸發StatusUpdate事件
        /// </summary>
        /// <param name="fileName">要保存的文件路徑</param>
        /// <param name="existsAction">指定的文件存在時的選項</param>
        /// <returns>是否向目標文件寫入了數據</returns>
        public bool SaveAsFile(string fileName, FileExistsAction existsAction)
        {
            byte[] data = GetBytes();
            switch (existsAction)
            {
                case FileExistsAction.Overwrite:
                    using (BinaryWriter writer = new BinaryWriter(new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.Write)))
                        writer.Write(data);
                    return true;

                case FileExistsAction.Append:
                    using (BinaryWriter writer = new BinaryWriter(new FileStream(fileName, FileMode.Append, FileAccess.Write)))
                        writer.Write(data);
                    return true;

                default:
                    if (!File.Exists(fileName))
                    {
                        using (
                            BinaryWriter writer =
                                new BinaryWriter(new FileStream(fileName, FileMode.Create, FileAccess.Write)))
                            writer.Write(data);
                        return true;
                    }
                    else
                    {
                        return false;
                    }
            }
        }
    }

    public class HttpClientContext
    {
        private CookieCollection cookies;
        private string referer;

        public CookieCollection Cookies
        {
            get { return cookies; }
            set { cookies = value; }
        }

        public string Referer
        {
            get { return referer; }
            set { referer = value; }
        }
    }

    public class RepeatPostData
    {
        public string key { get; set; }
        public string value { get; set; }
    }

    public enum HttpVerb
    {
        GET,
        POST,
        HEAD,
    }

    public enum FileExistsAction
    {
        Overwrite,
        Append,
        Cancel,
    }

    public class HttpUploadingFile
    {
        private string fileName;
        private string fieldName;
        private byte[] data;

        public string FileName
        {
            get { return fileName; }
            set { fileName = value; }
        }

        public string FieldName
        {
            get { return fieldName; }
            set { fieldName = value; }
        }

        public byte[] Data
        {
            get { return data; }
            set { data = value; }
        }

        public HttpUploadingFile(string fileName, string fieldName)
        {
            this.fileName = fileName;
            this.fieldName = fieldName;
            using (FileStream stream = new FileStream(fileName, FileMode.Open))
            {
                byte[] inBytes = new byte[stream.Length];
                stream.Read(inBytes, 0, inBytes.Length);
                data = inBytes;
            }
        }

        public HttpUploadingFile(byte[] data, string fileName, string fieldName)
        {
            this.data = data;
            this.fileName = fileName;
            this.fieldName = fieldName;
        }
    }

    public class StatusUpdateEventArgs : EventArgs
    {
        private readonly int bytesGot;
        private readonly int bytesTotal;

        public StatusUpdateEventArgs(int got, int total)
        {
            bytesGot = got;
            bytesTotal = total;
        }

        /// <summary>
        /// 已經下載的字節數
        /// </summary>
        public int BytesGot
        {
            get { return bytesGot; }
        }

        /// <summary>
        /// 資源的總字節數
        /// </summary>
        public int BytesTotal
        {
            get { return bytesTotal; }
        }
    }
}

　　然后我們先根據這個方法獲取博客園首頁的源代碼

        /// <summary>
        /// 根據網址獲取頁面源碼
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public string GetHtml(string url)
        {
            string ContentHtml = "";
            try
            {
                HttpClient hc = new HttpClient();
                hc.Url = url;
                if (!hc.Url.Contains("http://"))//如果輸入的網址沒有包含http:// 則手動添加
                {
                    hc.Url = "http://" + hc.Url;
                }
                ContentHtml = hc.GetString();
            }
            catch (Exception e)//如果上面的執行出錯，則返回繼續執行
            {
                return GetHtml(url);
            }
            return ContentHtml;
        }

　　然后再觀察每條隨筆的規律，我們發現沒條的開頭是<div class="post_item_body">，結尾是<div class="clear">，那我們就可以根據這個規律來寫出正則：Regex regexContent = new Regex("<div class=\"post_item_body\">(?<content>.*?)<div class=\"clear\"></div>",RegexOptions.Singleline);
     然后可以使用這個正則來獲取我們需要匹配的內容了

 　　　　　　 string Html= GetHtml("http://www.cnblogs.com/");
            Regex regexContent = new Regex("<div class=\"post_item_body\">(?<content>.*?)<div class=\"clear\"></div>",RegexOptions.Singleline);
            string blog = regexContent.Match(Html).Groups["content"].Value.ToString();

在這里我用到的正則匹配工具是Expresso,有需要的朋友可以留言。當然，如果我有什么地方寫的不好的，歡迎各位指出。晚上就先到這里了，該洗洗睡了。