獲取HTML源碼（只取文字，判斷編碼，過濾標簽）

private void button1_Click(object sender, EventArgs e)
        {
            string s1 = this.textBox1.Text;
            //正則表達式內容
            //string match = @"^(http|https|ftp)\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(:[a-zA-Z0-9]*)?/?([a-zA-Z0-9\-\._\?\,\'/\\\+&%\$#\=~])*$";
            //string match = @"[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(:[a-zA-Z0-9]*)?/?([a-zA-Z0-9\-\._\?\,\'/\\\+&%\$#\=~])*$";
            string match = @"[a-zA-z]+://[^\s]*";
            //初始化正則表達式實例
            Regex reg = new Regex(match);
            //開始驗證
            bool HasValidate = reg.IsMatch(s1);

            if (HasValidate)
            {
                //MessageBox.Show("這是網站有效URL格式。");
                try
                {
                    string tmp = GetHtml(s1);
                    string tmpend = StripHTML(tmp);

                }
                catch (Exception)
                {
                    //MessageBox.Show("3.該網站只能手動查詢！");
                }
            }
        }

 

 

1.獲取HTML

GetHtml(String Url)

        /// <summary>
        /// 獲取有效的HTML
        /// </summary>
        /// <param name="Url"></param>
        /// <returns></returns>
        public String GetHtml(String Url)
        {
            string sException = null;

            string sRslt = null;
            string GBsRslt = null;
            StreamReader htm = null;
            WebResponse oWebRps = null;
            WebResponse bWebRps = null;
            int a = 0;

            WebRequest oWebRqst = WebRequest.Create(Url);

            oWebRqst.Timeout = 50000;

            WebRequest bWebRqst = WebRequest.Create(Url);

            bWebRqst.Timeout = 50000;

            try
            {
                oWebRps = oWebRqst.GetResponse();
                bWebRps = bWebRqst.GetResponse();
            }
            catch (WebException e)
            {
                sException = e.Message.ToString();

                MessageBox.Show(sException);
            }
            catch (Exception e)
            {
                sException = e.ToString();

                MessageBox.Show(sException);
            }
            finally
            {
                if (oWebRps != null)
                {
                    StreamReader oStreamRd = new StreamReader(
                        oWebRps.GetResponseStream(), Encoding.GetEncoding("UTF-8")
                        );

                    StreamReader GBoStreamRd = new StreamReader(
                        bWebRps.GetResponseStream(), Encoding.GetEncoding("GB2312")
                        );

                    sRslt = oStreamRd.ReadToEnd();
                    GBsRslt = GBoStreamRd.ReadToEnd();

                    if (!isLuan(sRslt)) //判斷utf8是否有亂碼
                    {
                        htm = oStreamRd;
                    }

                    else
                    {
                        htm = GBoStreamRd;
                    }

                    if (htm == oStreamRd)
                    {
                        a = 1;
                    }
                    else
                    {
                        a = 2;
                    }

                    oStreamRd.Close();
                    GBoStreamRd.Close();
                    oWebRps.Close();

                }
            }
            if (a == 1)
            {
                return sRslt;
            }
            else
            {
                return GBsRslt;
            }

        }

  

2.去除HTML標記（正則表達式）

StripHTML(string strHtml)

        /// <summary>
        /// 去除HTML標記
        /// </summary>
        /// <param name="strHtml">包括HTML的源碼 </param>
        /// <returns>已經去除后的文字</returns>
        public static string StripHTML(string strHtml)
        {
            //regex_str="<script type=\\s*[^>]*>[^<]*?</script>";//替換<script>內容</script>為空格
            string regex_str = "(?is)<script[^>]*>.*?</script>";//替換<script>內容</script>為空格
            strHtml = Regex.Replace(strHtml, regex_str, "");

            //regex_str="<script type=\\s*[^>]*>[^<]*?</script>";//替換<style>內容</style>為空格
            regex_str = "(?is)<style[^>]*>.*?</style>";//替換<style>內容</style>為空格
            strHtml = Regex.Replace(strHtml, regex_str, "");

            //regex_str = "(&nbsp;)+";//替換&nbsp;為空格
            regex_str = "(?i)&nbsp;";//替換&nbsp;為空格
            strHtml = Regex.Replace(strHtml, regex_str, " ");

            //regex_str = "(\r\n)*";//替換\r\n為空
            regex_str = @"[\r\n]*";//替換\r\n為空
            strHtml = Regex.Replace(strHtml, regex_str, "", RegexOptions.IgnoreCase);

            //regex_str = "<[^<]*>";//替換Html標簽為空
            regex_str = "<[^<>]*>";//替換Html標簽為空
            strHtml = Regex.Replace(strHtml, regex_str, "");

            //regex_str = "\n*";//替換\n為空
            regex_str = @"\n*";//替換\n為空
            strHtml = Regex.Replace(strHtml, regex_str, "", RegexOptions.IgnoreCase);

            //可以這樣
            regex_str = "\t*";//替換\t為空
            strHtml = Regex.Replace(strHtml, regex_str, "", RegexOptions.IgnoreCase);

            //可以
            regex_str = "'";//替換'為’
            strHtml = Regex.Replace(strHtml, regex_str, "’", RegexOptions.IgnoreCase);

            //可以
            regex_str = " +";//替換若干個空格為一個空格
            strHtml = Regex.Replace(strHtml, regex_str, "  ", RegexOptions.IgnoreCase);

            Regex regex = new Regex("<.+?>", RegexOptions.IgnoreCase);

            string strOutput = regex.Replace(strHtml, "");//替換掉"<"和">"之間的內容
            strOutput = strOutput.Replace("<", "");
            strOutput = strOutput.Replace(">", "");
            strOutput = strOutput.Replace("&nbsp;", "");


            return strOutput;

        }

 

3.判斷是否為亂碼（編碼）:在StripHTML里調用。

        //判斷是否為亂碼
        bool isLuan(string txt)
        {

            var bytes = Encoding.UTF8.GetBytes(txt);

            //239 191 189

            for (var i = 0; i < bytes.Length; i++)
            {

                if (i < bytes.Length - 3)

                    if (bytes[i] == 239 && bytes[i + 1] == 191 && bytes[i + 2] == 189)
                    {

                        return true;

                    }
            }

            return false;

        }