爬一下国家统计局行政区划代码C#

目前NBS上有2015-2018四个年度的代码信息，写一个控制台程序爬一下县级行政区下的代码。

使用HttpWebRequest+HttpWebResponse获取html，使用HtmlAgilityPack类库解析HTML。

使用POST请求，请求头带Cookie信息，否则会被反爬机制挡死，返回“请开启JavaScript并刷新该页”。

县级URL Request获取数据的同时记录Response的Cookie信息，在请求镇级数据时，请求头发送此cookie。

“省-地-县-乡 ”与“省-县（地）-乡” 的URL长度不同，根据长度判断URL正确性时需注意，也许还有其他可能，暂未发现。

主方法

 class Program
    {
        static void Main(string[] args)
        {
            Console.ForegroundColor = ConsoleColor.Magenta;
            Console.WriteLine("\r\n----获取县级行政区乡、村二级区划代码");
            Console.WriteLine("----数据年份有：");
            Console.ResetColor();
            Cursor.WriteAt("A、2018", 2, 0);
            Cursor.WriteAt("B、2017", 12, 0);
            Cursor.WriteAt("C、2016", 2, 1);
            Cursor.WriteAt("D、2015", 12, 1);
            Input: Console.ForegroundColor = ConsoleColor.Magenta;
            Console.WriteLine();
            Console.WriteLine("----请输入一个年份代码（回车提交）：");
            Console.ResetColor();
            char chr = Convert.ToChar( Console.ReadLine().ToLower()[0]);
            if ((int)chr >= 97 &&(int)chr <= 100)
            {
                string year = string.Empty;
                switch (chr)
                {
                    case 'a':
                        year = "2018"; break;
                    case 'b':
                        year = "2017"; break;
                    case 'c':
                        year = "2016"; break;
                    default:
                        year = "2015"; break;
                }
                System.Diagnostics.Process.Start($"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/{year}");
                Console.ForegroundColor = ConsoleColor.Magenta;
                Console.WriteLine("浏览器已加载区划代码起始页，请进入县级行政单位页面，复制url，粘贴到下面（回车提交）：");
            }
            else
                goto Input;
            Console.ResetColor();
            string cityurl = Console.ReadLine();
            if (cityurl.Length != 66&& cityurl.Length!=71)
            {
                Console.ForegroundColor = ConsoleColor.Magenta;
                Console.WriteLine("url有误，请确认是县级行政单位页面，重新复制链接，粘贴到下面：");
                Console.ResetColor();
                cityurl = Console.ReadLine();
            }
            try
            {
                Console.ForegroundColor = ConsoleColor.Magenta;
                Func<object, List<TownInfo>> func = new Func<object, List<TownInfo>>(GetTownInfos);
                Task<List<TownInfo>> task = new Task<List<TownInfo>>(func, cityurl);
                task.Start();
                task.Wait();
                if (task.Status == TaskStatus.RanToCompletion && task.Result.Count > 0)
                {

                    List<VillageInfo> villageInfos = new List<VillageInfo>();
                    foreach (var item in task.Result)
                    {
                        //把乡镇信息写入村级列表，实现乡镇信息输出
                        VillageInfo villageInfo_town = new VillageInfo(item.Code, "", item.Name);
                        villageInfos.Add(villageInfo_town);
                        Func<object, List<VillageInfo>> func1 = new Func<object, List<VillageInfo>>(GetVillageInfos);
                        Task<List<VillageInfo>> task1 = new Task<List<VillageInfo>>(func1, item.Href);
                        task1.Start();
                        task1.Wait();
                        if (task1.Status == TaskStatus.RanToCompletion)
                        {
                            villageInfos.AddRange(task1.Result);
                        }
                    }
                    foreach (var item1 in villageInfos)
                    {
                        Console.WriteLine($"{item1.Name.Trim()}\t{item1.Cls.Trim()}\t{item1.Code.Trim()}");
                    }
                }
                else
                { Console.WriteLine("乡镇列表获取失败！"); }

            }
            catch (Exception)
            {
                throw new Exception("");
            }
            Console.ReadKey();
        }
        static string cookies = "AD_RS_COOKIE=20082854; wzws_cid=453a2d88181321410de83ba7eedaba3a141eb61ee7488027b6ab07a66054605e99e886827afa72708ce170398ea2fdfeec55455a7c0be8e779694026255f2166";
        //获取乡镇级信息列表
        static List<TownInfo> GetTownInfos(object cityurl)
        {
            List<TownInfo> townInfos = new List<TownInfo>();
            HttpGetHelper httpGetHelper = new HttpGetHelper() { Url =(string) cityurl, ContentType = "text/html; charset=gb2312", Encode = Encoding.GetEncoding(936),RequestMethod="post"};
            //HtmlAgilityPack类库解析HTML
            HtmlDocument document = new HtmlDocument();
            document.LoadHtml(httpGetHelper.GetHtml(1,ref cookies));
            //string html = httpGetHelper.GetHtml(ref cookies);
            //路径里"//"表示从根节点开始查找，两个斜杠‘//’表示查找所有childnodes；一个斜杠'/'表示只查找第一层的childnodes（即不查找grandchild）；点斜杠"./"表示从当前结点而不是根结点开始查找
            HtmlNodeCollection htmlNodes = document.DocumentNode.SelectNodes("//tr[@class='towntr']");
            foreach (var node in htmlNodes)
            {
                HtmlNodeCollection htmlNodes1 = node.SelectNodes("./td");
                HtmlNode htmlNodeHref = node.SelectSingleNode(".//a[@href]");
                HtmlAttribute htmlAttribute = htmlNodeHref.Attributes["href"];
                TownInfo townInfo = new TownInfo(htmlNodes1[0].InnerText, htmlNodes1[1].InnerText,
                    (cityurl as string).Substring(0, (cityurl as string).LastIndexOf('/') + 1) + htmlAttribute.Value);
                townInfos.Add(townInfo);
            }
            return townInfos;
        }
        //获取村级信息列表
        static List<VillageInfo> GetVillageInfos(object townurl)
        {
            List<VillageInfo> villageInfos = new List<VillageInfo>();
            HttpGetHelper httpGetHelper = new HttpGetHelper() { Url = (string)townurl, ContentType = "text/html; charset=gb2312", Encode = Encoding.GetEncoding(936), RequestMethod = "post"};
            HtmlDocument document = new HtmlDocument();
            document.LoadHtml(httpGetHelper.GetHtml(2,ref cookies));
            //string html = httpGetHelper.GetHtml(ref cookies);
            HtmlNodeCollection htmlNodes = document.DocumentNode.SelectNodes("//tr[@class='villagetr']");
            foreach (var node in htmlNodes)
            {
                HtmlNodeCollection htmlNodes1 = node.SelectNodes(".//td");
                VillageInfo villageInfo = new VillageInfo(htmlNodes1[0].InnerText,htmlNodes1[1].InnerText,htmlNodes1[2].InnerText);
                villageInfos.Add(villageInfo);
            }
            return villageInfos;
        }
    }

 

辅助类/结构

 

  internal class Cursor
    {
        const int origRow = 3;
        const int origCol = 0;
        public static void WriteAt(string s, int c, int r)
        {
            Console.SetCursorPosition(origCol + c, origRow + r);
            Console.Write(s);
        }
    }
    //乡镇信息结构 编码、名称、超链
    struct TownInfo
    {
        string code;
        public string Code{ get { return code; } }
        string name;
        public string Name{get { return name; } }
        string href;
        public string Href { get { return href; } }
        public TownInfo (string code,string name,string href)
        {
            this.code = code;
            this.name = name;
            this.href = href;
        }
    }
    //村信息结构 编码、城乡划分类，名称
    struct VillageInfo
    {
        string code;
        public string Code{ get { return code; } }
        string  cls;
        public string Cls{ get { return cls; } }
        string name;
        public string Name{ get { return name; } }
        public VillageInfo(string code,string cls,string name)
        {
            this.code = code;
            this.cls = cls;
            this.name = name;
        }
    }

 

获取HTML

    public class HttpGetHelper
    {
        string url = string.Empty;
        public string Url
        {
            set { url = value; }
        }

        int timeOut=10*1000;
        public int Timeout
        {
            set { timeOut = value; }
        }

        string contentType= "text/html;charset=utf-8";
        public string ContentType
        {
            set { contentType = value; }
        }

        string userAgent= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 ";
        public string UserAgent
        {
            set { userAgent = value; }
        }

        Encoding encode=Encoding.UTF8;
        public Encoding Encode
        {
            set { encode = value; }
        }
        string request_Method = "get";
        public string RequestMethod
        {
            set { request_Method = value; }
        }
        /// <summary>
        /// get html content
        /// </summary>
        /// <param name="cls">town=1;village=2</param>
        /// <param name="cookies">if cls=1 then ref cookies</param>
        /// <returns></returns>
        public string GetHtml(int cls,ref string cookies)
        {
            string html = string.Empty;
            try
            {
                if (url!=string.Empty)
                {
                    HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
                    request.Timeout = this.timeOut;
                    request.ContentType = this.contentType;
                    request.UserAgent = this.userAgent;
                    request.Headers.Add(HttpRequestHeader.Cookie, cookies);
                    request.Method = request_Method;
                    using (HttpWebResponse response =request.GetResponse()as HttpWebResponse)
                    {
                        if (response.StatusCode==HttpStatusCode.OK)
                        {//如果是县级url，则记录cookie
                            if (cls==1)
                            {
                                CookieCollection cookieCollection = response.Cookies;
                                foreach (Cookie item in cookieCollection)
                                {
                                    cookies = item.Name + "=" + item.Value + ";";
                                }
                                cookies.Remove(cookies.Length - 1);
                            }

                            using (StreamReader streamReader = new StreamReader(response.GetResponseStream(), encode))
                            {
                                html = streamReader.ReadToEnd();
                                streamReader.Close();
                            }
                        }
                    }
                }
            }
            catch (Exception)
            {
                throw new Exception($"GetHtml失败，url:{url}");
            }
            return html;
        }
    }