使用HtmlAgilityPack 爬取 国家统计局 区划和城乡划分代码

HtmlAgilityPack：Html解析神器，根据url地址解析html页面内容。

项目引用HtmlAgilityPack.dll文件或者通过安装 nuget 包 HtmlAgilityPack

特别注意：页面编码是GB2312时，会出现乱码问题。

解决方法：

.NET Framework 下处理方法：

var encoding = Encoding.GetEncoding("GB2312");

 

.NET Core 下处理方法：（安装 nuget 包 System.Text.Encoding.CodePages，需不需要？因为卸载后也没有出现问题）

System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
var encoding = System.Text.Encoding.GetEncoding("GB2312");

好了，说完处理过程中出现的问题，下面就放上程序代码：

using System;
using System.Threading;
using HtmlAgilityPack;                 //需要添加的命名空间

namespace ConsoleApp1
{
    class Program
    {
        static string baseUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/";
        static void Main(string[] args)
        {
            GetProvince("index.html");
            Console.ReadKey();
        }

        /// <summary>
        /// 通过url获取html内容
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static HtmlDocument GetHtmlDocument(string url)
        {
            HtmlDocument doc = null;
            try
            {
                HtmlWeb web = new HtmlWeb();
                System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
                var encoding = System.Text.Encoding.GetEncoding("GB2312");
                web.OverrideEncoding = encoding;
                //从url中加载
                doc = web.Load(baseUrl + url);

                //获得title标签节点，其子标签下的所有节点也在其中
                HtmlNode headNode = doc.DocumentNode.SelectSingleNode("//title");
                //获得title标签中的内容
                string Title = headNode.InnerText;
                //Console.WriteLine(Title);
                if (Title == "502 Bad Gateway")
                {
                    Thread.Sleep(10000);
                    doc = GetHtmlDocument(url);
                }
            }
            catch (Exception ex)
            {
                throw ex;
            }
            return doc;
        }
        /// <summary>
        /// 省份
        /// </summary>
        /// <param name="url"></param>
        public static void GetProvince(string url)
        {
            //从url中加载
            HtmlDocument doc = GetHtmlDocument(url);

            //table class='provincetable'
            //provincetable table标签节点
            HtmlNode tableNode = doc.DocumentNode.SelectSingleNode("//table[@class='provincetable']");
            //获得tr标签下的所有子节点
            HtmlNodeCollection trCollection = tableNode.ChildNodes;
            foreach (var trItem in trCollection)
            {
                var classNode = trItem.Attributes["class"];
                //判断是否存在class属性
                if (classNode != null)
                {
                    //获得标签属性为class的值
                    string className = classNode.Value;
                    if (className == "provincetr")
                    {
                        //获得tr标签下的所有子节点
                        HtmlNodeCollection tdCollection = trItem.ChildNodes;
                        foreach (var tdItem in tdCollection)
                        {
                            //获得td标签下的所有子节点
                            HtmlNodeCollection aCollection = tdItem.ChildNodes;
                            foreach (var item in aCollection)
                            {
                                //只查找a标签 派出其他标签
                                if (item.Name == "a")
                                {
                                    //获得标签属性为href的值
                                    string aValue = item.Attributes["href"].Value;
                                    //获得标签内的内容
                                    string aInterText = item.InnerText.Replace("<br/>", "");
                                    string[] split = aValue.Split(new Char[] { '.' });
                                    string code = split[0];

                                    Console.WriteLine("区划代码：" + code + "\t" + " 省份:" + aInterText);

                                    //Thread.Sleep(5000);
                                    GetCity(aValue);
                                }
                            }
                        }
                    }
                }
            }
        }
        /// <summary>
        /// 市
        /// </summary>
        /// <param name="url"></param>
        public static void GetCity(string url)
        {
            //从url中加载
            HtmlDocument doc = GetHtmlDocument(url);

            //table class='provincetable'
            //citytable table标签节点
            HtmlNode tableNode = doc.DocumentNode.SelectSingleNode("//table[@class='citytable']");
            //获得tr标签下的所有子节点
            HtmlNodeCollection trCollection = tableNode.ChildNodes;
            foreach (var trItem in trCollection)
            {
                var classNode = trItem.Attributes["class"];
                //判断是否存在class属性
                if (classNode != null)
                {
                    //获得标签属性为class的值
                    string className = classNode.Value;
                    if (className == "citytr")
                    {
                        //获得tr标签下的所有子节点
                        HtmlNodeCollection tdCollection = trItem.ChildNodes;
                        int i = 0;
                        foreach (var tdItem in tdCollection)
                        {
                            //获取第二td
                            if (i == 1)
                            {
                                //获得td标签下的所有子节点
                                HtmlNodeCollection aCollection = tdItem.ChildNodes;

                                foreach (var item in aCollection)
                                {
                                    //只查找a标签 派出其他标签
                                    if (item.Name == "a")
                                    {
                                        //获得标签属性为href的值
                                        string aValue = item.Attributes["href"].Value;
                                        //获得标签内的内容
                                        string aInterText = item.InnerText.Replace("<br/>", "");
                                        string[] split = aValue.Split(new Char[] { '.' })[0].Split(new Char[] { '/' });

                                        string code = split[1];


                                        Console.WriteLine("区划代码：" + code + "\t" + " 市:" + aInterText);

                                        //11/1101.html  2019/11/1101.html
                                        //Thread.Sleep(5000);
                                        GetCounty(aValue);
                                    }
                                }
                            }
                            i++;
                        }
                    }
                }
            }
        }
        /// <summary>
        /// 区/县
        /// </summary>
        /// <param name="url"></param>
        public static void GetCounty(string url)
        {
            //从url中加载
            HtmlDocument doc = GetHtmlDocument(url);

            //table class='countytable'
            //countytable table标签节点
            HtmlNode tableNode = doc.DocumentNode.SelectSingleNode("//table[@class='countytable']");
            //获得tr标签下的所有子节点
            HtmlNodeCollection trCollection = tableNode.ChildNodes;
            foreach (var trItem in trCollection)
            {
                var classNode = trItem.Attributes["class"];
                //判断是否存在class属性
                if (classNode != null)
                {
                    //获得标签属性为class的值
                    string className = classNode.Value;
                    if (className == "countytr")
                    {
                        //获得tr标签下的所有子节点
                        HtmlNodeCollection tdCollection = trItem.ChildNodes;
                        int i = 0;
                        foreach (var tdItem in tdCollection)
                        {
                            //获取第二td
                            if (i == 1)
                            {
                                //获得td标签下的所有子节点
                                HtmlNodeCollection aCollection = tdItem.ChildNodes;

                                foreach (var item in aCollection)
                                {
                                    //只查找a标签 派出其他标签
                                    if (item.Name == "a")
                                    {
                                        //获得标签属性为href的值
                                        string aValue = item.Attributes["href"].Value;
                                        //获得标签内的内容
                                        string aInterText = item.InnerText.Replace("<br/>", "");
                                        string[] split = aValue.Split(new Char[] { '.' })[0].Split(new Char[] { '/' });
                                        string code = split[1];

                                        Console.WriteLine("区划代码：" + code + "\t" + " 区/县:" + aInterText);

                                        //01/1101.html  2019/11/01/1101.html
                                        string provinceCode = code.Substring(0, 2);
                                        var townUrl = provinceCode + "/" + aValue;
                                        //Thread.Sleep(5000);
                                        GetTown(townUrl);
                                    }
                                }
                            }
                            i++;
                        }
                    }
                }
            }
        }

        /// <summary>
        /// 乡/镇/街道
        /// </summary>
        /// <param name="url"></param>
        public static void GetTown(string url)
        {
            //从url中加载
            HtmlDocument doc = GetHtmlDocument(url);

            //table class='towntable'
            //towntable table标签节点
            HtmlNode tableNode = doc.DocumentNode.SelectSingleNode("//table[@class='towntable']");
            //获得tr标签下的所有子节点
            HtmlNodeCollection trCollection = tableNode.ChildNodes;
            foreach (var trItem in trCollection)
            {
                var classNode = trItem.Attributes["class"];
                //判断是否存在class属性
                if (classNode != null)
                {
                    //获得标签属性为class的值
                    string className = classNode.Value;
                    if (className == "towntr")
                    {
                        //获得tr标签下的所有子节点
                        HtmlNodeCollection tdCollection = trItem.ChildNodes;
                        int i = 0;
                        foreach (var tdItem in tdCollection)
                        {
                            //获取第二td
                            if (i == 1)
                            {
                                //获得td标签下的所有子节点
                                HtmlNodeCollection aCollection = tdItem.ChildNodes;

                                foreach (var item in aCollection)
                                {
                                    //只查找a标签 派出其他标签
                                    if (item.Name == "a")
                                    {
                                        //获得标签属性为href的值
                                        string aValue = item.Attributes["href"].Value;
                                        //获得标签内的内容
                                        string aInterText = item.InnerText.Replace("<br/>", "");
                                        string[] split = aValue.Split(new Char[] { '.' })[0].Split(new Char[] { '/' });
                                        string code = split[1];

                                        Console.WriteLine("区划代码：" + code + "\t" + " 乡/镇/街道:" + aInterText);

                                        //01/110101001.html  2019/11/01/01/110101001.html
                                        string provinceCode = code.Substring(0, 2);
                                        string cityCode = code.Substring(2, 2);
                                        var villageUrl = provinceCode + "/" + cityCode + "/" + aValue;
                                        Thread.Sleep(5000);
                                        GetVillage(villageUrl);
                                    }
                                }
                            }
                            i++;
                        }
                    }
                }
            }
        }

        /// <summary>
        /// 村庄
        /// </summary>
        /// <param name="url"></param>
        public static void GetVillage(string url)
        {
            //从url中加载
            HtmlDocument doc = GetHtmlDocument(url);

            //table class='villagetable'
            //villagetable table标签节点
            HtmlNode tableNode = doc.DocumentNode.SelectSingleNode("//table[@class='villagetable']");
            //获得tr标签下的所有子节点
            HtmlNodeCollection trCollection = tableNode.ChildNodes;
            foreach (var trItem in trCollection)
            {
                var classNode = trItem.Attributes["class"];
                //判断是否存在class属性
                if (classNode != null)
                {
                    //获得标签属性为class的值
                    string className = classNode.Value;
                    if (className == "villagetr")
                    {
                        //获得tr标签下的所有子节点
                        HtmlNodeCollection tdCollection = trItem.ChildNodes;

                        string code = "";
                        string typeCode = "";
                        string villageName = "";

                        int i = 0;
                        foreach (var tdItem in tdCollection)
                        {
                            //获得标签内的内容
                            string aInterText = tdItem.InnerText.Replace("<br/>", "");

                            //获取第二td
                            if (i == 0) //区划代码
                            {
                                code = aInterText;
                            }
                            else if (i == 1) //城乡分类代码
                            {
                                typeCode = aInterText;
                            }
                            else if (i == 2) //村庄名称
                            {
                                villageName = aInterText;
                            }

                            i++;

                        }
                        //城乡分类代码
                        Console.WriteLine("区划代码：" + code + "\t" + " 城乡分类代码：" + typeCode + "\t" + " 村庄:" + villageName);
                    }
                }
            }
        }
    }
}

再上传一个效果图吧，方便大家看到结果：