Java使用Jsoup獲取全國省市區（縣）街道村等信息

1、在pom.xml文件中配置jsoup

 <!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.14.1</version>
        </dependency>

 

 

 

2、配置獲取地址（目前最新的是2020，建議去列表看最新的，按照最新的配）：

http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/

 

 

http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/

 

 

3、獲取到txt文件中的數據可以根據自己想要的格式在parseNextLevel方法和printInfo方法中修改

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


/**
 * 全國省市縣鎮村數據爬取
 * @author Saffichan
 * @date 2021-07-22 10:19:39
 * @version 1.0.0
 */
public class JsoupTest
{
    private static Map<Integer, String> cssMap = new HashMap<Integer, String>();
    private static BufferedWriter bufferedWriter = null;

    static
    {
        cssMap.put(1, "provincetr");// 省
        cssMap.put(2, "citytr");// 市
        cssMap.put(3, "countytr");// 縣/區
        cssMap.put(4, "towntr");// 鎮
        cssMap.put(5, "villagetr");// 村
    }

    public static void main(String[] args) throws IOException
    {
        int level = 1;

        initFile();

        // 獲取全國各個省級信息
        Document connect = connect("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/");
        Elements rowProvince = connect.select("tr." + cssMap.get(level));
        for (Element provinceElement : rowProvince)// 遍歷每一行的省份城市
        {

            Elements select = provinceElement.select("a");

            for (Element province : select)// 每一個省份
            {

                printInfoProvice(province, level);
                parseNextLevel(province, level + 1);
            }

        }


        closeStream();
    }

    private static void initFile()
    {
        try
        {
            bufferedWriter = new BufferedWriter(new FileWriter(new File("d:\\ShowAdress.txt"), true));
        } catch (IOException e)
        {
            e.printStackTrace();
        }
    }

    private static void closeStream()
    {
        if (bufferedWriter != null)
        {
            try
            {
                bufferedWriter.close();
            } catch (IOException e)
            {
                e.printStackTrace();
            }
            bufferedWriter = null;
        }
    }

    private static void parseNextLevel(Element parentElement, int level) throws IOException
    {
        try
        {
            Thread.sleep(500);//睡眠一下，否則可能出現各種錯誤狀態碼
        } catch (InterruptedException e)
        {
            e.printStackTrace();
        }

        Document doc = connect(parentElement.attr("abs:href"));
        if (doc != null)
        {
            Elements newsHeadlines = doc.select("tr." + cssMap.get(level));//
            // 獲取表格的一行數據
            for (Element element : newsHeadlines)
            {
                printInfo(parentElement,element, level + 1);
                Elements select = element.select("a");// 在遞歸調用的時候，這里是判斷是否是村一級的數據，村一級的數據沒有a標簽
                if (select.size() != 0)
                {
                    parseNextLevel(select.last(), level + 1);
                }
            }
        }
    }

    /**
     * 寫一行數據到數據文件中去
     * @param element 爬取到的數據元素
     * *  @param level 城市級別
    */
    private static void printInfo(Element parentElement,Element element, int level)
    {
        try
        {
            String reg = "[^\u4e00-\u9fa5]";

            if(level==3){
                bufferedWriter.write(parentElement.toString().replaceAll(reg, "")+"-["+(level-1)+"]"+element.select("td").last().text()+",");
            }else if(level==4){
                bufferedWriter.write(parentElement.toString().replaceAll(reg, "")+"-["+(level-1)+"]"+element.select("td").last().text()+",");
            }else if(level==5){
                bufferedWriter.write(parentElement.toString().replaceAll(reg, "")+"-["+(level-1)+"]"+element.select("td").last().text()+",");
            }

            bufferedWriter.newLine();
            bufferedWriter.flush();
        } catch (IOException e)
        {
            e.printStackTrace();
        }
    }

    private static void printInfoProvice(Element element, int level)
    {
        try
        {
            String reg = "[^\u4e00-\u9fa5]";

            bufferedWriter.write("province"+element.toString().replaceAll(reg, "")+",");
            bufferedWriter.newLine();
            bufferedWriter.flush();
        } catch (IOException e)
        {
            e.printStackTrace();
        }
    }

    private static Document connect(String url)
    {
        if (url == null || url.isEmpty())
        {
            throw new IllegalArgumentException("The input url('" + url + "') is invalid!");
        }
        try
        {
            return Jsoup.connect(url).timeout(100 * 1000).get();
        } catch (IOException e)
        {
            e.printStackTrace();
            return null;
        }
    }
}

 

獲取到的數據格式，我是根據我的數據表字段獲取的，獲取完之后再做處理存進數據表，中間會有部分地名亂碼，會在另一個博客中提供處理方式和數據表