获取省市区镇爬虫

 需要做分批导入数据库！ 需要做分批导入数据库！ 需要做分批导入数据库！

 

package com.mock.utils;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebClientOptions;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.justsy.army.mgt.mock.model.City;

public class NationalBureauOfStatics {
    private static final String ADDRESS = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/";
    private static final String fix = ".html";

    public static void main(String[] args) {
        List<City> provinceList = new ArrayList<>();
        List<City> cityList = new ArrayList<>();
        List<City> countyList = new ArrayList<>();
        List<City> townList = new ArrayList<>();
        provinceList = getTVMall(provinceList, new City(), ADDRESS, 0);
        for (City city : provinceList) {
            cityList = getTVMall(cityList, city, city.getHtmlAddr(), 1);
        }
        for (City city : cityList) {
            countyList = getTVMall(countyList, city, city.getHtmlAddr(), 2);
        }
        for (City city : countyList) {
            townList = getTVMall(townList, city, city.getHtmlAddr(), 3);

        for (City item: townList) {
            System.out.println(item.toString());
        }
        townList .clear();
        }
    }

    public static List<City> getTVMall(List<City> list, City city, String address, int type) {
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        // webclient参数载体
        WebClientOptions clientOptions = webClient.getOptions();
        // 设置webClient的相关参数
        clientOptions.setJavaScriptEnabled(true);
        clientOptions.setCssEnabled(false);
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
        clientOptions.setTimeout(35000);
        clientOptions.setThrowExceptionOnScriptError(false);
        try {
            HtmlPage htmlPage = webClient.getPage(address);
            Document dom = Jsoup.parse(htmlPage.asXml());
            Elements ele = null;
            if (type == 0) {
                ele = dom.getElementsByClass("provincetable");
            } else if (type == 1) {
                ele = dom.getElementsByClass("citytable");
            } else if (type == 2) {
                ele = dom.getElementsByClass("countytable");
            } else if (type == 3) {
                ele = dom.getElementsByClass("towntable");
            }
            dom = Jsoup.parse(ele.toString());
            ele = dom.getElementsByTag("tr");
            if (ele != null) {
                getList(list, ele, city, type);
            }
        } catch (FailingHttpStatusCodeException e) {
            e.printStackTrace();
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return list;
    }

    private static List<City> getList(List<City> list, Elements ele, City city, int type) {
        if (type == 0) {
            for (int i = 3; i < ele.size(); i++) {
                Element item = ele.get(i);
                Elements aElements = item.getElementsByTag("a");
                for (int j = 0; j < aElements.size(); j++) {
                    City c = new City();
                    String html = aElements.get(j).attr("href");
                    String name = aElements.get(j).text();
                    c.setProvince(name);
                    c.setHtmlAddr(ADDRESS + html);
                    c.setCode(html.replace(fix, "0000000000"));
                    list.add(c);
                }
            }
            return list;
        }
        for (int i = 0; i < ele.size(); i++) {
            Element item = ele.get(i);
            Elements aElements = item.getElementsByTag("a");
            if (aElements.size() > 0) {
                City c = new City();
                String html = aElements.get(0).attr("href");
                String code = aElements.get(0).text();
                String name = aElements.get(1).text();
                if (type == 1) {
                    c.setProvince(city.getProvince());
                    c.setCity(name);
                } else if (type == 2) {
                    c.setProvince(city.getProvince());
                    c.setCity(city.getCity());
                    c.setCounty(name);
                } else if (type == 3) {
                    c.setProvince(city.getProvince());
                    c.setCity(city.getCity());
                    c.setCounty(city.getCounty());
                    c.setTown(name);
                }
                c.setCode(code);
                String provinceCode = city.getCode().substring(0, 2);
                if (!html.startsWith(provinceCode + "/")) {
                    html = provinceCode + "/" + html;
                }
                c.setHtmlAddr(ADDRESS + html);
                list.add(c);
                System.out.println(c.toString());
            }
        }
        return list;
    }
}