【反反爬】使用Jsoup爬取数据保存Excel

 　　本文主要使用Jsoup爬取XXX房屋信息，抓取一些房屋信息，比如房屋楼盘、户型、价格、地址等信息，然后保存到Excel，便于对比和筛选，选出符合预期的好房。

　　注意，有些网站有防爬取机制，需要设置动态更换代理IP进行重试。

        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>1.7.25</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <version>1.7.25</version>
        </dependency>

        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.11.3</version>
        </dependency>

        <dependency>
            <groupId>com.squareup.okhttp3</groupId>
            <artifactId>okhttp</artifactId>
            <version>3.3.0</version>
        </dependency>

        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi</artifactId>
            <version>4.1.2</version>
        </dependency>

        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi-ooxml</artifactId>
            <version>4.1.2</version>
        </dependency>

 

爬取数据的部分代码

public static List<List<String>> getData(String urls) throws Exception {

        // 读取数据
        List<List<String>> data = Lists.newArrayList();

        // 代理ip和端口，需动态替换（可以本地新建一个ip列表，动态读取，获取失败替换代理ip即可）
        String ip = "xxx.xxx.xxx.xxx";
        int port = 80;

        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(ip, port));
        URL url = new URL(urls);
        URLConnection urlConnection = url.openConnection(proxy);
        urlConnection.setUseCaches(false);
        urlConnection.connect();
        InputStream is = urlConnection.getInputStream();
        BufferedReader buffer = new BufferedReader(new InputStreamReader(is));
        StringBuffer bs = new StringBuffer();
        String l = null;
        while ((l = buffer.readLine()) != null) {
            bs.append(l);
        }
        System.out.println(bs.toString());

        Document doc = Jsoup.parse(bs.toString());
        Elements els = doc.body().getElementsByClass("list-item");
        for (Element el : els) {
            List<String> rowData = Lists.newArrayList();
            Elements titleEls = el.getElementsByClass("house-title");
            log.info("# 标题:{}", titleEls.get(0).getElementsByTag("a").text());
            rowData.add(titleEls.get(0).getElementsByTag("a").text());

            Elements itemEls = el.getElementsByClass("details-item");
            Elements itemSpanEls = itemEls.get(0).getElementsByTag("span");
            log.info("# 户型:{}", itemSpanEls.get(0).text());
            rowData.add(itemSpanEls.get(0).text());
            log.info("# 面积:{}", itemSpanEls.get(1).text());
            rowData.add(itemSpanEls.get(1).text());
            log.info("# 楼层:{}", itemSpanEls.get(2).text());
            rowData.add(itemSpanEls.get(2).text());
            log.info("# 年限:{}", itemSpanEls.get(3).text());
            rowData.add(itemSpanEls.get(3).text());

            String[] address = itemEls.get(1).getElementsByTag("span").text().split(" ");
            if (address==null || address.length == 0) {
                continue;
            }
            log.info("# 楼盘:{}", address[0]);
            log.info("# 地址:{}", address[1]);
            rowData.add(address[0]);
            rowData.add(address[1]);

            Elements priceEls = el.getElementsByClass("pro-price");
            Elements priceSpanEls = priceEls.get(0).getElementsByTag("span");
            log.info("# 总价:{}", priceSpanEls.get(0).getElementsByTag("strong").text());
            rowData.add(priceSpanEls.get(0).getElementsByTag("strong").text());
            log.info("# 单价:{}", priceSpanEls.get(1).text());
            rowData.add(priceSpanEls.get(1).text());
            data.add(rowData);
        }

        return data;
    }

 

数据写入excel的代码

public static void writeExcel(List<String> titleList, List<List<String>> dataList) throws Exception {
        //open file.
        File excel = new File("D:\\Users\\Desktop\\data.xls");
        excel.deleteOnExit();
        excel.createNewFile();
        FileOutputStream fos = new FileOutputStream(excel);

        Workbook book = new HSSFWorkbook();

        //create Sheet named "Sheet_1". 0 means this is 1st page.
        Sheet sheet = book.createSheet("安居客房源信息");

        // 写入标题
        Row titleRow = sheet.createRow(0);
        for (int x = 0; x < titleList.size(); x++) {
            Cell cell0 = titleRow.createCell(x);
            cell0.setCellValue(titleList.get(x));
        }

        // 写入数据
        for (int i = 0; i < dataList.size(); i++) {
            int row = i + 1;
            Row dataRow = sheet.createRow(row);
            List<String> rowData = dataList.get(i);
            for (int j = 0; j < titleList.size(); j++) {
                Cell dataCell = dataRow.createCell(j);
                dataCell.setCellValue(rowData.get(j));
            }
        }

        book.write(fos);
        book.close();

        log.info("# write data success");
    }

 

运行方法（建议第一次先不获取所有数据，只获取第一页数据，用来看实际效果，不然ip被封了无法继续使用）

public static void main(String[] args) {
        try {
            String url = "file:///D:/Users/Desktop/test.html";
//            url = "https://hanchuanshi.anjuke.com/sale/p1-rd1/#filtersort";

            List<List<String>> data = Lists.newArrayList();
            for (int i = 1; i <= 50; i++) {
                url = "https://hanchuanshi.anjuke.com/sale/p" + i + "-rd1/#filtersort";
                data.addAll(getData(url));
            }

            List<String> titleList = Arrays.asList("标题", "户型", "面积", "楼层", "年限", "楼盘", "地址", "总价", "单价");
            writeExcel(titleList, data);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }