Jsoup模拟表单提交爬取数据

package com.wl.getInfo;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import cn.hutool.core.text.StrBuilder;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


public class PhoneInfo {

    public static String LOGIN_URL = "https://phonedb.net/index.php?m=device&s=list";
    public static String PHONE_URL = "https://phonedb.net/";
    public static String USER_AGENT = "User-Agent";
    public static String USER_AGENT_VALUE = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";

    public static void main(String[] args) throws Exception {
        simulateLogin("SHV47");
    }

    public static Map<String,String> simulateLogin(String keyword) throws Exception {

        Connection con = Jsoup.connect(LOGIN_URL);  // 获取connection
        con.header(USER_AGENT, USER_AGENT_VALUE);   // 配置模拟浏览器
        Response rs = con.execute();                // 获取响应
        Document d1 = Jsoup.parse(rs.body());       // 转换为Dom树


        List<Element> eleList = d1.select("form");  // 获取提交form表单

        // 获取cooking和表单属性
        Map<String, String> datas = new HashMap<>();
        eleList.forEach(
                eList -> {
                    Elements allElements = eList.getAllElements();
                    allElements.forEach(allEle -> {
                                if (allElements.attr("name").equals("search_exp")) {
                                    allElements.attr("value", keyword);
                                }
                                // 排除空值表单属性
                                if (allElements.attr("name").length() > 0) {
                                    datas.put(allElements.attr("name"), allElements.attr("value"));
                                }
                            }
                    );
                }
        );

        /*
         * 第二次请求，以post方式提交表单数据以及cookie信息
         */
        Connection con2 = Jsoup.connect(LOGIN_URL);
        con2.header(USER_AGENT, USER_AGENT_VALUE);
        // 设置cookie和post上面的map数据
        Response response = con2.ignoreContentType(true).followRedirects(true).method(Method.POST).data(datas).cookies(rs.cookies()).execute();
        // 查询手机界面信息
        Document d = Jsoup.parse(response.body());
        // 获取href里面的内容
        String targetUrl = "";
        Elements div = d.getElementsByClass("content_block_title");
        for (Element element : div) {
            Elements link = element.getElementsByTag("a").eq(0);
            targetUrl += link.attr("href");
            System.out.println(link.attr("href"));
        }
        // 拼接访问地址
        String phoneUrl = PHONE_URL + targetUrl;
        /**
         * 第三次请求
         */
        Connection con3 = Jsoup.connect(phoneUrl);
        con2.header(USER_AGENT, USER_AGENT_VALUE);
        // 获取响应
        Response rs3 = con3.execute();
        // 转换为Dom树
        Document d3 = Jsoup.parse(rs3.body());

        // 提取数据

        // 先获取tr
        List<Element> tr = d3.select("tr");
        Map<String, String> map = new HashMap();
        new StrBuilder();
        for (Element td : tr) {
            String strong = td.getElementsByTag("strong").eq(0).text();
            if (strong.equals("Brand")) {
                String value = td.getElementsByTag("a").eq(1).text();
                map.put("Brand", value);
            }
            if (strong.equals("Model")) {
                String value = FilterText("Model", td.text());
                map.put("Model", value);
            }
            if (strong.equals("Platform")) {
                String value = td.getElementsByTag("a").eq(1).text();
                map.put("os", value);
            }
            if (strong.equals("Resolution")) {
                String value = FilterText("Resolution", td.text());
                map.put("screen", value);
            }
            if (strong.equals("Pixel Density")) {
                String value = td.getElementsByTag("a").eq(1).text();
                map.put("ppi", value);
            }
            if (strong.equals("RAM Capacity (converted)")) {
                String value = td.getElementsByTag("a").eq(1).text();
                map.put("RAM_GB", value);
            }
            if (strong.equals("CPU")) {
                String value = td.getElementsByTag("a").eq(1).text();
                map.put("chipset", value);
            }
            if (strong.equals("Graphical Controller")) {
                String value = td.getElementsByTag("a").eq(1).text();
                map.put("GPU", value);
            }
            if (strong.equals("Released")) {
                String value = FilterText("Released", td.text());
                map.put("released", value);
            }
        }
        return map;

    }

    private static String FilterText(String target, String text) {
        return text.replaceAll(target, "").trim();
    }

}

由于这个网站爬去的速度较慢，并且在爬取的过程中可能会出现异常行为导致丢失数据，于是我是用了多线程加上异常处理
改进后的工具类如下

package com.oasis.mdata.services

import com.oasis.mdata.entities.GameInfo
import com.oasis.mdata.entities.PhoneInfo
import org.jsoup.Connection
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import java.util.function.Consumer


/**
 *@author 没有梦想的java菜鸟
 * @date 2022/01/21 11:05 上午
 */
class JsoupUtils {
    companion object {
        // 定义爬取信息
        val Phone_Url = "https://phonedb.net/index.php?m=device&s=list"
        val PHONE_URL = "https://phonedb.net/"
        val Game_URL="https://www.qimai.cn/rank/index/brand/grossing/device/iphone/country/us/genre/6014/date"
        val USER_AGENT = "User-Agent"

        //        val USER_AGENT_VALUE = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0"
        val USER_AGENT_VALUE =
            "Mozilla/5.0 (Windows; U; WindowsNT 5.1; en-US; rv1.8.1.6) Gecko/20070725 Firefox/2.0.0.6"


        fun getPhoneinfo(keyword: String?): PhoneInfo {
            var conn: Connection = Jsoup.connect(Phone_Url)
            conn.header(USER_AGENT, USER_AGENT_VALUE)
            // 创建集合
            val phoneInfo = PhoneInfo()
            // 获取响应
            val response = conn.execute()
            // 转换为dom
            val dom: Document = Jsoup.parse(response.body())
            // 获取提交form表单
            val eleList: List<Element> = dom.select("form")
            val datas = HashMap<String, String>()
            // 搜索赋值
            eleList.forEach(
                Consumer { ele: Element ->
                    val allElements = ele.allElements
                    allElements.forEach(
                        Consumer { allEle ->
                            if (allElements.attr("name").equals("search_exp")) {
                                allElements.attr("value", keyword);
                            }
                            // 排除空值表单属性

                            if (allElements.attr("name").length > 0) {
                                datas.put(allElements.attr("name"), allElements.attr("value"));
                            }
                        }
                    )
                }
            )

            /**
             * 第二次请求
             */
            val con2 = Jsoup.connect(Phone_Url)
            con2.header(USER_AGENT, USER_AGENT_VALUE)
            // 表单提交
            val response2 = con2.ignoreContentType(true)
                .followRedirects(true)
                .method(Connection.Method.POST).data(datas)
                .cookies(response.cookies())
                .execute()
            // 查询手机界面信息
            val document = Jsoup.parse(response2.body())
            var targetUrl = ""
            var phoneUrl = ""
            var flag = false
            // 获取div元素
            val div = document.getElementsByClass("content_block_title")
            if (div.size == 1) {
                for (element in div) {
                    val link = element.getElementsByTag("a").eq(0)
                    // 获取a标签里面的链接
                    targetUrl += link.attr("href")
                }
                // 拼接地址
                phoneUrl = PHONE_URL + targetUrl
                Execute(phoneUrl, phoneInfo)
                phoneInfo.to_link = keyword
                phoneInfo.status = "0"

            } else if (div.size > 1) {// 说明有多种结果
                for (element in div) {
                    val link = element.getElementsByTag("a").eq(0)
                    val s = link.text()
                    if (keyword?.let { s.contains(it) } == true) {
                        // 获取a标签里面的链接
                        targetUrl += link.attr("href")
                        flag = true
                    }
                    break
                }
                if (flag) {
                    // 拼接地址
                    phoneUrl = PHONE_URL + targetUrl
                    Execute(phoneUrl, phoneInfo)
                    phoneInfo.to_link = keyword
                    phoneInfo.status = "1"
                } else {
                    phoneInfo.to_link = keyword
                    phoneInfo.status = "2"
                }

            } else {// 说明没有此型号
                phoneInfo.to_link = keyword
                phoneInfo.status = "2"
            }
            return phoneInfo
        }

        private fun CheckDate(values: String): String {
            val Month = values.replace(values.substring(0, 4), "").trim()
            val sb = StringBuilder(values.replace(Month, "")?.trim())
            CheckQuarter(Month, sb)
            return sb.toString()

        }

        private fun FilterText(target: String, text: String): String {
            return text.replace(target, "").trim()
        }

        private fun Execute(phoneUrl: String, phoneInfo: PhoneInfo) {
            /**
             * 第三次请求
             */
            val conn3 = Jsoup.connect(phoneUrl)
            conn3.header(USER_AGENT, USER_AGENT_VALUE)
            val response3 = conn3.execute()
            val document3 = Jsoup.parse(response3.body())

            val tr: List<Element> = document3.select("tr")

            for (td in tr) {
                val strong = td.getElementsByTag("strong").eq(0).text()
                if (strong == "Brand") {
                    val value = td.getElementsByTag("a").eq(1).text()
                    phoneInfo.brand = value
                }
                if (strong == "Model") {
                    val value = FilterText("Model", td.text())
                    phoneInfo.model = value
                }
                if (strong == "Platform") {
                    val value = td.getElementsByTag("a").eq(1).text()
                    phoneInfo.os = value
                }
                if (strong == "Resolution") {
                    val value = FilterText("Resolution", td.text())
                    phoneInfo.screen = value
                }
                if (strong == "Pixel Density") {
                    val values = td.getElementsByTag("a").eq(1).text()
                    val value = FilterText("PPI", values)
                    phoneInfo.ppi = value
                }
                if (strong == "RAM Capacity (converted)") {
                    val values = td.getElementsByTag("a").eq(1).text()
                    val value = FilterText("GiB RAM", values)
                    phoneInfo.ram_gb = value
                }
                if (strong == "CPU") {
                    val values = td.getElementsByTag("a").eq(1).text()
                    val value = values.split(",")[0]
                    phoneInfo.chipset = value
                }
                if (strong == "Graphical Controller") {
                    val value = td.getElementsByTag("a").eq(1).text()
                    phoneInfo.GPU = value
                }
                if (strong == "Released") {
                    val values = FilterText("Released", td.text())
                    val value = CheckDate(values)
                    phoneInfo.released = value
                }
            }
        }

        private fun CheckQuarter(Month: String, sb: StringBuilder) {
            if (Month.contains("Jan")) {
                sb.append("Q1")
            } else if (Month.contains("Feb")) {
                sb.append("Q1")
            } else if (Month.contains("Mar")) {
                sb.append("Q1")
            } else if (Month.contains("Apr")) {
                sb.append("Q2")
            } else if (Month.contains("May")) {
                sb.append("Q2")
            } else if (Month.contains("Jun")) {
                sb.append("Q2")
            } else if (Month.contains("Jul")) {
                sb.append("Q3")
            } else if (Month.contains("Aug")) {
                sb.append("Q3")
            } else if (Month.contains("Sep")) {
                sb.append("Q3")
            } else if (Month.contains("Oct")) {
                sb.append("Q4")
            } else if (Month.contains("Nov")) {
                sb.append("Q4")
            } else if (Month.contains("Dec")) {
                sb.append("Q4")
            }
        }
    }
}

我将 需要爬取的 关键词 放到 phones.txt中，然后用代码将其中的关键词 分割成等份的txt文件，并把生成的txt文件放到lastResolve.txt中 ，多线程处理读取lastResolve.txt中的 txt文件所对应的内容。

分割文件的类

package com.oasis.mdata.services

import java.io.BufferedReader
import java.io.FileReader
import java.io.FileWriter
import java.io.PrintWriter
import java.util.*
import kotlin.collections.ArrayList

/**
 *@author 没有梦想的java菜鸟
 * @date 2022/02/28 6:10 下午
 */
class SubFile {

    fun subFile(filePath: String):Int {
        val buffered = BufferedReader(FileReader(filePath))
        var line: String = ""
        val list = ArrayList<String>()
        // 读取目标文件放进集合
        while ((buffered.readLine()?.also { line = it }) != null) {
            list.add(line)
        }

        // 将集合分为若干个集合
        val subList = subList(list, list.size /6)
        val uuidFileList = ArrayList<String>()
        subList.map {
            // 将集合中的内容写进txt文件中
            val uuid = UUID.randomUUID().toString().substring(0 until 10)
            // 记录生成的文件
            uuidFileList.add("$uuid.txt")
            val pw = PrintWriter(FileWriter("src/main/resources/$uuid.txt"), true)
            it.map { str ->
                pw.println(str)
                pw.flush()
            }
            pw.close()
        }
        // 写进要处理的文件里
        val pw = PrintWriter(FileWriter("src/main/resources/lastResolve.txt"), true)
        uuidFileList.map {
            pw.println(it)
            pw.flush()
        }
        pw.close()
        return if (list.size%(list.size/6)==0 && list.size>=6) list.size/(list.size/6) else list.size/(list.size/6)+1
    }


    fun subList(subList: MutableList<String>, subNumber: Int): MutableList<MutableList<String>> {
        var beginIndex = 0
        var endIndex = subNumber
        val list = ArrayList<MutableList<String>>()
        var totalCount = subList.size / subNumber

        for (i in 0..totalCount) {
            if (i == totalCount) {
                subList.subList(beginIndex, subList.size).let {
                    if (it.isNotEmpty()) {
                        list.add(it)
                    }
                }

            } else {
                list.add(subList.subList(beginIndex, endIndex))
                beginIndex = endIndex
                endIndex += subNumber
            }
        }
        return list
    }
}

fun main() {
    SubFile().subFile("src/main/resources/test.txt")
}

多线程处理的类

package com.oasis.mdata.services

import com.alibaba.excel.EasyExcel
import com.oasis.mdata.entities.PhoneInfo
import com.oasis.mdata.services.JsoupUtils
import java.io.BufferedReader
import java.io.File
import java.io.FileReader
import java.util.*
import java.util.concurrent.*
import kotlin.collections.ArrayList

/**
 *@author 没有梦想的java菜鸟
 * @date 2022/02/22 2:00 下午
 */
class Tests {

}

val threadPool = ThreadPoolExecutor(
    8,
    10,
    20,
    TimeUnit.SECONDS,
    LinkedBlockingQueue(10),
    Executors.defaultThreadFactory(),
    ThreadPoolExecutor.AbortPolicy()
)

fun main() {
    val beginTime = Date()
    // 将主文件生成多个txt文件用于多线程读取
    val threadNumber=SubFile().subFile("src/main/resources/phones.txt")
    val buffered = BufferedReader(FileReader("src/main/resources/lastResolve.txt"))
    var line: String = ""
    // 线程安全集合 存储excel的位置
    val excelList = CopyOnWriteArrayList<String>()
    // 线程安全的计数器
    val countDownLatch=CountDownLatch(threadNumber)
    // 记录临时文件
    val temFileList=ArrayList<String>()
    while ((buffered.readLine()?.also { line = it }) != null) {
        Thread.sleep(10)
        temFileList.add(line)
        read(line, excelList,countDownLatch)
    }
    countDownLatch.await()
    // 删除生成为临时txt文件
    temFileList.map {
        File("src/main/resources/$it").delete()
    }
    val fileName = "src/main/resources/手机信息.xlsx"
    val listener = Listener()
    excelList.map {
        // 删除临时的excel
        EasyExcel.read(it, PhoneInfo::class.java, listener).sheet().doRead()
        File(it).delete()
    }
    println(excelList.size)
    EasyExcel.write(fileName, PhoneInfo::class.java).sheet("PhoneInfo").doWrite(listener.list)


    println("程序运行结束")
    val endTime = Date()
    val time = (endTime.time - beginTime.time) / 1000
    println("程序共耗时 $time 秒")
}

fun read(txt: String?, excelList: MutableList<String>,countDownLatch:CountDownLatch) {
    threadPool.execute {
        val beginTime = Date()
        val path = "src/main/resources/"
        val fileName = path + System.currentTimeMillis() + "手机信息.xlsx"
        excelList.add(fileName)
        val buffered = BufferedReader(FileReader("$path/$txt"))
        var line: String = ""
        val list = ArrayList<PhoneInfo>()
        val tempList = ArrayList<String>()
        while ((buffered.readLine()?.also { line = it }) != null) {
            tempList.add(line)
        }
        // 循环读取 出异常 继续读取
        do {
            val iterator = tempList.iterator()
            while (iterator.hasNext()) {
                var info = iterator.next()
                try {
                    val phoneInfo = JsoupUtils.getPhoneinfo(info)
                    list.add(phoneInfo)
                    iterator.remove()
                    println(info)
                } catch (e: Exception) {
                    println("$info 异常")
                }
            }
        } while (tempList.isNotEmpty())
        // 写自己的临时文件
        EasyExcel.write(fileName, PhoneInfo::class.java).sheet("PhoneInfo").doWrite(list)
        val endTime = Date()
        val time = (endTime.time - beginTime.time) / 1000
        println("读取$txt 共耗费 $time 秒")
        countDownLatch.countDown()
    }


}