java爬虫

创建Maven工程

加入依赖

<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
<dependency>
    <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpclient</artifactId>
    <version>4.5.13</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
<dependency>
    <groupId>org.slf4j</groupId>
    <artifactId>slf4j-log4j12</artifactId>
    <version>1.7.25</version>
</dependency>

加入log4j.properties

log4j.rootLogger=DEBUG,A1
log4j.logger.cn.itcast = DEBUG

log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c]-[%p] %m%n

编写代码

package com.blfsg.test;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class CrawlerTest {
    public static void main(String[] args) throws IOException {
        // 创建httpclient
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 声明访问地址
        HttpGet httpGet = new HttpGet("http://www.yxans.cn");
        // 发起请求
        CloseableHttpResponse response = httpClient.execute(httpGet);
        // 判断请求是否成功
        if (response.getStatusLine().getStatusCode() == 200) {
            // 解析数据
            String content = EntityUtils.toString(response.getEntity(), "UTF-8");
            System.out.println(content);
        }
    }
}

GET请求

package com.blfsg.test;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class CrawlerGetTest {
    public static void main(String[] args) throws IOException {
        // 创建httpclient
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 声明访问地址
        HttpGet httpGet = new HttpGet("http://www.yxans.cn");
        CloseableHttpResponse response = null;
        try {
            // 发送请求
            response = httpClient.execute(httpGet);
            // 判断响应码
            if (response.getStatusLine().getStatusCode() == 200) {
                // 解析数据
                String content = EntityUtils.toString(response.getEntity(), "utf-8");
                System.out.println(content.length());
            }

        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            // 释放连接
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            httpClient.close();
        }
    }
}

带参数的GET请求

package com.blfsg.test;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.net.URI;

public class CrawlerGetWithParamTest {
    public static void main(String[] args) throws Exception {
        // 创建httpclient
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 创建URIBuilder
        URIBuilder uriBuilder = new URIBuilder("https://cn.bing.com/search");
        // 添加参数
        uriBuilder.addParameter("q","世界杯");
        // 生成访问地址
        URI uri = uriBuilder.build();
        // 创建httpGet对象
        HttpGet httpGet = new HttpGet(uri);
        CloseableHttpResponse response = null;
        try {
            // 发送请求
            response = httpClient.execute(httpGet);
            // 判断状态码
            if (response.getStatusLine().getStatusCode() == 200) {
                // 解析数据
                String content = EntityUtils.toString(response.getEntity(), "utf-8");
                System.out.println(content.length());
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            // 关闭连接
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            httpClient.close();
        }
    }
}

POST请求

package com.blfsg.test;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class CrawlerPostTest {
    public static void main(String[] args) throws IOException {
        // 创建httpclient
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 声明访问地址
        HttpPost httpPost = new HttpPost("http://www.yxans.cn");
        CloseableHttpResponse response = null;
        try {
            // 发起请求
            response = httpClient.execute(httpPost);
            // 判断状态码
            if (response.getStatusLine().getStatusCode() == 200) {
                // 解析数据
                String content = EntityUtils.toString(response.getEntity(), "utf-8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            httpClient.close();
        }
    }
}

带参数的POST的请求

package com.blfsg.test;

import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class CrawlerPostWithparamTest {
    public static void main(String[] args) throws Exception {
        // 创建httpclient
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 声明访问地址
        HttpPost httpPost = new HttpPost("https://www.oschina.net");
        // 设置User-Agent属性，解决开源中国限制的问题
        httpPost.setHeader("User-Agent", "");
        // 设置参数
        List<NameValuePair> params = new ArrayList<NameValuePair>();
        params.add(new BasicNameValuePair("scope", "project"));
        params.add(new BasicNameValuePair("q", "HttpClient"));
        // 创建form表单实体
        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params, "UTF-8");
        // 设置表单到httpPost中
        httpPost.setEntity(formEntity);
        CloseableHttpResponse response = null;
        try {
            // 发起请求
            response = httpClient.execute(httpPost);

            // 判断状态码是否是200
            if (response.getStatusLine().getStatusCode() == 200) {
                // 解析数据
                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
                System.out.println(content);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            // 释放连接
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                httpClient.close();
            }
        }
    }
}

连接池

package com.blfsg.test;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class CrawLerPoolTest {
    public static void main(String[] args) {
        // 创建连接池
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
        // 设置最大连接数
        cm.setMaxTotal(100);
        // 设置每个主机的并发数
        cm.setDefaultMaxPerRoute(10);
        doGet(cm);
        doGet(cm);
    }

    private static void doGet(PoolingHttpClientConnectionManager cm) {
        // 获取连接
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
        // 声明访问地址
        HttpGet httpGet = new HttpGet("http://www.yxans.cn");
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);
            // 判断状态码
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "utf-8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
}

请求参数

package com.blfsg.test;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class CrawlerParamTest {
    public static void main(String[] args) {
        // 创建httpclient
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 声明访问地址
        HttpGet httpGet = new HttpGet("http://www.yxans.cn");
        // 设置参数信息
        RequestConfig config = RequestConfig.custom().setConnectTimeout(1000) // 创建连接最大时长，单位毫秒
                .setConnectionRequestTimeout(500)   // 获取连接最大时长
                .setSocketTimeout(10 * 1000)   // 数据传输的最长时间
                .build();
        // 配置参数信息
        httpGet.setConfig(config);
        CloseableHttpResponse response = null;
        try {
            // 发起请求
            response = httpClient.execute(httpGet);

            // 判断状态码是否是200
            if (response.getStatusLine().getStatusCode() == 200) {
                // 解析数据
                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
                System.out.println(content.length());
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            // 释放连接
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
}

使用jsoup

加入依赖

<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency>
    <groupId>org.jsoup</groupId>
    <artifactId>jsoup</artifactId>
    <version>1.15.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
    <groupId>org.apache.commons</groupId>
    <artifactId>commons-lang3</artifactId>
    <version>3.12.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
<dependency>
    <groupId>commons-io</groupId>
    <artifactId>commons-io</artifactId>
    <version>2.11.0</version>
</dependency>

jsoup输入

import org.apache.commons.io.FileUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.junit.Test;

import java.io.File;
import java.io.IOException;
import java.net.URL;

public class JsoupTest {
    // 解析url
    @Test
    public void TestUrl() throws IOException {
        // jsoup解析url
        Document document = Jsoup.parse(new URL("http://www.yxans.cn"),10000);
        // 把获取的内容写入文件
        FileUtils.writeStringToFile(new File("D:/test.html"),document.toString(),"utf-8");
        // 获取title
        Element title = document.getElementsByTag("title").first();
        System.out.println(title.text());
    }
    // 输入字符串
    @Test
    public void TestString() throws IOException {
        // 读取文件，获取字符串
        String html = FileUtils.readFileToString(new File("D:/test.html"),"utf-8");
        // jsoup解析字符串
        Document document = Jsoup.parse(html);
        // 获取title标签
        Element element = document.getElementsByTag("title").first();
        // 打印title内容
        System.out.println(element.text());
    }
    // 输入文件
    @Test
    public void TestDoc() throws IOException {
        // 使用jsoup解析文件
        Document document = Jsoup.parse(new File("D:/test.html"), "utf-8");
        // 获取title标签
        Element title = document.getElementsByTag("title").first();
        // 打印title内容
        System.out.println(title);
    }
}
posted @   不浪费时光  阅读(42)  评论(0)  编辑  收藏  举报
相关博文：
·  java项目部署
·  IDEA创建Mybatis逆向工程
·  使用Java实现爬虫
·  新冠疫情防控指挥作战平台项目
·  java学习之爬虫
阅读排行：
· 震惊！C++程序真的从main开始吗？99%的程序员都答错了
· 【硬核科普】Trae如何「偷看」你的代码？零基础破解AI编程运行原理
· 单元测试从入门到精通
· 上周热点回顾（3.3-3.9）
· Vue3状态管理终极指南：Pinia保姆级教程
点击右上角即可分享
微信分享提示
AI IDE Trae