通用HttpClient请求工具类

前言

现在大多数Java项目开发中，经常都用通过HTTP协议来调用网络资源数据（1、爬虫爬取网页数据；2、请求第三方系统进行数据交互等），虽然JDK8及以前的版本，也提供了响应的请求工具包，但是使用起来很不灵活，所以大多数都是采用Apache的HttpClient包来封装自己的请求工具类，方便整个项目开发使用。

使用流程

1、引入maven依赖

<dependency>
    <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpclient</artifactId>
    <version>4.5.13</version>
</dependency>

<dependency>
    <groupId>com.alibaba</groupId>
    <artifactId>fastjson</artifactId>
    <version>1.2.75</version>
</dependency>

2、HttpClient使用

2.1 创建请求客户端对象

/*
 1、创建自定义请求客户端构建对象
 */
HttpClientBuilder httpClientBuilder = HttpClients.custom();
// 创建连接池，并对连接池进行设置后赋值给请求对象构造器
HttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
httpClientBuilder.setConnectionManager(cm);
CloseableHttpClient build = httpClientBuilder.build();
/*
 2、创建默认的请求客户端构建对象
 */
CloseableHttpClient request2 = HttpClients.createDefault();

2.2 创建连接池管理对象

// 创建请求连接池管理，可创建空的，也可以创建带注册对象（注册对象示例：https的连接认证注册）
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
// 设置连接池最大连接数（设置时可以设置部署机器的CPU线程数）
cm.setMaxTotal(8);
// 设置每个路由最大默认连接数（可以认为一个域名就是路由）
cm.setDefaultMaxPerRoute(8);

2.3 请求方式对象

// get请求对象
HttpGet httpGet = new HttpGet(url);
// post请求对象
HttpPost httpPost = new HttpPost(url);

2.4 请求头设置

// 设置用户代理为浏览器
httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Edg/94.0.992.38");
// 设置请求的数据类型
httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");

2.5 参数设置

注意：随着请求方式的不同，传参方式也大不相同

/*
 1、get请求
 */
String url = "url";
Map<String, String> params = new HashMap<>();
params.put("username", "chris");
URIBuilder uriBuilder = new URIBuilder(url);
if (params!=null) { // 判空操作要有，避免空指针异常
    for (String paramKey : params.keySet()) {
        uriBuilder.addParameter(paramKey, params.get(paramKey));
    }
}
HttpGet httpGet = new HttpGet(uriBuilder.build());

/*
 2、post请求（表单形式：application/x-www-form-urlencoded）
 */
String url = "url";
Map<String, String> params = new HashMap<>();
params.put("username", "chris");
HttpPost httpPost = new HttpPost(url);
if (params!=null) { // 判空操作要有，避免空指针异常
    List<NameValuePair> nvList = new ArrayList<>(params.size());
    NameValuePair nv = null;
    for (String paramKey : params.keySet()) {
　　　　 // 构建参数键值对象
        nv = new BasicNameValuePair(paramKey, params.get(paramKey));
        nvList.add(nv);
    }
    // 传送参数的对象
    HttpEntity paramsEntity = new UrlEncodedFormEntity(nvList, StandardCharsets.UTF_8);
    // 设置参数
    httpPost.setEntity(paramsEntity);
}

/*
 3、post请求（json形式：application/json）
 */
String url = "url";
Map<String, String> params = new HashMap<>();
params.put("username", "chris");
HttpPost httpPost = new HttpPost(url);
if (params!=null) { // 判空操作要有，避免空指针异常
    // 需要将请求对象转换为json字符串形式
    HttpEntity paramEntity = new StringEntity(JSON.toJSONString(params), StandardCharsets.UTF_8);
    httpPost.setEntity(paramEntity);
}

2.6 执行请求并处理返回对象

CloseableHttpResponse response = null;
try {
    // 执行请求
    response = closeableHttpClient.execute(httpPost);
    int statusCode = response.getStatusLine().getStatusCode();
    // 判断请求响应是否成功
    if (statusCode== HttpStatus.SC_OK) { 
        // 处理响应数据并返回
        HttpEntity entity = response.getEntity();
        return EntityUtils.toString(entity, StandardCharsets.UTF_8);
    } else {
        log.error("请求地址({})失败:{}", url, statusCode);
    }
} catch (IOException e) {
    log.error("请求地址({})失败", url, e);
    throw new RuntimeException("请求地址("+url+")失败");
} finally { // 确认数据消费并关闭http响应对象
    HttpClientUtils.closeQuietly(response);
}

3、书写请求工具类

/**
 * http请求工具类
 */
@Slf4j
public class HttpUtil {

    /**
     * 请求连接构造对象
     */
    private static final HttpClientBuilder httpClientBuilder = HttpClients.custom();

    /**
     * 连接池最大连接数
     */
    private static final int MAX_TOTAL = 8;

    /**
     * 每个路由最大默认连接数
     */
    private static final int DEFAULT_MAX_RER_ROUTE = 8;

    /**
     * 获取连接获取超时时间
     */
    private static final int CONNECTION_REQUEST_TIMEOUT = 2000;

    /**
     * 连接超时时间
     */
    private static final int CONNECTION_TIMEOUT = 2000;

    /**
     * 数据响应超时时间
     */
    private static final int SOCKET_TIMEOUT = 10000;



    static {
        /*
         1、绕开不安全的https请求的证书验证(不需要可以注释，然后使用空参数的PoolingHttpClientConnectionManager构造连接池管理对象)
         */
        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("http", PlainConnectionSocketFactory.INSTANCE)
                .register("https", trustHttpsCertificates())
                .build();

        /*
         2、创建请求连接池管理
         */
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
        // 设置连接池最大连接数
        cm.setMaxTotal(MAX_TOTAL);
        // 设置每个路由最大默认连接数
        cm.setDefaultMaxPerRoute(DEFAULT_MAX_RER_ROUTE);
        httpClientBuilder.setConnectionManager(cm);

        /*
        3、设置默认请求配置
         */
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT) // 设置获取连接获取超时时间
                .setConnectTimeout(CONNECTION_TIMEOUT) // 设置连接超时时间
                .setSocketTimeout(SOCKET_TIMEOUT) // 设置数据响应超时时间
                .build();
        httpClientBuilder.setDefaultRequestConfig(requestConfig);
    }


    /**
     *  执行get请求（网页）
     * @param url 请求地址(含有特殊符号需要URLEncoder编码)
     * @param headers 请求头参数
     * @return 响应数据
     */
    public static String getPage(String url, Map<String, String> headers) {

        CloseableHttpClient closeableHttpClient = httpClientBuilder.build();
        HttpGet httpGet = new HttpGet(url);

        // 请求头设置，如果常用的请求头设置，也可以写死，特殊的请求才传入
        httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Edg/94.0.992.38");
        if (headers != null) {
            for (String headerKey : headers.keySet()) {
                httpGet.setHeader(headerKey, headers.get(headerKey));
            }
        }

        CloseableHttpResponse response = null;
        try {
            response = closeableHttpClient.execute(httpGet);
            int statusCode = response.getStatusLine().getStatusCode();
            if (statusCode== HttpStatus.SC_OK) { // 请求响应成功
                HttpEntity entity = response.getEntity();
                return EntityUtils.toString(entity, StandardCharsets.UTF_8);
            } else {
                log.error("请求地址({})失败:{}", url, statusCode);
            }
        } catch (Exception e) {
            log.error("请求地址({})失败", url, e);
            throw new RuntimeException("请求地址("+url+")失败");
        } finally {
            HttpClientUtils.closeQuietly(response);
        }
        return null;
    }

    /**
     *  执行post请求（form表单）
     * @param url 请求地址
     * @param headers 请求头参数
     * @return 响应数据
     */
    public static String postForm(String url, Map<String, String> headers, Map<String, String> params) {
        CloseableHttpClient closeableHttpClient = httpClientBuilder.build();
        HttpPost httpPost = new HttpPost(url);
         // 请求头设置，如果常用的请求头设置，也可以写死，特殊的请求才传入
        httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Edg/94.0.992.38");
        httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
        if (headers != null) {
            for (String headerKey : headers.keySet()) {
                httpPost.setHeader(headerKey, headers.get(headerKey));
            }
        }

        // 设置请求参数
        if (params!=null) {
            List<NameValuePair> nvList = new ArrayList<>(params.size());
            for (String paramKey : params.keySet()) {
                NameValuePair nv = new BasicNameValuePair(paramKey, params.get(paramKey));
                nvList.add(nv);
            }
            HttpEntity paramsEntity = new UrlEncodedFormEntity(nvList, StandardCharsets.UTF_8);
            httpPost.setEntity(paramsEntity);
        }

        CloseableHttpResponse response = null;
        try {
            response = closeableHttpClient.execute(httpPost);
            int statusCode = response.getStatusLine().getStatusCode();
            if (statusCode== HttpStatus.SC_OK) { // 请求响应成功
                HttpEntity entity = response.getEntity();
                return EntityUtils.toString(entity, StandardCharsets.UTF_8);
            } else {
                log.error("请求地址({})失败:{}", url, statusCode);
            }
        } catch (IOException e) {
            log.error("请求地址({})失败", url, e);
            throw new RuntimeException("请求地址("+url+")失败");
        } finally {
            HttpClientUtils.closeQuietly(response);
        }
        return null;
    }

    /**
     *  执行post请求（接口）
     * @param url 请求地址
     * @param headers 请求头参数
     * @return 响应数据
     */
    public static String getJson(String url, Map<String, String> headers) {
        CloseableHttpClient closeableHttpClient = httpClientBuilder.build();
        HttpGet httpGet = new HttpGet(url);
        // 请求头设置，如果常用的请求头设置，也可以写死，特殊的请求才传入
        if (headers != null) {
            for (String headerKey : headers.keySet()) {
                httpGet.setHeader(headerKey, headers.get(headerKey));
            }
        }
        CloseableHttpResponse response = null;
        try {
            response = closeableHttpClient.execute(httpGet);
            int statusCode = response.getStatusLine().getStatusCode();
            if (statusCode== HttpStatus.SC_OK) { // 请求响应成功
                HttpEntity entity = response.getEntity();
                return EntityUtils.toString(entity, StandardCharsets.UTF_8);
            } else {
                log.error("请求地址({})失败:{}", url, statusCode);
            }
        } catch (IOException e) {
            log.error("请求地址({})失败", url, e);
            throw new RuntimeException("请求地址("+url+")失败");
        } finally {
            HttpClientUtils.closeQuietly(response);
        }
        return null;
    }

    /**
     *  执行post请求（接口）
     * @param url 请求地址
     * @param headers 请求头参数
     * @return 响应数据
     */
    public static String postJson(String url, Map<String, String> headers, Map<String, String> params) {
        CloseableHttpClient closeableHttpClient = httpClientBuilder.build();
        HttpPost httpPost = new HttpPost(url);
        // 请求头设置，如果常用的请求头设置，也可以写死，特殊的请求才传入
        httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
        if (headers != null) {
            for (String headerKey : headers.keySet()) {
                httpPost.setHeader(headerKey, headers.get(headerKey));
            }
        }
        if (params!=null) {
            HttpEntity paramEntity = new StringEntity(JSON.toJSONString(params), StandardCharsets.UTF_8);
            httpPost.setEntity(paramEntity);
        }

        CloseableHttpResponse response = null;
        try {
            response = closeableHttpClient.execute(httpPost);
            int statusCode = response.getStatusLine().getStatusCode();
            if (statusCode== HttpStatus.SC_OK) { // 请求响应成功
                HttpEntity entity = response.getEntity();
                return EntityUtils.toString(entity, StandardCharsets.UTF_8);
            } else {
                log.error("请求地址({})失败:{}", url, statusCode);
            }
        } catch (IOException e) {
            log.error("请求地址({})失败", url, e);
            throw new RuntimeException("请求地址("+url+")失败");
        } finally {
            HttpClientUtils.closeQuietly(response);
        }
        return null;
    }

    /**
     * 构建https安全连接工厂
     * @return 安全连接工厂
     */
    private static ConnectionSocketFactory trustHttpsCertificates() {
        SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
        try {
            sslContextBuilder.loadTrustMaterial(null, new TrustStrategy() {
                @Override
                public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
                    return true;
                }
            });
            SSLContext sslContext = sslContextBuilder.build();
            SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(sslContext,
                    new String[]{"SSLv2Hello", "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"}, // 支持的https安全认证协议
                    null, NoopHostnameVerifier.INSTANCE);
            return sslConnectionSocketFactory;
        } catch (Exception e) {
            log.error("构建安全连接工厂失败", e);
            throw new RuntimeException("构建安全连接工厂失败");
        }
    }
}

4、额外说明

项目中HttpClient基本最好构建请求连接池进行请求，提升性能，加快请求速度；
如果你的应用不请求不信任的https连接，则不需要绕过https安全认证（不需要使用到trustHttpsCertificates方法）；
项目请求在工具类一般可以确定请求头，可以写死，从而少传入一个参数；（特殊要求传入的请求才保留该参数设置）
每次请求后一定要确认消费和关闭响应；（HttpClientUtils.closeQuietly(response)）
部分爬虫需要设置代理，使用示例如下：

HttpGet httpGet = new HttpGet(url);
HttpHost proxy = new HttpHost("49.70.17.48", 8888);
RequestConfig config = RequestConfig.custom()
        .setProxy(proxy) //设置代理
        .setConnectTimeout(2000) // 设置HTTP连接超时时间
        .setSocketTimeout(3000)  // 设置数据响应超时时间
        .setConnectionRequestTimeout(2000) // 设置从连接池获取连接的超时时间
        .build();
httpGet.setConfig(config);

　　成功 = 正确的选择 + 实际的行动 + 长期的坚持；

posted @ 2021-10-12 20:32 知行乐语阅读(739) 评论(0) 编辑收藏举报

刷新页面返回顶部

azuresss