Using HttpClient properly to avoid CLOSE_WAIT TCP connections
Apache的HttpComponent组件，用的人不在少数。但是能用好的人，却微乎其微，为什么？很简单，TCP里面的细节实现不是每个人都能捕获到的（细节是魔鬼），像并发请求控制&资源释放，Nagle算法参数优化，Connection eviction，跟ulimit配对的total connection，重定向策略定制化，两类超时时间的合理设置，流读写等等。
在最近的项目中，更是破天荒的遇到了close_wait问题，所以利用业余时间索性将之前同学写的HttpClient优化了一遍。下面我将贴出代码，如果大家发现了还有改进的余地，记得千万要留言知会我，共创最棒的代码：
/**
* 史上最棒的HttpClient4封装,details please see
* http://hc.apache.org/httpcomponents-client-ga/tutorial/html/index.html
*
* @author von gosling 2013-5-7
*/
public class HttpClientManager {
//Consider ulimit
private static final int DEFAULT_MAX_TOTAL_CONNECTIONS = 7500;
//notice IE 6,7,8
private static final int DEFAULT_MAX_CONNECTIONS_PER_ROUTE = 200;
private static final int DEFAULT_CONN_TIMEOUT_MILLISECONDS = 5 * 1000;
private static final int DEFAULT_READ_TIMEOUT_MILLISECONDS = 60 * 1000;
private static final int INIT_DELAY = 5 * 1000;
private static final int CHECK_INTERVAL = 5 * 60 * 1000;
private static String HTTP_REQUEST_ENCODING = "UTF-8";
private static String LINE_SEPARATOR = "\r\n";
private static final Logger LOG = LoggerFactory
.getLogger(HttpClientManager.class);
private static ThreadSafeClientConnManager connectionManager;
static {
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
//schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
connectionManager = new ThreadSafeClientConnManager(schemeRegistry);
connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL_CONNECTIONS);
connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_PER_ROUTE);
//Connection eviction
ScheduledExecutorService scheduledExeService = Executors.newScheduledThreadPool(1,
new DaemonThreadFactory("Http-client-ConenctionPool-Monitor"));
scheduledExeService.scheduleAtFixedRate(new IdleConnectionMonitor(connectionManager),
INIT_DELAY, CHECK_INTERVAL, TimeUnit.MILLISECONDS);
}
public static String doPost(String reqURL, Map<String, String> params, String encoding,
Boolean enableSSL) {
HttpClient httpClient = getHttpClient(enableSSL);
String responseContent = "";
try {
HttpPost httpPost = buildHttpPostRequest(reqURL, params, encoding);
HttpResponse response = httpClient.execute(httpPost);
// validateResponse(response, httpPost);
HttpEntity entity = response.getEntity();
if (entity != null) {
// responseLength = entity.getContentLength();
responseContent = EntityUtils.toString(entity, encoding);
//Ensure that the entity content has been fully consumed and the underlying stream has been closed.
EntityUtils.consume(entity);
} else {
LOG.warn("Http entity is null! request url is {},response status is {}", reqURL,
response.getStatusLine());
}
} catch (ConnectTimeoutException e) {
LOG.warn(e.getMessage());
} catch (SocketTimeoutException e) {
LOG.warn("Read time out!");
} catch (SSLPeerUnverifiedException e) {
LOG.warn("Peer not authenticated!");
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
return responseContent;
}
public static String doPost(String reqURL, final String entities, String encoding) {
HttpClient httpClient = getHttpClient(false);
String responseContent = "";
try {
AbstractHttpEntity printWriterEntity = new AbstractHttpEntity() {
public boolean isRepeatable() {
return false;
}
public long getContentLength() {
return -1;
}
public boolean isStreaming() {
return false;
}
public InputStream getContent() throws IOException {
// Should be implemented as well but is irrelevant for this case
throw new UnsupportedOperationException();
}
public void writeTo(final OutputStream outstream) throws IOException {
PrintWriter writer = new PrintWriter(new OutputStreamWriter(outstream,
HTTP_REQUEST_ENCODING));
writer.print(entities);
writer.print(LINE_SEPARATOR);
writer.flush();
}
};
HttpPost httpPost = new HttpPost(reqURL);
//If the data is large enough that you need to stream it,
//you can write to a temp file and use FileEntity or possibly set up a pipe and use InputStreamEntity
httpPost.setEntity(printWriterEntity);
HttpResponse response = httpClient.execute(httpPost);
validateResponse(response, httpPost);
HttpEntity entity = response.getEntity();
if (entity != null) {
responseContent = EntityUtils.toString(entity, encoding);
//Ensure that the entity content has been fully consumed and the underlying stream has been closed.
EntityUtils.consume(entity);
} else {
LOG.warn("Http entity is null! request url is {},response status is {}", reqURL,
response.getStatusLine());
}
} catch (SocketTimeoutException e) {
LOG.warn("Read time out!");
} catch (SSLPeerUnverifiedException e) {
LOG.warn("Peer not authenticated!");
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
return responseContent;
}
private static X509TrustManager customTrustManager(HttpClient httpClient) {
//Trusting all certificates
X509TrustManager xtm = new X509TrustManager() {
public void checkClientTrusted(X509Certificate[] chain, String authType)
throws CertificateException {
}
public void checkServerTrusted(X509Certificate[] chain, String authType)
throws CertificateException {
}
public X509Certificate[] getAcceptedIssuers() {
return null;
}
};
try {
SSLContext ctx = SSLContext.getInstance("TLS");
if (null != ctx) {
ctx.init(null, new TrustManager[] { xtm }, null);
SSLSocketFactory socketFactory = new SSLSocketFactory(ctx,
SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
httpClient.getConnectionManager().getSchemeRegistry()
.register(new Scheme("https", 443, socketFactory));
}
} catch (Exception e) {
LOG.error(e.getMessage());
}
return xtm;
}
private static HttpClient getHttpClient(Boolean enableSSL) {
DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager);
httpClient.setRedirectStrategy(new RedirectStrategy() { //设置重定向处理方式为自行处理
@Override
public boolean isRedirected(HttpRequest request, HttpResponse response,
HttpContext context) throws ProtocolException {
return false;
}
@Override
public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response,
HttpContext context) throws ProtocolException {
return null;
}
});
httpClient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,
DEFAULT_READ_TIMEOUT_MILLISECONDS);
httpClient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,
DEFAULT_CONN_TIMEOUT_MILLISECONDS);
//According to http use-case to decide to whether to open TCP_NODELAY option,So does SO_LINGER option
httpClient.getParams().setParameter(CoreConnectionPNames.TCP_NODELAY, Boolean.TRUE);
httpClient.getParams().setParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,
Boolean.FALSE);
if (enableSSL) {
customTrustManager(httpClient);
}
return httpClient;
}
public static Map.Entry<Integer, String> doGetHttpResponse(String url, String encoding) {
HttpClient httpClient = getHttpClient(false);
HttpGet httpget = new HttpGet(url);
try {
EncodingResponseHandler responseHandler = new EncodingResponseHandler();
if (StringUtils.isBlank(encoding)) {
encoding = HTTP_REQUEST_ENCODING;
}
responseHandler.setEncoding(encoding);
return httpClient.execute(httpget, responseHandler);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
return null;
}
public static String doGet(String url, String encoding) {
Map.Entry<Integer, String> ret = doGetHttpResponse(url, encoding);
if (ret == null) {
return "";
}
if (ret.getKey() != HttpStatus.SC_OK) {
LOG.error(
"Did not receive successful HTTP response: status code = {}, request url = {}",
ret.getKey(), url);
}
return ret.getValue();
}
public static void doPost(String url, Map<String, String> params) {
HttpClient httpClient = getHttpClient(false);
try {
HttpPost httpPost = buildHttpPostRequest(url, params, HTTP.UTF_8);
ResponseHandler<byte[]> handler = new ResponseHandler<byte[]>() {
public byte[] handleResponse(HttpResponse response) throws ClientProtocolException,
IOException {
HttpEntity entity = response.getEntity();
if (entity != null) {
return EntityUtils.toByteArray(entity);
} else {
return null;
}
}
};
httpClient.execute(httpPost, handler);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
private static HttpPost buildHttpPostRequest(String url, Map<String, String> params,
String encoding)
throws UnsupportedEncodingException {
HttpPost httpPost = new HttpPost(url);
//Encode the form parameters
if (!CollectionUtils.isEmpty(params)) {
List<NameValuePair> nvps = Lists.newArrayList();
Set<Entry<String, String>> paramEntrys = params.entrySet();
for (Entry<String, String> entry : paramEntrys) {
nvps.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
}
httpPost.setEntity(new UrlEncodedFormEntity(nvps, encoding));
}
return httpPost;
}
// private static void validateResponse(HttpResponse response, HttpGet get) throws IOException {
// StatusLine status = response.getStatusLine();
// if (status.getStatusCode() >= HttpStatus.SC_MULTIPLE_CHOICES) {
// LOG.warn(
// "Did not receive successful HTTP response: status code = {}, status message = {}",
// status.getStatusCode(), status.getReasonPhrase());
// get.abort();
// return;
// }
// }
private static void validateResponse(HttpResponse response, HttpPost post) throws IOException {
StatusLine status = response.getStatusLine();
if (status.getStatusCode() >= HttpStatus.SC_MULTIPLE_CHOICES) {
LOG.warn(
"Did not receive successful HTTP response: status code = {}, status message = {}",
status.getStatusCode(), status.getReasonPhrase());
post.abort();
return;
}
}
}
