package com.wp.nevel.base.utils;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import org.junit.Test;
import com.wp.nevel.base.exception.ParserException;
import com.wp.nevel.base.service.impl.LogServiceHelp;
public class HttpClientUtils {
public static Logger logger = Logger.getLogger( LogServiceHelp.class);
private static HttpClient httpclient;
static {
httpclient = HttpClients.createDefault();
}
@Test
public void test(){
String url="http://www.shuchongw.com/files/article/html/23/23114/index.html";
doGetHtmlContent2byte(url);
}
/**
* 根据简单url获取网页数据,转换成byte [] 存储
* */
public static byte[] doGetHtmlContent2byte(String url) {
CloseableHttpResponse response = null;
byte[] resultByte = {};
try {
HttpGet get = new HttpGet(url);
System.out.println(url);
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(30000).setConnectTimeout(30000).build();
get.setConfig(requestConfig);
try {
response = (CloseableHttpResponse) HttpClientUtils.httpclient.execute(get);
} catch (UnknownHostException e) {
e.printStackTrace();
logger.info("链接主网失败");
}
int statusCode = response.getStatusLine().getStatusCode();
System.out.println(statusCode);
if (statusCode == 200) {
HttpEntity entity = response.getEntity();
resultByte = EntityUtils.toByteArray(entity);
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (SocketException e) {
e.printStackTrace();
}catch(IOException e){
e.printStackTrace();
} finally {
try {
if(response!=null){
response.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return resultByte;
}
/**
* 根据复杂url获取网页数据,转换成byte [] 存储
* @throws ParserException
* @throws IOException
* @throws UnknownHostException
* @throws ClientProtocolException
* @throws SocketException
* */
public static byte [] doGetHtmlByParams2Byte(Map<String, String> params, String paramsEncoding, String url) {
if (params != null) {
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
for (Entry<String, String> entry : params.entrySet()) {
formparams.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
}
if (!formparams.isEmpty()) {
String paramsStr = URLEncodedUtils.format(formparams, paramsEncoding!=null?paramsEncoding:"utf-8");
url = url + "?" + paramsStr;
}
}
return doGetHtmlContent2byte(url);
}
/**
* 根据复杂url获取网页数据,转换成String 存储
* @throws ParserException
* @throws IOException
* @throws UnknownHostException
* @throws ClientProtocolException
* @throws SocketException
* */
public static String doGetHtmlByParams2Text(Map<String, String> params, String paramsEncoding, String url,String htmlEncoding) throws ClientProtocolException, UnknownHostException, SocketException{
try {
return getHtmlContentByText(doGetHtmlByParams2Byte(params,paramsEncoding,url),htmlEncoding!=null?htmlEncoding:"utf-8");
} catch (Exception e) {
e.printStackTrace();
return "";
}
}
/**
* 根据简单url获取网页数据,转换成String 存储
* @throws ParserException
* @throws IOException
* @throws UnknownHostException
* @throws ClientProtocolException
* @throws SocketException
* */
public static String doGetHtmlContentToString(String url, String encoding){
try {
return getHtmlContentByText(doGetHtmlContent2byte(url),encoding);
} catch (Exception e) {
e.printStackTrace();
return "";
}
}
/**
* 根据简单url获取网页图片数据, 保存路径[saveImagePath]
* @throws ParserException
* @throws IOException
* @throws UnknownHostException
* @throws ClientProtocolException
* @throws SocketException
* */
public static void getHtml2Image(String url,String saveImagPath){
try {
downloadData(doGetHtmlContent2byte(url),saveImagPath);
} catch (Exception e) {
e.printStackTrace();
}
}
public static String getHtmlContentByText(byte [] htmlBytes, String encoding){
try {
return new String (htmlBytes,encoding!=null?encoding:"utf-8");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
return "";
}
}
/**
* 执行下载io
*
* @param byte [] data 网页字节流,filename 存储地址和文件名 return
* */
public static void downloadData(byte[] data, String filename) {
try {
DataOutputStream writer = new DataOutputStream(
new FileOutputStream(new File(filename)));
BufferedOutputStream out = new BufferedOutputStream(writer);
out.write(data);
out.flush();
out.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
public static String readFile(String filename) throws IOException{
String xmlContent =null;
File file = new File(filename);
BufferedReader reader =null;
try {
if(!file.exists()){
new RuntimeException("文件不存在");
return xmlContent;
}
StringBuilder buider = new StringBuilder();
String readata ="";
reader = new BufferedReader(new FileReader(file));
while(true){
readata = reader.readLine();
if(readata==null){
break;
}
buider.append(readata).append("\n");
}
xmlContent=buider.toString();
} catch (FileNotFoundException e) {
e.printStackTrace();
}finally{
if(reader!=null)
reader.close();
}
return xmlContent;
}
public static byte [] doGetByteByHttpclient2Url(HttpContext httpContext,CloseableHttpClient client,String url){
byte [] resultBytes = null;
try {
HttpGet get = new HttpGet(url);
CloseableHttpResponse response =client.execute(get, httpContext);
int status = response.getStatusLine().getStatusCode();
System.out.println("链接状态="+status);
if(status!=200)
return resultBytes;
HttpEntity entity = response.getEntity();
resultBytes = EntityUtils.toByteArray(entity);
} catch (ClientProtocolException e) {
throw new RuntimeException("失败连接地址"+url, e);
} catch (IOException e) {
throw new RuntimeException("失败连接地址"+url, e);
}
if(resultBytes==null){
try {
Thread.sleep(2000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
return resultBytes;
}
}
