HttpClient 大量连接等待异常的处理
前几天要扫一批连接,因为涉及到ua的适配测试,所以选用了httpclient处理。
httpclient其实很好用,也用过几次,还曾经写过自动刷人人网人气的程序,不过对于它掌握还是处在比较浅显的时代
我这人比较懒,学习动力还是太弱,不逼到份上都懒得学习哎!
这次又用到这东西了就直接看了看api 就开始写了,不过写的时候发现一个问题。
开始跑100多个url的时候是没问题的,不过过了一定量数后,会大量的抛出如下异常
org.apache.http.conn.ConnectionPoolTimeoutException: Timeout waiting for connection
at org.apache.http.impl.conn.tsccm.ConnPoolByRoute.getEntryBlocking(ConnPoolByRoute.java:417)
at org.apache.http.impl.conn.tsccm.ConnPoolByRoute$1.getPoolEntry(ConnPoolByRoute.java:300)
at org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager$1.getConnection(ThreadSafeClientConnManager.java:224)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:401)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:820)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:754)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:732)
at com.ua.engine.Engine.httpService(Engine.java:97)
at com.ua.engine.Engine.main(Engine.java:27)
对此很不解,还是了解的不够多啊,后来发现问题了,原来之前一直使用连接池中的一条连接进行链接访问,用的多了,连接到期,会造成连接等待的情况,
因为使用的HttpClient4 ,于是查了下资料,发现使用完每个连接后一定要及时的释放掉,不然上面的错会无限的报下去。
而释放连接只用InputStream 就能轻松搞定了
最后整体的代码如下
1 package com.ua.engine; 2 3 import java.io.BufferedReader; 4 import java.io.BufferedWriter; 5 import java.io.FileReader; 6 import java.io.IOException; 7 import java.io.InputStream; 8 import java.io.PrintWriter; 9 import java.util.ArrayList; 10 import java.util.HashMap; 11 import java.util.List; 12 13 import org.apache.http.HttpEntity; 14 import org.apache.http.HttpResponse; 15 import org.apache.http.client.HttpClient; 16 import org.apache.http.client.methods.HttpGet; 17 18 19 public class Engine { 20 21 public static void main(String[] args) { 22 try { 23 HashMap<String,List<String>> map = readURL(); 24 BufferedWriter bw = new BufferedWriter(new PrintWriter("D://furl/log.txt")); 25 List<String> weblist = map.get("web"); 26 int i = 1; 27 for(String url :weblist){ 28 System.out.println(i); 29 String temp = Engine.httpService("http://52.easou.com"+url,"web")+" "+url; 30 System.out.println(temp); 31 bw.write(temp); 32 bw.write("\r\n"); 33 i++; 34 } 35 List<String> touchlist = map.get("touch"); 36 for(String url :touchlist){ 37 System.out.println(i); 38 String temp = Engine.httpService("http://52.easou.com"+url,"touch")+" "+url; 39 System.out.println(temp); 40 bw.write(temp); 41 bw.write("\r\n"); 42 i++; 43 } 44 List<String> hdlist = map.get("hd"); 45 for(String url :hdlist){ 46 System.out.println(i); 47 String temp = Engine.httpService("http://52.easou.com"+url,"hd")+" "+url; 48 System.out.println(temp); 49 bw.write(temp); 50 bw.write("\r\n"); 51 i++; 52 } 53 List<String> cslist = map.get("cs"); 54 for(String url :cslist){ 55 System.out.println(i); 56 String temp = Engine.httpService("http://52.easou.com"+url,"cs")+" "+url; 57 System.out.println(temp); 58 bw.write(temp); 59 bw.write("\r\n"); 60 i++; 61 } 62 bw.flush(); 63 bw.close(); 64 } 65 catch (Exception e) { 66 e.printStackTrace(); 67 } 68 } 69 private static HttpClient httpclient = null; 70 static { 71 HttpClientUtils hcu = new HttpClientUtils(); 72 hcu.setConnTimeout(10000); 73 hcu.setSoTimeout(10000); 74 hcu.setBufSize(128 * 1024); 75 hcu.setAgent(""); 76 hcu.setMaxRedirect(3); 77 hcu.setMaxTotal(500); 78 hcu.setMaxPerRoute(100); 79 httpclient = hcu.newClient(); 80 } 81 public static String httpService(String url,String version) { 82 HttpGet get = new HttpGet(url); 83 if("web".equals(version)){ 84 get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.52 Safari/536.5"); 85 }else if("touch".equals(version)){ 86 get.setHeader("User-Agent", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16"); 87 }else if("hd".equals(version)){ 88 get.setHeader("User-Agent", "Mozilla/5.0 (iPad; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16"); 89 }else { 90 get.setHeader("User-Agent", "lg-kp500 teleca/wap2.0 midp-2.0/cldc-1.1 untrusted/1.0 nokia7610 (3.0417.0ch) symbianos/7.0s series60/2.1 profile/midp-2.0 conf"); 91 } 92 93 // HttpEntity entity = response.getEntity(); 94 // System.out.println(EntityUtils.toString(entity)); 95 HttpResponse response = null; 96 String code = "error "+url; 97 InputStream in = null; 98 try { 99 response = httpclient.execute(get); 100 HttpEntity entity =response.getEntity(); 101 if( entity != null ){ 102 in = entity.getContent(); //之前没使用这个造成了大量异常抛出,只要是 103 } 104 code = response.getStatusLine().getStatusCode()+" "+version+" "+url; 105 return code; 106 } 107 catch (Exception e) { 108 System.out.println(code); 109 e.printStackTrace(); 110 }finally { 111 if(in!=null){ 112 try { 113 in.close(); 114 } 115 catch (IOException e) { 116 e.printStackTrace(); 117 } 118 } 119 } 120 return code; 121 } 122 123 public static HashMap<String,List<String>> readURL() throws Exception{ 124 List<String> weblist = new ArrayList<String>(); 125 List<String> touchlist = new ArrayList<String>(); 126 List<String> hdlist = new ArrayList<String>(); 127 List<String> cslist = new ArrayList<String>(); 128 129 BufferedReader br = new BufferedReader(new FileReader("D://furl/1.txt")); 130 while(br.ready()){ 131 String url = br.readLine(); 132 if(url.toLowerCase().indexOf(".html")>-1){ 133 weblist.add(url); 134 }else if(url.toLowerCase().indexOf("touch")>-1){ 135 touchlist.add(url); 136 }else if(url.toLowerCase().indexOf("/hd/")>-1){ 137 hdlist.add(url); 138 }else { 139 cslist.add(url); 140 } 141 } 142 br = new BufferedReader(new FileReader("D://furl/url1.txt")); 143 while(br.ready()){ 144 String url = br.readLine(); 145 if(url.toLowerCase().indexOf(".html")>-1){ 146 weblist.add(url); 147 }else if(url.toLowerCase().indexOf("touch")>-1){ 148 touchlist.add(url); 149 }else if(url.toLowerCase().indexOf("/hd/")>-1){ 150 hdlist.add(url); 151 }else { 152 cslist.add(url); 153 } 154 } 155 br = new BufferedReader(new FileReader("D://furl/urljian1.txt")); 156 while(br.ready()){ 157 String url = br.readLine(); 158 if(url.toLowerCase().indexOf(".html")>-1){ 159 weblist.add(url); 160 }else if(url.toLowerCase().indexOf("touch")>-1){ 161 touchlist.add(url); 162 }else if(url.toLowerCase().indexOf("/hd/")>-1){ 163 hdlist.add(url); 164 }else { 165 cslist.add(url); 166 } 167 } 168 HashMap<String,List<String>> map = new HashMap<String,List<String>>(); 169 System.out.println(weblist.size()); 170 System.out.println(touchlist.size()); 171 System.out.println(hdlist.size()); 172 System.out.println(cslist.size()); 173 map.put("web", weblist); 174 map.put("touch", touchlist); 175 map.put("hd", hdlist); 176 map.put("cs", cslist); 177 return map; 178 } 179 }
1 package com.ua.engine; 2 3 import org.apache.http.HttpVersion; 4 import org.apache.http.client.HttpClient; 5 import org.apache.http.client.params.ClientPNames; 6 import org.apache.http.impl.client.DefaultHttpClient; 7 import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; 8 import org.apache.http.params.BasicHttpParams; 9 import org.apache.http.params.CoreConnectionPNames; 10 import org.apache.http.params.CoreProtocolPNames; 11 import org.apache.http.params.HttpParams; 12 13 public class HttpClientUtils { 14 private int connTimeout = 5000; 15 private int soTimeout = 5000; 16 private int bufSize = 128 * 1024; 17 private String agent = ""; 18 private int maxRedirect = 3; 19 private int maxTotal = 500; 20 private int maxPerRoute = 100; 21 22 public HttpClient newClient() { 23 HttpParams params = new BasicHttpParams(); 24 params.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 25 this.connTimeout); 26 params.setParameter(CoreConnectionPNames.SO_TIMEOUT, this.soTimeout); 27 params.setParameter(CoreConnectionPNames.SO_REUSEADDR, true); 28 params.setParameter(CoreConnectionPNames.TCP_NODELAY, false); 29 params.setParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, 30 this.bufSize); 31 params.setParameter(CoreProtocolPNames.PROTOCOL_VERSION, 32 HttpVersion.HTTP_1_1); 33 params.setParameter(CoreProtocolPNames.USER_AGENT, this.agent); 34 params.setParameter(ClientPNames.CONNECTION_MANAGER_FACTORY_CLASS_NAME, 35 "org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager"); 36 params.setParameter(ClientPNames.MAX_REDIRECTS, this.maxRedirect); 37 ThreadSafeClientConnManager ccm = new ThreadSafeClientConnManager(); 38 ccm.setMaxTotal(this.maxTotal); 39 ccm.setDefaultMaxPerRoute(this.maxPerRoute); 40 return new DefaultHttpClient(ccm, params); 41 } 42 43 public long getConnTimeout() { 44 return connTimeout; 45 } 46 47 public void setConnTimeout(int connTimeout) { 48 this.connTimeout = connTimeout; 49 } 50 51 public long getSoTimeout() { 52 return soTimeout; 53 } 54 55 public void setSoTimeout(int soTimeout) { 56 this.soTimeout = soTimeout; 57 } 58 59 public int getBufSize() { 60 return bufSize; 61 } 62 63 public void setBufSize(int bufSize) { 64 this.bufSize = bufSize; 65 } 66 67 public String getAgent() { 68 return agent; 69 } 70 71 public void setAgent(String agent) { 72 this.agent = agent; 73 } 74 75 public int getMaxRedirect() { 76 return maxRedirect; 77 } 78 79 public void setMaxRedirect(int maxRedirect) { 80 this.maxRedirect = maxRedirect; 81 } 82 83 public int getMaxTotal() { 84 return maxTotal; 85 } 86 87 public void setMaxTotal(int maxTotal) { 88 this.maxTotal = maxTotal; 89 } 90 91 public int getMaxPerRoute() { 92 return maxPerRoute; 93 } 94 95 public void setMaxPerRoute(int maxPerRoute) { 96 this.maxPerRoute = maxPerRoute; 97 } 98 }