简单的生产消费者模型
案例场景：httpclient4.3.5抓取网页，用自带的线程池进行多线程测试。
httpclient4.3.5简单介绍：对于同一主机的请求，会保存路由信息，下次的请求会根据保存的路由走，减少了查找主机的时间。
类介绍：数据结构用的阻塞队列结构；监控线程、生产线程、消费线程
代码如下：
import java.io.IOException; import java.util.Random; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.conn.routing.HttpRoute; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; public class AQSTest { public static void main(String[] args){ BlockQueue queue = new BlockQueue(); monitor motor = new monitor(queue) ; motor.start() ;// 监控队列大小 Producer prod1 = new Producer(queue) ; Producer prod2 = new Producer(queue) ; Producer prod3 = new Producer(queue) ; prod1.start(); prod2.start(); prod3.start(); Crawel crawel = new Crawel(queue); crawel.start() ; try { Thread.sleep(60*1000*2); prod1.shutdown(); prod2.shutdown(); } catch (InterruptedException e) { e.printStackTrace(); } /* int i = new Random().nextInt()%5; System.out.println(i); */ } public static class Crawel extends Thread{ private BlockQueue queue ; public Crawel(BlockQueue _queue){ queue = _queue ; } @Override public void run() { PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); // 将最大连接数增加到200 cm.setMaxTotal(200); // 将每个路由基础的连接增加到20 cm.setDefaultMaxPerRoute(20); //将目标主机的最大连接数增加到50 HttpHost localhost = new HttpHost("www.yeetrack.com", 80); cm.setMaxPerRoute(new HttpRoute(localhost), 50); CloseableHttpClient httpClient = HttpClients.custom() .setConnectionManager(cm) .build(); // 为每个url创建一个线程，GetThread是自定义的类 GetThread[] threads = new GetThread[50]; for (int i = 0; i < threads.length; i++) { threads[i] = new GetThread(httpClient,queue ); } // 启动线程 for (int j = 0; j < threads.length; j++) { threads[j].start(); } // join the threads for (int j = 0; j < threads.length; j++) { try { threads[j].join(); } catch (InterruptedException e) { e.printStackTrace(); } } } } public static class GetThread extends Thread { private final CloseableHttpClient httpClient; private final HttpContext context; private HttpGet httpget; private volatile boolean stop = false; private final BlockQueue queue ; public GetThread(CloseableHttpClient httpClient,BlockQueue _queue) { this.httpClient = httpClient; this.context = HttpClientContext.create(); this.queue = _queue ; } public void shutdown(){ synchronized (this) { stop = true ; } } @Override public void run() { while(!stop){ try { String url = queue.take() ; httpget = new HttpGet(url); CloseableHttpResponse response = httpClient.execute( httpget, context); try { HttpEntity entity = response.getEntity(); String resp = EntityUtils.toString(entity); System.out.println("成功获取源码"); EntityUtils.consume(entity); } finally { response.close(); } } catch (ClientProtocolException ex) { // Handle protocol errors } catch (IOException ex) { // Handle I/O errors } } } } public static class monitor extends Thread{ private BlockQueue queue ; private volatile boolean shutdown = false; public monitor(BlockQueue _queue){ queue = _queue ; } public void run() { while(!shutdown){ System.out.println("Queue Size: "+queue.size()); try { sleep(500) ; } catch (InterruptedException e) { e.printStackTrace(); } } } public void shutdown(){ synchronized (this) { shutdown = true; } } } public static class Const { private static String[] urls = {"http://www.baidu.com","http://www.bing.com","http://www.hao123.com","http://www.163.com" ,"http://www.csdn.net"}; private static Lock lock = new ReentrantLock() ; public static String get(){ lock.lock() ; int i = new Random().nextInt()%urls.length; i = Math.abs(i); String str = urls[i] ; lock.unlock() ; return str ; } } public static class Producer extends Thread { private volatile boolean stop = false; private BlockQueue queue ; public Producer(BlockQueue _queue){ queue = _queue ; } @Override public void run() { while(!stop){ queue.put(Const.get()) ; try { sleep(500); } catch (InterruptedException e) { e.printStackTrace(); } } } public void shutdown(){ synchronized (this) { stop = true; } } } public static class BlockQueue { public LinkedBlockingQueue<String> queue = new LinkedBlockingQueue<String>(1000); public void put(String url){ try { queue.put(url) ; } catch (InterruptedException e) { e.printStackTrace(); queue.clear(); } } public String take(){ String element = null ; try { element = queue.take() ; } catch (InterruptedException e) { e.printStackTrace(); } return element ; } public int size(){ int size = queue.size() ; return size ; } } public static class IdleConnectionMonitorThread extends Thread { private final HttpClientConnectionManager connMgr; private volatile boolean shutdown; public IdleConnectionMonitorThread(HttpClientConnectionManager connMgr) { super(); this.connMgr = connMgr; } @Override public void run() { try { while (!shutdown) { synchronized (this) { wait(5000); // 关闭失效的连接 connMgr.closeExpiredConnections(); // 可选的, 关闭30秒内不活动的连接 connMgr.closeIdleConnections(30, TimeUnit.SECONDS); } } } catch (InterruptedException ex) { // terminate } } public void shutdown() { shutdown = true; synchronized (this) { notifyAll(); } } } }
