Java--多线程读取网络图片并保存在本地
本例用到了多线程、时间函数、网络流、文件读写、正则表达式（在读取html内容response时，最好不要用正则表达式来抓捕html文本内容里的特征，因为服务器返回的多个页面的文本内容不一定使用相同的模式），是一个综合性的实例。
package javatest; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.regex.Matcher; import java.util.regex.Pattern; class urlTest { public static void main(String[] args) throws IOException { //String url = "http://www.ik6.com/meinv/10000/index.html"; String dir = "d:\\result\\201601282"; int base = 40624; // 多线程方法，从网上下载多个图片并保存 ArrayList<Thread> threads = new ArrayList<Thread>(); urlTest test=new urlTest(); int threadCount=1;//开5个线程，用于下载 int themePerThread=1; Date start=new Date(); System.out.println("threads start.."); for (int i = 0; i < threadCount; i++) { Thread t = new Thread(test.new workerThread(dir, base, themePerThread)); threads.add(t); t.start(); base+=themePerThread; } for (Thread t : threads) { try { t.join();//让主线程等待此子线程执行完毕 } catch (InterruptedException e) { e.printStackTrace(); } } System.out.println("threads complete.."); Date end=new Date(); //计算总耗时 long diff = end.getTime() - start.getTime(); String info=String.format("it takes %f seconds to run.", diff / 1000.00); System.out.println(info); //单线程方法 // for (int themeCount = 0; themeCount < 200; themeCount++) // { // for (int pageIndex = 1; pageIndex <= 20; pageIndex++) // { // if (pageIndex==1) // {url = String.format( // "http://www.ik6.com/meinv/%d/index.html", base // + themeCount); // } // else // { // url = String.format( // "http://www.ik6.com/meinv/%d/index_%d.html", base // + themeCount, pageIndex); // } // // String data = GetResponseText(url); // if (!IsContentPage(data)) // break; // System.out.println(url); // ArrayList<String> imgUrls = GetImgUrls(data); // for (String imgUrl : imgUrls) // { // String imageSavedPath = String.format("%s\\%d_%d.jpg", dir,base+ // themeCount,pageIndex); // RetrieveImg2(imgUrl, imageSavedPath); // } // // } // } } public class workerThread implements Runnable { String dir = null; int base = 0; int themeCount = 0; int totalPage=0; int totalImg=0; public workerThread(String dir, int base, int themeCount) { this.dir = dir; this.base = base; this.themeCount = themeCount; } public void run() { String url=null; int pageNo=0; for (int themeIndex = 0; themeIndex < themeCount; themeIndex++) { for (int pageIndex = 1; pageIndex <= 50; pageIndex++) { pageNo=base+ themeIndex; if (pageIndex == 1) { url = String.format("http://www.ik6.com/meinv/%d/index.html", pageNo); } else { url = String.format( "http://www.ik6.com/meinv/%d/index_%d.html", pageNo, pageIndex); } String data = GetResponseText(url); if (!IsContentPage(data)) break; ArrayList<String> imgUrls = GetImgUrls(data); for (String imgUrl : imgUrls) { String imageSavedPath = String.format("%s\\%d_%d.jpg", dir, pageNo, pageIndex); RetrieveImg2(imgUrl, imageSavedPath); } } } } } //日期格式化 public static String GetTimeString() { Date dt = new Date(); SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHHmmss"); String s = df.format(dt); return s; } //通过特征判断 public static boolean IsContentPage(String pageContent) { return pageContent.indexOf("<center>") != -1; } public static ArrayList<String> GetImgUrls(String srcStr) { // 利用正则表达式，读取页面中所有图片的url // Pattern p1 = Pattern.compile("<center.+\n?.+\n?</center>"); // Pattern p2 = Pattern.compile("lazysrc=\"http\\:.+\\.jpg\""); // Matcher m = p2.matcher(srcStr); // ArrayList<String> imgUrls = new ArrayList<String>(); // while (m.find()) // { // String match = m.group(); // imgUrls.add(match.substring(match.indexOf("\"")+1,match.lastIndexOf("\""))); // } // return imgUrls; // 仅读取主题图片的url，为何不能匹配center? // Pattern p1 = Pattern.compile("<center.+\n*.+\n*</center>"); // Pattern p2 = Pattern.compile("lazysrc=\"http\\:.+\\.jpg\""); // Matcher m = p1.matcher(srcStr); // ArrayList<String> imgUrls = new ArrayList<String>(); // if (m.find()) // { // String matchCenter = m.group(); // Matcher m2 = p2.matcher(matchCenter); // while (m2.find()) // { // String matchImage = m2.group(); // imgUrls.add(matchImage.substring(matchImage.indexOf("\"") + 1, // matchImage.lastIndexOf("\""))); // } // } // return imgUrls; // 用字符串的indexOf方法找出所有图片的url srcStr = srcStr.substring(srcStr.indexOf("<center"), srcStr.indexOf("</center>")); // Pattern p2 = Pattern.compile("lazysrc=http\\:.+\\.jpg"); srcStr = srcStr.substring(srcStr.indexOf("src=")); srcStr = srcStr.substring(srcStr.indexOf("http"), srcStr.indexOf(".jpg") + 4); ArrayList<String> imgUrls = new ArrayList<String>(); imgUrls.add(srcStr); return imgUrls; } //通过url获取html页面 public static String GetResponseText(String url) { String response = null; try { URL _url = new URL(url); HttpURLConnection urlcon = (HttpURLConnection) _url .openConnection(); // 获取连接 InputStream is = urlcon.getInputStream(); BufferedReader buffer = new BufferedReader(new InputStreamReader( is, "utf-8")); StringBuffer sb = new StringBuffer(); String line = null; while ((line = buffer.readLine()) != null) { sb.append(line).append('\n'); // System.out.println(l); } response = sb.toString(); } catch (Exception e) { e.printStackTrace(); } return response; } //通过图片的url，获取图片并保存在本地.注意：此法有缺点 public static void RetrieveImg(String imgURL, String savepath) { try { File file = new File(savepath); if (file.exists()) { return; } else { file.createNewFile(); URL _url = new URL(imgURL); HttpURLConnection urlcon = (HttpURLConnection) _url .openConnection(); // urlcon.setRequestMethod("GET"); // 超时响应时间为5秒 // urlcon.setConnectTimeout(3 * 1000); // 获取连接 InputStream is = urlcon.getInputStream(); byte[] buffer = new byte[1024]; FileOutputStream out = new FileOutputStream(file); while (is.read(buffer) != -1) ; out.write(buffer);// 为何不行 is.close(); out.close(); } } catch (Exception e) { e.printStackTrace(); } } //通过图片的url，获取图片并保存在本地 public static void RetrieveImg2(String imgURL, String savepath) { try { File file = new File(savepath); if (file.exists()) { return; } else { file.createNewFile(); URL _url = new URL(imgURL); HttpURLConnection conn = (HttpURLConnection) _url .openConnection(); conn.setRequestMethod("GET"); // 超时响应时间为5秒 conn.setConnectTimeout(5 * 1000); // 通过输入流获取图片数据 InputStream inStream = conn.getInputStream(); byte[] data = readInputStream(inStream); // 写入到新文件当中 FileOutputStream out = new FileOutputStream(file); out.write(data); out.close(); } } catch (Exception e) { e.printStackTrace(); } } //将输入流的内容写入内存保存起来，以便稍后写入到文件当中 public static byte[] readInputStream(InputStream inStream) throws Exception { ByteArrayOutputStream outStream = new ByteArrayOutputStream(); byte[] buffer = new byte[1024]; int len = 0;// 关键，否则图片不完整，因为不知道写入多少 while ((len = inStream.read(buffer)) != -1) { outStream.write(buffer, 0, len); } inStream.close(); // 把outStream里的数据写入内存 return outStream.toByteArray(); } }
