抓取历史天气
工具类
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
package com.example.demo.util; import org.apache.poi.hssf.usermodel.*; import org.apache.poi.xssf.usermodel.XSSFCell; import org.apache.poi.xssf.usermodel.XSSFRow; import org.apache.poi.xssf.usermodel.XSSFSheet; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; public class ReadExcel { public static List<List<String>> readExcel(String path) throws IOException { if (path.endsWith(".xlsx")) { return readXlsx(path); } else if (path.endsWith(".xls")) { return readXlsx(path); } else { return null; } } /** * Read the Excel 2010 */ public static List<List<String>> readXlsx(String path) throws IOException { InputStream is = new FileInputStream(path); XSSFWorkbook xssfWorkbook = new XSSFWorkbook(is); List<List<String>> list = new ArrayList<List<String>>(); // Read the Sheet int numSheets = xssfWorkbook.getNumberOfSheets();// 获取sheet页数 for (int numSheet = 0; numSheet < 1; numSheet++) { XSSFSheet xssfSheet = xssfWorkbook.getSheetAt(numSheet); if (xssfSheet == null) { continue; } // Read the Row int rowNums = xssfSheet.getLastRowNum(); for (int rowNum = 0; rowNum <= rowNums; rowNum++) { List<String> l = new ArrayList<String>(); // 获取对应的行数据 XSSFRow xssfRow = xssfSheet.getRow(rowNum); if (xssfRow != null) { // 获取列数 int columnNum = xssfRow.getLastCellNum(); for (int cloNum = 0; cloNum < columnNum; cloNum++) { l.add(getValue(xssfRow.getCell(cloNum))); } list.add(l); } } } return list; } /** * Read the Excel 2003 */ public void readXls(String path) throws IOException { InputStream is = new FileInputStream(path); HSSFWorkbook hssfWorkbook = new HSSFWorkbook(is); List<List<String>> list = new ArrayList<List<String>>(); // Read the Sheet int numSheets = hssfWorkbook.getNumberOfSheets();// 获取sheet页数 for (int numSheet = 0; numSheet < 1; numSheet++) { HSSFSheet hssfSheet = hssfWorkbook.getSheetAt(numSheet); if (hssfSheet == null) { continue; } // Read the Row int rowNums = hssfSheet.getLastRowNum(); for (int rowNum = 0; rowNum <= rowNums; rowNum++) { // 获取对应的行数据 HSSFRow hssfRow = hssfSheet.getRow(rowNum); List<String> l = new ArrayList<String>(); if (hssfRow != null) { // 获取列数 int columnNum = hssfRow.getLastCellNum(); for (int cloNum = 0; cloNum < columnNum; cloNum++) { l.add(getValue(hssfRow.getCell(cloNum))); } list.add(l); } } } } /** * 取值2010excel * * @param cell * @return */ private static String getValue(XSSFCell cell) { if (cell == null) { return ""; } String value = ""; switch (cell.getCellType()) { case HSSFCell.CELL_TYPE_STRING: value = cell.getStringCellValue(); break; case HSSFCell.CELL_TYPE_NUMERIC: if (HSSFDateUtil.isCellDateFormatted(cell)) { Date date = cell.getDateCellValue(); if (date != null) { value = new SimpleDateFormat("yyyy-MM-dd").format(date); } else { value = ""; } } else { value = new DecimalFormat("0").format(cell.getNumericCellValue()); } break; case HSSFCell.CELL_TYPE_FORMULA: // 导入时如果为公式生成的数据则无值 if (!cell.getStringCellValue().equals("")) { value = cell.getStringCellValue(); } else { value = cell.getNumericCellValue() + ""; } break; case HSSFCell.CELL_TYPE_BLANK: break; case HSSFCell.CELL_TYPE_ERROR: break; case HSSFCell.CELL_TYPE_BOOLEAN: value = (cell.getBooleanCellValue() == true ? "Y" : "N"); break; default: value = ""; } return value.trim(); } /** * 取值2003excel * * @param cell * @return */ private String getValue(HSSFCell cell) { if (cell == null) { return ""; } String value = ""; switch (cell.getCellType()) { case HSSFCell.CELL_TYPE_STRING: value = cell.getStringCellValue(); break; case HSSFCell.CELL_TYPE_NUMERIC: if (HSSFDateUtil.isCellDateFormatted(cell)) { Date date = cell.getDateCellValue(); if (date != null) { value = new SimpleDateFormat("yyyy-MM-dd").format(date); } else { value = ""; } } else { value = new DecimalFormat("0").format(cell.getNumericCellValue()); } break; case HSSFCell.CELL_TYPE_FORMULA: // 导入时如果为公式生成的数据则无值 if (!cell.getStringCellValue().equals("")) { value = cell.getStringCellValue(); } else { value = cell.getNumericCellValue() + ""; } break; case HSSFCell.CELL_TYPE_BLANK: break; case HSSFCell.CELL_TYPE_ERROR: break; case HSSFCell.CELL_TYPE_BOOLEAN: value = (cell.getBooleanCellValue() == true ? "Y" : "N"); break; default: value = ""; } return value.trim(); } }
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
package com.example.demo.util; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; import java.net.URL; import java.net.URLConnection; import java.util.List; import java.util.Map; public class HttpRequest { /** * 向指定URL发送GET方法的请求 * * @param url 发送请求的URL * @param param 请求参数,请求参数应该是 name1=value1&name2=value2 的形式。 * @return URL 所代表远程资源的响应结果 */ public static String sendGet(String url, String param) { String result = ""; BufferedReader in = null; try { String urlNameString = url + "?" + param; URL realUrl = new URL(urlNameString); // 打开和URL之间的连接 URLConnection connection = realUrl.openConnection(); // 设置通用的请求属性 connection.setRequestProperty("accept", "*/*"); connection.setRequestProperty("connection", "Keep-Alive"); connection.setRequestProperty("Charsert", "GBK"); //设置请求编码 connection.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)"); // 建立实际的连接 connection.connect(); // 获取所有响应头字段 Map<String, List<String>> map = connection.getHeaderFields(); // 遍历所有的响应头字段 for (String key : map.keySet()) { System.out.println(key + "--->" + map.get(key)); } // 定义 BufferedReader输入流来读取URL的响应 in = new BufferedReader(new InputStreamReader( connection.getInputStream(), "GBK")); String line; while ((line = in.readLine()) != null) { result += line; } } catch (Exception e) { System.out.println("get请求异常, 网站[" + url + "]"); // System.out.println("发送GET请求出现异常!" + e); // e.printStackTrace(); } // 使用finally块来关闭输入流 finally { try { if (in != null) { in.close(); } } catch (Exception e2) { e2.printStackTrace(); } } return result; } /** * 向指定 URL 发送POST方法的请求 * * @param url 发送请求的 URL * @param param 请求参数,请求参数应该是 name1=value1&name2=value2 的形式。 * @return 所代表远程资源的响应结果 */ public static String sendPost(String url, String param) { PrintWriter out = null; BufferedReader in = null; String result = ""; try { URL realUrl = new URL(url); // 打开和URL之间的连接 URLConnection conn = realUrl.openConnection(); // 设置通用的请求属性 conn.setRequestProperty("accept", "*/*"); conn.setRequestProperty("connection", "Keep-Alive"); conn.setRequestProperty("Charsert", "GBK"); //设置请求编码 conn.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)"); // 发送POST请求必须设置如下两行 conn.setDoOutput(true); conn.setDoInput(true); // 获取URLConnection对象对应的输出流 out = new PrintWriter(conn.getOutputStream()); // 发送请求参数 out.print(param); // flush输出流的缓冲 out.flush(); // 定义BufferedReader输入流来读取URL的响应 in = new BufferedReader( new InputStreamReader(conn.getInputStream(), "GBK")); String line; while ((line = in.readLine()) != null) { result += line; } } catch (Exception e) { System.out.println("发送 POST 请求出现异常!" + e); e.printStackTrace(); } //使用finally块来关闭输出流、输入流 finally { try { if (out != null) { out.close(); } if (in != null) { in.close(); } } catch (IOException ex) { ex.printStackTrace(); } } return result; } public static void main(String[] args) { //发送 GET 请求 String s = HttpRequest.sendGet("http://tianqi.2345.com/t/wea_history/js/201701/70188_201701.js", ""); s = s.substring("var weather_str=".length(), s.length() - 1); System.out.println(s); JSONObject object = JSON.parseObject(s); String city = object.get("city").toString(); System.out.println(object.get("city")); JSONArray array = JSONArray.parseArray(object.get("tqInfo").toString()); for (int i = 0; i < array.size(); i++) { String ymd = array.getJSONObject(i).get("ymd").toString(); String bWendu = array.getJSONObject(i).get("bWendu").toString(); String yWendu = array.getJSONObject(i).get("yWendu").toString(); String tianqi = array.getJSONObject(i).get("tianqi").toString(); String fengxiang = array.getJSONObject(i).get("fengxiang").toString(); String fengli = array.getJSONObject(i).get("fengli").toString(); String aqi = array.getJSONObject(i).get("aqi").toString(); String aqiInfo = array.getJSONObject(i).get("aqiInfo").toString(); System.out.println(fengli); } /* //发送 POST 请求 String sr=HttpRequest.sendPost("http://localhost:6144/Home/RequestPostString", "key=123&v=456"); System.out.println(sr);*/ } }
结果类 TqBean
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
package com.example.demo.tianqi; public class TqBean { private String shen; private String city; private String url; private String code; private String ymd; private String bWendu; private String yWendu; private String tianqi; private String fengxiang; private String fengli; private String aqi; private String aqiInfo; public TqBean(String shen, String city, String ymd, String bWendu, String yWendu, String tianqi, String fengxiang, String fengli, String aqi, String aqiInfo) { this.shen = shen; this.city = city; this.ymd = ymd; this.bWendu = bWendu; this.yWendu = yWendu; this.tianqi = tianqi; this.fengxiang = fengxiang; this.fengli = fengli; this.aqi = aqi; this.aqiInfo = aqiInfo; } public TqBean(String shen, String city, String url, String code) { this.shen = shen; this.city = city; this.url = url; this.code = code; } public TqBean() { } public String getShen() { return shen; } public void setShen(String shen) { this.shen = shen; } public String getCity() { return city; } public void setCity(String city) { this.city = city; } public String getUrl() { return url; } public void setUrl(String url) { this.url = url; } public String getYmd() { return ymd; } public void setYmd(String ymd) { this.ymd = ymd; } public String getbWendu() { return bWendu; } public void setbWendu(String bWendu) { this.bWendu = bWendu; } public String getyWendu() { return yWendu; } public void setyWendu(String yWendu) { this.yWendu = yWendu; } public String getTianqi() { return tianqi; } public void setTianqi(String tianqi) { this.tianqi = tianqi; } public String getFengxiang() { return fengxiang; } public void setFengxiang(String fengxiang) { this.fengxiang = fengxiang; } public String getFengli() { return fengli; } public void setFengli(String fengli) { this.fengli = fengli; } public String getAqi() { return aqi; } public void setAqi(String aqi) { this.aqi = aqi; } public String getAqiInfo() { return aqiInfo; } public void setAqiInfo(String aqiInfo) { this.aqiInfo = aqiInfo; } public String getCode() { return code; } public void setCode(String code) { this.code = code; } }
线程类 TianQiThread
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
package com.example.demo.tianqi; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.example.demo.util.HttpRequest; import java.util.Set; public class TianQiThread implements Runnable { private String shen; private String url; private Set<TqBean> resList; private Set<String> errirUrls; public TianQiThread(String shen, String url, Set<TqBean> resList, Set<String> errirUrls) { this.shen = shen; this.url = url; this.resList = resList; this.errirUrls = errirUrls; } @Override public void run() { String res = HttpRequest.sendGet(url, ""); if (res.equals("")) { errirUrls.add(url); } else { res = res.substring("var weather_str=".length(), res.length() - 1); JSONObject object = JSON.parseObject(res); String city = object.get("city").toString(); JSONArray array = JSONArray.parseArray(object.get("tqInfo").toString()); for (int i = 0; i < array.size(); i++) { String ymd = array.getJSONObject(i).get("ymd").toString(); String bWendu = array.getJSONObject(i).get("bWendu").toString(); String yWendu = array.getJSONObject(i).get("yWendu").toString(); String tianqi = array.getJSONObject(i).get("tianqi").toString(); String fengxiang = array.getJSONObject(i).get("fengxiang").toString(); String fengli = array.getJSONObject(i).get("fengli").toString(); String aqi = array.getJSONObject(i).get("aqi").toString(); String aqiInfo = array.getJSONObject(i).get("aqiInfo").toString(); TqBean tq = new TqBean(shen, city, ymd, bWendu, yWendu, tianqi, fengxiang, fengli, aqi, aqiInfo); resList.add(tq); } } } }
处理类 TianQiByYear
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
package com.example.demo.tianqi; import com.example.demo.util.ReadExcel; import org.apache.poi.xssf.usermodel.XSSFCell; import org.apache.poi.xssf.usermodel.XSSFRow; import org.apache.poi.xssf.usermodel.XSSFSheet; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import java.io.File; import java.io.FileOutputStream; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; public class TianQiByYear { private static int corePoolSize = 100; private static int maximumPoolSize = 1000; private static long keepAliveTime = 200; private static Set<TqBean> resList = new HashSet<>(); private static Set<String> errirUrls = new HashSet<>(); public static void main(String[] args) { try { Set<TqBean> list = new HashSet<>(); List<List<String>> citylist = ReadExcel.readExcel("d:\\2345.xlsx"); // 以年为周期生成excel for (int year = 2017; year <= 2017; year++) { resList = new HashSet<>(); errirUrls = new HashSet<>(); for (List<String> row : citylist) { String shen = row.get(0); String city = row.get(1); String code = row.get(2); if (!"河南".equals(shen)) { continue; } for (int month = 1; month <= 12; month++) { String url = "http://tianqi.2345.com/t/wea_history/js/"; String m = month > 9 ? ("" + month) : "0" + month; url += year + m + "/" + code + "_" + year + m + ".js"; TqBean tq = new TqBean(shen, city, url, code); list.add(tq); } } ThreadPoolExecutor executor = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>()); for (TqBean tq : list) { executor.execute(new TianQiThread(tq.getShen(), tq.getUrl(), resList, errirUrls)); } executor.shutdown(); while (true) { //等待线程处理完毕 if (executor.isTerminated()) { for (String s : errirUrls) { System.out.println("失败url: [" + s + "]"); } break; } Thread.sleep(300); } // 导出excel //创建新工作簿 XSSFWorkbook workbook = new XSSFWorkbook(); //新建工作表 XSSFSheet sheet = workbook.createSheet("sheet"); XSSFRow row0 = sheet.createRow(0); XSSFCell cell = row0.createCell(0); XSSFCell cell1 = row0.createCell(1); XSSFCell cell2 = row0.createCell(2); XSSFCell cell3 = row0.createCell(3); XSSFCell cell4 = row0.createCell(4); XSSFCell cell5 = row0.createCell(5); XSSFCell cell6 = row0.createCell(6); XSSFCell cell7 = row0.createCell(7); XSSFCell cell8 = row0.createCell(8); XSSFCell cell9 = row0.createCell(9); cell.setCellValue("省份"); cell1.setCellValue("城市"); cell2.setCellValue("日期"); cell3.setCellValue("最高气温"); cell4.setCellValue("最低气温"); cell5.setCellValue("天气"); cell6.setCellValue("风向"); cell7.setCellValue("风力"); cell8.setCellValue("空气质量"); cell9.setCellValue("质量指数"); int rownum = 1; for (TqBean t : resList) { //创建行,行号作为参数传递给createRow()方法,第一行从0开始计算 XSSFRow row = sheet.createRow(rownum++); //创建单元格,row已经确定了行号,列号作为参数传递给createCell(),第一列从0开始计算 XSSFCell mcell = row.createCell(0); XSSFCell mcell1 = row.createCell(1); XSSFCell mcell2 = row.createCell(2); XSSFCell mcell3 = row.createCell(3); XSSFCell mcell4 = row.createCell(4); XSSFCell mcell5 = row.createCell(5); XSSFCell mcell6 = row.createCell(6); XSSFCell mcell7 = row.createCell(7); XSSFCell mcell8 = row.createCell(8); XSSFCell mcell9 = row.createCell(9); mcell.setCellValue(t.getShen()); mcell1.setCellValue(t.getCity()); mcell2.setCellValue(t.getYmd()); mcell3.setCellValue(t.getbWendu()); mcell4.setCellValue(t.getyWendu()); mcell5.setCellValue(t.getTianqi()); mcell6.setCellValue(t.getFengxiang()); mcell7.setCellValue(t.getFengli()); mcell8.setCellValue(t.getAqi()); mcell9.setCellValue(t.getAqiInfo()); } //输出到磁盘中 FileOutputStream fos = new FileOutputStream(new File("F:\\tianqi\\" + year + "年天气.xlsx")); workbook.write(fos); fos.close(); System.out.println("===========ok========================"); } } catch (Exception e) { e.printStackTrace(); } } }
不积跬步,无以至千里;不积小流,无以成江海。