Python 爬取 北京市政府首都之窗信件列表-[信息展示]
日期：2020.01.25
博客期：133
星期六
【代码说明，如果要使用此页代码，必须在本博客页面评论区给予说明】
//博客总体说明
1、准备工作
2、爬取工作
3、数据处理
4、信息展示（本期博客）
大年初一，先给大家说声“过年好”！嗯~说说今天写的部分，因为今天过大年嘛，也没有写什么特别难的东西！就是把上一回博客的内容进行整理，然后根据ECharts的接口进行数据可视化的展示！
下面是项目的目录结构：
嗯！老师说基本的页面是按照我的那个老模板来做的，这样比较省力气！
Java部分：
com.dblink包（连接数据库的包），参考博客：https://www.cnblogs.com/onepersonwholive/p/12168785.html
com.servlet包
1 package com.servlet; 2 3 import java.io.IOException; 4 import java.sql.SQLException; 5 import java.util.List; 6 7 import javax.servlet.ServletException; 8 import javax.servlet.ServletOutputStream; 9 import javax.servlet.annotation.WebServlet; 10 import javax.servlet.http.HttpServlet; 11 import javax.servlet.http.HttpServletRequest; 12 import javax.servlet.http.HttpServletResponse; 13 14 import org.json.JSONArray; 15 import org.json.JSONObject; 16 17 import com.dblink.basic.utils.SqlUtils; 18 import com.dblink.basic.utils.sqlKind.MySql_s; 19 import com.dblink.basic.utils.user.UserInfo; 20 import com.dblink.bean.BeanGroup; 21 import com.dblink.sql.DBLink; 22 23 @SuppressWarnings("unused") 24 public class ServletForEveryYear extends HttpServlet{ 25 /** 26 * 27 */ 28 private static final long serialVersionUID = 1L; 29 //----------------------------------------------------------------------// 30 public void doPost(HttpServletRequest request,HttpServletResponse response) throws ServletException, IOException 31 { 32 request.setCharacterEncoding("utf-8"); 33 response.setCharacterEncoding("utf-8"); 34 response.setContentType("application/json"); 35 response.setHeader("Cache-Control", "no-cache"); 36 response.setHeader("Pragma", "no-cache"); 37 38 JSONObject jsonobject = new JSONObject(); 39 40 JSONArray dt = new JSONArray(); 41 JSONArray num = new JSONArray(); 42 43 DBLink dbLink = new DBLink(new SqlUtils(new MySql_s("edu"),new UserInfo("root","123456"))); 44 BeanGroup bg = null; 45 try { 46 bg = dbLink.getSelect("Select * From yeardata").beans; 47 int leng = bg.size(); 48 for(int i=0;i<leng;++i) 49 { 50 dt.put(bg.get(i).get(0)); 51 num.put(bg.get(i).get(1)); 52 } 53 } catch (SQLException e) { 54 // Do Nothing ... 55 } 56 dbLink.free(); 57 jsonobject.put("YEAR",dt); 58 jsonobject.put("NUM",num); 59 60 ServletOutputStream os = response.getOutputStream(); 61 os.write(jsonobject.toString().getBytes()); 62 os.flush(); 63 os.close(); 64 } 65 //---------------------------------------------------------------------------------// 66 }
1 package com.servlet; 2 3 import java.io.IOException; 4 import java.sql.SQLException; 5 import java.util.List; 6 7 import javax.servlet.ServletException; 8 import javax.servlet.ServletOutputStream; 9 import javax.servlet.annotation.WebServlet; 10 import javax.servlet.http.HttpServlet; 11 import javax.servlet.http.HttpServletRequest; 12 import javax.servlet.http.HttpServletResponse; 13 14 import org.json.JSONArray; 15 import org.json.JSONObject; 16 17 import com.dblink.basic.utils.SqlUtils; 18 import com.dblink.basic.utils.sqlKind.MySql_s; 19 import com.dblink.basic.utils.user.UserInfo; 20 import com.dblink.bean.BeanGroup; 21 import com.dblink.sql.DBLink; 22 23 @SuppressWarnings("unused") 24 public class ServletForResponser extends HttpServlet{ 25 /** 26 * 27 */ 28 private static final long serialVersionUID = 1L; 29 //----------------------------------------------------------------------// 30 public void doPost(HttpServletRequest request,HttpServletResponse response) throws ServletException, IOException 31 { 32 request.setCharacterEncoding("utf-8"); 33 response.setCharacterEncoding("utf-8"); 34 response.setContentType("application/json"); 35 response.setHeader("Cache-Control", "no-cache"); 36 response.setHeader("Pragma", "no-cache"); 37 38 JSONObject jsonobject = new JSONObject(); 39 40 JSONArray dt = new JSONArray(); 41 JSONArray num = new JSONArray(); 42 43 DBLink dbLink = new DBLink(new SqlUtils(new MySql_s("edu"),new UserInfo("root","123456"))); 44 BeanGroup bg = null; 45 try { 46 bg = dbLink.getSelect("Select * From responserdata").beans; 47 int leng = bg.size(); 48 for(int i=0;i<5;++i) 49 { 50 dt.put(bg.get(i).get(0)); 51 num.put(bg.get(i).get(1)); 52 } 53 } catch (SQLException e) { 54 // Do Nothing ... 55 } 56 dbLink.free(); 57 jsonobject.put("RESPONSER",dt); 58 jsonobject.put("NUM",num); 59 60 ServletOutputStream os = response.getOutputStream(); 61 os.write(jsonobject.toString().getBytes()); 62 os.flush(); 63 os.close(); 64 } 65 //---------------------------------------------------------------------------------// 66 }
1 package com.servlet; 2 3 import java.io.IOException; 4 import java.sql.SQLException; 5 import java.util.List; 6 7 import javax.servlet.ServletException; 8 import javax.servlet.ServletOutputStream; 9 import javax.servlet.annotation.WebServlet; 10 import javax.servlet.http.HttpServlet; 11 import javax.servlet.http.HttpServletRequest; 12 import javax.servlet.http.HttpServletResponse; 13 14 import org.json.JSONArray; 15 import org.json.JSONObject; 16 17 import com.dblink.basic.utils.SqlUtils; 18 import com.dblink.basic.utils.sqlKind.MySql_s; 19 import com.dblink.basic.utils.user.UserInfo; 20 import com.dblink.bean.BeanGroup; 21 import com.dblink.sql.DBLink; 22 23 @SuppressWarnings("unused") 24 public class ServletForKindData extends HttpServlet{ 25 /** 26 * 27 */ 28 private static final long serialVersionUID = 1L; 29 //----------------------------------------------------------------------// 30 public void doPost(HttpServletRequest request,HttpServletResponse response) throws ServletException, IOException 31 { 32 request.setCharacterEncoding("utf-8"); 33 response.setCharacterEncoding("utf-8"); 34 response.setContentType("application/json"); 35 response.setHeader("Cache-Control", "no-cache"); 36 response.setHeader("Pragma", "no-cache"); 37 38 JSONObject jsonobject = new JSONObject(); 39 40 JSONArray kind = new JSONArray(); 41 JSONArray num = new JSONArray(); 42 JSONArray jsonA = new JSONArray(); 43 44 DBLink dbLink = new DBLink(new SqlUtils(new MySql_s("edu"),new UserInfo("root","123456"))); 45 BeanGroup bg = null; 46 try { 47 bg = dbLink.getSelect("Select * From kinddata").beans; 48 int leng = bg.size(); 49 for(int i=0;i<leng;++i) 50 { 51 JSONObject json = new JSONObject(); 52 json.put("name",bg.get(i).get(0)); 53 json.put("value",bg.get(i).get(1)); 54 kind.put(bg.get(i).get(0)); 55 num.put(bg.get(i).get(1)); 56 jsonA.put(json); 57 } 58 } catch (SQLException e) { 59 // Do Nothing ... 60 } 61 dbLink.free(); 62 jsonobject.put("KIND",kind); 63 jsonobject.put("NUM",num); 64 jsonobject.put("DATA",jsonA); 65 66 ServletOutputStream os = response.getOutputStream(); 67 os.write(jsonobject.toString().getBytes()); 68 os.flush(); 69 os.close(); 70 } 71 //---------------------------------------------------------------------------------// 72 }
JSP + JS 部分:
web.xml部分（配置servlet）
1 <?xml version="1.0" encoding="UTF-8"?> 2 <web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://xmlns.jcp.org/xml/ns/javaee" xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee http://xmlns.jcp.org/xml/ns/javaee/web-app_4_0.xsd" id="WebApp_ID" version="4.0"> 3 <display-name>BeiJingEmailPostAndDeal</display-name> 4 <servlet> 5 <description>This is the description of my J2EE component</description> 6 <display-name>This is the display name of my J2EE component</display-name> 7 <servlet-name>ServletForEveryYear</servlet-name> 8 <servlet-class>com.servlet.ServletForEveryYear</servlet-class> 9 </servlet> 10 <servlet-mapping> 11 <servlet-name>ServletForEveryYear</servlet-name> 12 <url-pattern>/com/servlet/ServletForEveryYear</url-pattern> 13 </servlet-mapping> 14 <servlet> 15 <description>This is the description of my J2EE component</description> 16 <display-name>This is the display name of my J2EE component</display-name> 17 <servlet-name>ServletForResponser</servlet-name> 18 <servlet-class>com.servlet.ServletForResponser</servlet-class> 19 </servlet> 20 <servlet-mapping> 21 <servlet-name>ServletForResponser</servlet-name> 22 <url-pattern>/com/servlet/ServletForResponser</url-pattern> 23 </servlet-mapping> 24 <servlet> 25 <description>This is the description of my J2EE component</description> 26 <display-name>This is the display name of my J2EE component</display-name> 27 <servlet-name>ServletForKindData</servlet-name> 28 <servlet-class>com.servlet.ServletForKindData</servlet-class> 29 </servlet> 30 <servlet-mapping> 31 <servlet-name>ServletForKindData</servlet-name> 32 <url-pattern>/com/servlet/ServletForKindData</url-pattern> 33 </servlet-mapping> 34 <welcome-file-list> 35 <welcome-file>index.html</welcome-file> 36 <welcome-file>index.htm</welcome-file> 37 <welcome-file>index.jsp</welcome-file> 38 <welcome-file>default.html</welcome-file> 39 <welcome-file>default.htm</welcome-file> 40 <welcome-file>default.jsp</welcome-file> 41 </welcome-file-list> 42 </web-app>
jspFiles
1 <%@ page language="java" contentType="text/html; charset=utf-8" 2 pageEncoding="utf-8"%> 3 <!DOCTYPE html> 4 <html><!-- xmlns="http://www.w3.org/1999/xhtml" --> 5 <head> 6 <!--<meta charset="utf-8" />--> 7 <meta name="viewport" content="width=device-width, initial-scale=1.0" charset="utf-8"/> 8 <title>北京市人民政府信件数据分析</title> 9 <!-- BOOTSTRAP STYLES--> 10 <link href="../assets/css/bootstrap.css" rel="stylesheet" /> 11 <!-- FONTAWESOME STYLES--> 12 <link href="../assets/css/font-awesome.css" rel="stylesheet" /> 13 <!-- CUSTOM STYLES--> 14 <link href="../assets/css/custom.css" rel="stylesheet" /> 15 <!-- PERSONAL FONTS--> 16 <link href='../cssFiles/basic.css' rel='stylesheet' type='text/css' /> 17 <!-- GOOGLE FONTS--> 18 <link href='http://fonts.googleapis.com/css?family=Open+Sans' rel='stylesheet' type='text/css' /> 19 </head> 20 <script src="../jsFiles/jquery/jquery-3.4.1.min.js" charset="utf-8"></script> 21 <script src="../jsFiles/echarts/echarts.min.js" charset="utf-8"></script> 22 <script src="../jsFiles/basic.js" charset="utf-8"></script> 23 <script src="../jsFiles/part1.js" charset="utf-8"></script> 24 <script src="../jsFiles/part2.js" charset="utf-8"></script> 25 <script src="../jsFiles/part3.js" charset="utf-8"></script> 26 <body> 27 <div id="wrapper"> 28 <div class="navbar navbar-inverse navbar-fixed-top"> 29 <div class="adjust-nav"> 30 <div class="navbar-header"> 31 <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".sidebar-collapse"> 32 <span class="icon-bar"></span> 33 <span class="icon-bar"></span> 34 <span class="icon-bar"></span> 35 </button> 36 <a class="navbar-brand"><i class="fa fa-square-o "></i> 欢迎您使用本系统</a> 37 </div> 38 </div> 39 </div> 40 <!-- /. NAV TOP --> 41 <div class="navbar-default navbar-side"> <!-- nav role="navigation" --> 42 <div class="sidebar-collapse"> 43 <ul class="nav" id="main-menu"> 44 <li class="text-center user-image-back"> 45 <img src="../assets/img/find_user.png" class="img-responsive" /> 46 </li> 47 <li> 48 <a href="#" onclick="makePageToMain()"><i class="fa fa-table "></i>主页</a> 49 </li> 50 <li> 51 <a href="#"><i class="fa fa-edit "></i>可视化界面<span class="fa arrow"></span></a> 52 <ul class="nav nav-second-level"> 53 <li> 54 <a href="#" onclick="makePageToPart1()">每年信件数量</a> 55 </li> 56 <li> 57 <a href="#" onclick="makePageToPart2()">每种类型的信件数量</a> 58 </li> 59 <li> 60 <a href="#" onclick="makePageToPart3()">每个政府部门处理的信件数量</a> 61 </li> 62 </ul> 63 </li> 64 </ul> 65 </div> 66 </div> 67 <!-- /. NAV SIDE --> 68 <div id="page-wrapper" > 69 <div id="page-inner"> 70 <div class="row"> 71 <div class="col-md-12"> 72 <h2>主页</h2> 73 </div> 74 </div> 75 <!-- /. ROW --> 76 <hr /> 77 <!-- /. ROW --> 78 <br> 79 <br> 80 <div id="MessageArea"> 81 <br> 82 <h3>欢迎您使用本系统，本系统将为您提供三项需求(2020-01-24)</h3> 83 <br> 84 <br> 85 <br> 86 <br> 87 <br> 88 <br> 89 <br> 90 <br> 91 <br> 92 <br> 93 <br> 94 <br> 95 <br> 96 <br> 97 <span> 如有疑问，也欢迎您访问</span><a href='http://www.beijing.gov.cn/hudong/hdjl/com.web.search.mailList.flow'>原网站</a>以访问更多！ 98 <br> 99 </div> 100 </div> 101 <!-- /. PAGE INNER --> 102 </div> 103 <!-- /. PAGE WRAPPER --> 104 </div> 105 <!-- /. WRAPPER --> 106 <!-- SCRIPTS -AT THE BOTOM TO REDUCE THE LOAD TIME--> 107 <!-- JQUERY SCRIPTS --> 108 <script src="../assets/js/jquery-1.10.2.js"></script> 109 <!-- BOOTSTRAP SCRIPTS --> 110 <script src="../assets/js/bootstrap.min.js"></script> 111 <!-- METISMENU SCRIPTS --> 112 <script src="../assets/js/jquery.metisMenu.js"></script> 113 <!-- CUSTOM SCRIPTS --> 114 <script src="../assets/js/custom.js"></script> 115 </body> 116 </html>
jsFiles
1 function makePageToPart1() 2 { 3 var Area = ''; 4 Area += '<div class="row">'; 5 Area += '<div class="col-md-12">'; 6 Area += '<h2>每年信件数量</h2>'; 7 Area += '</div>'; 8 Area += '</div>'; 9 Area += '<hr />'; 10 Area += '<br>'; 11 Area += '<div id="MessageArea">'; 12 Area += ' <div id="displayArea" style="width:1000px;height:500px;"></div>'; 13 Area += '</div>'; 14 document.getElementById("page-inner").innerHTML = Area; 15 loadPart1(); 16 } 17 function loadPart1() 18 { 19 var xmlHttp = null; 20 try{ 21 xmlHttp = new XMLHttpRequest(); 22 } catch (e1) { 23 try { 24 xmlHttp = new ActiveXObject("Microsoft.XMLHTTP"); 25 } catch (e2) { 26 alert("Your browser does not support XMLHTTP!"); 27 return; 28 } 29 } 30 xmlHttp.onreadystatechange = function() { 31 if (xmlHttp.readyState == 4) { 32 if (xmlHttp.status == 200) 33 { 34 s = xmlHttp.responseText; 35 var InformationSet = eval('('+s+')'); 36 var leng = InformationSet; 37 var myChart = echarts.init(document.getElementById('displayArea')); 38 // 指定图表的配置项和数据 39 var option = { 40 title: { 41 subtext: '单位:条' 42 }, 43 xAxis: { 44 name: '年份', 45 type: 'category', 46 boundaryGap: false, 47 axisLine: {onZero: false}, 48 data: InformationSet.YEAR 49 }, 50 yAxis: { 51 name: '数量', 52 type: 'value' 53 }, 54 tooltip: { 55 trigger: 'axis' 56 }, 57 series: [{ 58 name: '数量', 59 data: InformationSet.NUM, 60 type: 'line', 61 markPoint: { 62 data: [ 63 {type: 'max', name: '最大值'}, 64 {type: 'min', name: '最小值'} 65 ] 66 } 67 }] 68 }; 69 // 使用刚指定的配置项和数据显示图表。 70 myChart.setOption(option); 71 } 72 } 73 }; 74 var url ="../com/servlet/ServletForEveryYear"; 75 76 xmlHttp.open("POST", url, true); 77 xmlHttp.setRequestHeader("Content-Type","application/x-www-form-urlencoded"); 78 xmlHttp.send(null); 79 }
1 function makePageToPart2() 2 { 3 var Area = ''; 4 Area += '<div class="row">'; 5 Area += '<div class="col-md-12">'; 6 Area += '<h2>每种类型的信件数量</h2>'; 7 Area += '</div>'; 8 Area += '</div>'; 9 Area += '<hr />'; 10 Area += '<br>'; 11 Area += '<div id="MessageArea">'; 12 Area += ' <div id="displayArea" style="width:1000px;height:500px;"></div>'; 13 Area += '</div>'; 14 document.getElementById("page-inner").innerHTML = Area; 15 loadPart2(); 16 } 17 function loadPart2() 18 { 19 var xmlHttp = null; 20 try{ 21 xmlHttp = new XMLHttpRequest(); 22 } catch (e1) { 23 try { 24 xmlHttp = new ActiveXObject("Microsoft.XMLHTTP"); 25 } catch (e2) { 26 alert("Your browser does not support XMLHTTP!"); 27 return; 28 } 29 } 30 xmlHttp.onreadystatechange = function() { 31 if (xmlHttp.readyState == 4) { 32 if (xmlHttp.status == 200) 33 { 34 s = xmlHttp.responseText; 35 var InformationSet = eval('('+s+')'); 36 var myChart = echarts.init(document.getElementById('displayArea')); 37 var option = { 38 title: { 39 text: '每种类型的信件数量', 40 subtext: '数量', 41 left: 'center' 42 }, 43 tooltip: { 44 trigger: 'item', 45 formatter: '数量 <br/>{b} : {c} ({d}%)' 46 }, 47 legend: { 48 orient: 'vertical', 49 left: 'left', 50 data: InformationSet.KIND 51 }, 52 series: [ 53 { 54 name: '访问来源', 55 type: 'pie', 56 radius: '55%', 57 center: ['50%', '60%'], 58 data: InformationSet.DATA, 59 emphasis: { 60 itemStyle: { 61 shadowBlur: 10, 62 shadowOffsetX: 0, 63 shadowColor: 'rgba(0, 0, 0, 0.5)' 64 } 65 } 66 } 67 ] 68 }; 69 // 使用刚指定的配置项和数据显示图表。 70 myChart.setOption(option); 71 } 72 } 73 }; 74 var url ="../com/servlet/ServletForKindData"; 75 76 xmlHttp.open("POST", url, true); 77 xmlHttp.setRequestHeader("Content-Type","application/x-www-form-urlencoded"); 78 xmlHttp.send(null); 79 }
1 function makePageToPart3() 2 { 3 var Area = ''; 4 Area += '<div class="row">'; 5 Area += '<div class="col-md-12">'; 6 Area += '<h2>每个政府部门处理的信件数量</h2>'; 7 Area += '</div>'; 8 Area += '</div>'; 9 Area += '<hr />'; 10 Area += '<br>'; 11 Area += '<div id="MessageArea">'; 12 Area += ' <div id="displayArea" style="width:1200px;height:500px;"></div>'; 13 Area += '</div>'; 14 document.getElementById("page-inner").innerHTML = Area; 15 loadPart3(); 16 } 17 function loadPart3() 18 { 19 var xmlHttp = null; 20 try{ 21 xmlHttp = new XMLHttpRequest(); 22 } catch (e1) { 23 try { 24 xmlHttp = new ActiveXObject("Microsoft.XMLHTTP"); 25 } catch (e2) { 26 alert("Your browser does not support XMLHTTP!"); 27 return; 28 } 29 } 30 xmlHttp.onreadystatechange = function() { 31 if (xmlHttp.readyState == 4) { 32 if (xmlHttp.status == 200) 33 { 34 s = xmlHttp.responseText; 35 var InformationSet = eval('('+s+')'); 36 var myChart = echarts.init(document.getElementById('displayArea')); 37 var option = { 38 xAxis: { 39 type: 'category', 40 data: InformationSet.RESPONSER 41 }, 42 yAxis: { 43 type: 'value' 44 }, 45 series: [{ 46 data: InformationSet.NUM, 47 type: 'bar' 48 }] 49 }; 50 // 使用刚指定的配置项和数据显示图表。 51 myChart.setOption(option); 52 } 53 } 54 }; 55 var url ="../com/servlet/ServletForResponser"; 56 57 xmlHttp.open("POST", url, true); 58 xmlHttp.setRequestHeader("Content-Type","application/x-www-form-urlencoded"); 59 xmlHttp.send(null); 60 }
接下来是页面展示：
----[ 主页面 ]
----[ 每年信件数量 ]
----[ 每种类型的信件数量]
----[ 每个政府部门处理的信件数量 ]
