全国疫情数据爬取+可视化展示
练习要求：
1.编程爬取每日最新的疫情统计数据。
2.并将爬取结果导入到数据库中。
3.将可视化结果与统计数据结合，实时显示当前最新数据
设计思路，先找到相对应的的网站，用python爬取网站数据，在进行筛选，找到最新的疫情数据，获取数据并存储到mysql数据库中，再进行jsp界面设计、获取数据库数据，并通过Echart显示在web界面中，以柱状图实现数据的可视化。
代码部分：
python爬虫
# coding=utf-8 import pymysql import requests import json url = "https://c.m.163.com/ug/api/wuhan/app/data/list-total?t=316765403234" header = {"user-agent": "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL Build/OPD1.170816.004) AppleWebKit/537.36 (" "KHTML, like Gecko) Chrome/80.0.3987.116 Mobile Safari/537.36"} response = requests.get(url, headers=header) mydir = json.loads(response.content.decode()) # 数据库的连接 db = pymysql.connect(host='localhost', port=3306, user='root', passwd='123456', db='yiqing_new', charset='utf8') # 使用cursor方法创建一个游标 cursor = db.cursor() # mydir['data']['areaTree'][0]['children']再往后是省份 # print(mydir['data']['areaTree'][0]['children'][33]) for item in mydir['data']['areaTree'][0]['children']: name = item['name'] confirm = item['total']['confirm'] severe = item['total']['severe'] heal = item['total']['heal'] dead = item['total']['dead'] suspect = item['total']['suspect'] idcode = item['id'] lastUpdateTime = item['lastUpdateTime'] # 插入数据 """sql = "insert into new_number(id,name,confirm,suspect,heal,dead,severe,idcode,lastupdatetime) values ({},'{}'," \ "'{}','{}','{}','{}','{}','{}','{}');".format( 0, name, confirm, suspect, heal, dead, severe, idcode, lastUpdateTime)"""
#插入并更新数据
sql = "insert into new_number(id,name,confirm,suspect,heal,dead,severe,idcode,lastupdatetime) values (%s,%s,%s,%s,%s,%s,%s,%s,%s)" \ "on duplicate key update id=values (id),confirm=values (confirm),suspect=values (suspect),heal=values (heal),dead=values (dead),severe=values (severe),idcode=values (idcode),lastupdatetime=values (lastupdatetime)" print(sql) cursor.executemany(sql, [[0, name, confirm, severe, heal,dead,severe,idcode,lastUpdateTime]]) #cursor.execute(sql) db.commit()
页面展示：jsp
1 <%@page import="com.alibaba.fastjson.JSON"%> 2 <%@ page language="java" contentType="text/html; charset=UTF-8" 3 pageEncoding="UTF-8"%> 4 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> 5 <html> 6 <head> 7 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> 8 <title>Insert title here</title> 9 <script src="echarts.min.js"></script> 10 <script src="https://cdn.staticfile.org/jquery/3.2.1/jquery.min.js"></script> 11 <script src="https://cdn.staticfile.org/popper.js/1.15.0/umd/popper.min.js"></script> 12 <script src="https://cdn.staticfile.org/twitter-bootstrap/4.3.1/js/bootstrap.min.js"></script> 13 <style> 14 #one{ 15 /*border: 1px solid black;*/ 16 width: 1000px; 17 height:auto; 18 background-color: ghostwhite; 19 margin: auto; 20 } 21 #main{ 22 width:1000px; 23 height:outo; 24 background-color: ghostwhite; 25 margin: auto; 26 } 27 .table 28 { 29 margin: auto; 30 } 31 </style> 32 </head> 33 <body> 34 <div id="one"> 35 点击查询最新疫情情况: 36 <input type="submit" value="查询" onclick="search()"> 37 </div> 38 <br/> 39 <div id="main" style="width: 100%;height:400px;overflow: auto;"></div> 40 <table class="table"> 41 <thead class="head"> 42 </thead> 43 <tbody class="main"></tbody> 44 </table> 45 <script type="text/javascript"> 46 // 基于准备好的dom，初始化echarts实例 47 var myChart = echarts.init(document.getElementById('main')); 48 49 // 指定图表的配置项和数据 50 myChart.showLoading(); 51 function search() { 52 var date=$("#date").val(); 53 alert("查找成功"); 54 $.post( 55 'YiqingSelvlet', 56 {}, 57 function(msg){ 58 var json=JSON.parse(msg); 59 var size=json.length; 60 var names=[]; 61 var nums=[]; 62 for(i=0;i<size;i++){ 63 names.push(json[i].name); 64 nums.push(json[i].confirm); 65 i++; 66 } 67 68 myChart.hideLoading(); 69 var option = { 70 title: { 71 text: '最新确诊人数' 72 }, 73 tooltip: {}, 74 legend: { 75 data:['确诊人数'] 76 }, 77 xAxis: { 78 data: names 79 }, 80 yAxis: {}, 81 series: [{ 82 name: '确诊人数', 83 type: 'bar', 84 data: nums 85 }] 86 }; 87 myChart.setOption(option); 88 tr="<tr><th>省份</th><th>确诊人数</th><th>疑似人数</th><th>治愈人数</th><th>死亡人数</th><th>重症人数</th><th>编码</th><th>最后更新时间</th></tr>"; 89 $('.head').append(tr); 90 for(i=0;i<size;i++) 91 $('.main').append("<tr></tr>"); 92 $('.main tr').each(function(i){ 93 $(this).append("<td>"+json[i].name+"</td>"); 94 $(this).append("<td>"+json[i].confirm+"</td>"); 95 $(this).append("<td>"+json[i].suspect+"</td>"); 96 $(this).append("<td>"+json[i].heal+"</td>"); 97 $(this).append("<td>"+json[i].dead+"</td>"); 98 $(this).append("<td>"+json[i].severe+"</td>"); 99 $(this).append("<td>"+json[i].idcode+"</td>"); 100 $(this).append("<td>"+json[i].lastupdatetime+"</td>"); 101 }) 102 103 }, 104 //"json" 105 ); 106 107 } 108 109 </script> 110 </body> 111 </html>
selvlet部分代码：
package selvlet; import java.io.IOException; import javax.servlet.ServletException; import javax.servlet.annotation.WebServlet; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import dbutil.YiqingDButil; /** * Servlet implementation class YiqingSelvlet */ @WebServlet("/YiqingSelvlet") public class YiqingSelvlet extends HttpServlet { private static final long serialVersionUID = 1L; /** * @see HttpServlet#HttpServlet() */ public YiqingSelvlet() { super(); // TODO Auto-generated constructor stub } /** * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) */ protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { // TODO Auto-generated method stub response.getWriter().append("Served at: ").append(request.getContextPath()); } /** * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) */ protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { // TODO Auto-generated method stub //doGet(request, response); request.setCharacterEncoding("utf-8"); response.setContentType("text/html;charset=UTF-8"); response.getWriter().write(YiqingDButil.getall()); } }
DBUtil部分代码：
1 package dbutil; 2 3 import java.sql.Connection; 4 import java.sql.Date; 5 import java.sql.DriverManager; 6 import java.sql.PreparedStatement; 7 import java.sql.ResultSet; 8 import java.util.ArrayList; 9 10 import com.alibaba.fastjson.JSONArray; 11 import com.alibaba.fastjson.JSONObject; 12 13 import javabeen.Things; 14 import javabeen.yiqing; 15 16 public class YiqingDButil { 17 //数据库URL和账号密码 18 public static final String connectionURL="jdbc:mysql://localhost:3306/yiqing_new?useUnicode=true&characterEncoding=GB18030&useSSL=false&serverTimezone=GMT&allowPublicKeyRetrieval=true"; 19 public static final String username="root"; 20 public static final String password="123456"; 21 static Connection connection; 22 static ResultSet rSet; 23 static PreparedStatement sql; 24 //数据库连接 25 public static Connection getConnection() 26 { 27 try { 28 Class.forName("com.mysql.cj.jdbc.Driver"); 29 return DriverManager.getConnection(connectionURL, username, password); 30 } catch (Exception e) { 31 // TODO: handle exception 32 System.out.println("数据库连接失败"); 33 e.printStackTrace(); 34 } 35 return null; 36 } 37 public static String getall()//bianli 38 { 39 try { 40 41 connection=getConnection(); 42 sql=connection.prepareStatement("select * from new_number"); 43 System.out.println(sql); 44 rSet=sql.executeQuery(); 45 JSONArray jsonArray=new JSONArray(); 46 while(rSet.next()) 47 { 48 //System.out.println(rSet.getString(3)); 49 JSONObject json=new JSONObject(); 50 json.put("name",rSet.getString(2) ); 51 json.put("confirm", rSet.getString(3)); 52 json.put("suspect",rSet.getString(4)); 53 json.put("heal", rSet.getString(5)); 54 json.put("dead",rSet.getString(6)); 55 json.put("severe", rSet.getString(7)); 56 json.put("idcode", rSet.getString(8)); 57 json.put("lastupdatetime", rSet.getString(9)); 58 jsonArray.add(json); 59 } 60 return jsonArray.toString(); 61 } catch (Exception e) { 62 // TODO: handle exception 63 return null; 64 } 65 66 } 67 // public static void main(String[] args) { 68 // search("2020-02-12"); 69 // } 70 71 }
结果展示：
爬取的2020-03-11的疫情数据
最后是本次实验的PSP表格：
