爬取最新疫情数据
题目:
编程爬取每日最新的疫情统计数据。
并将爬取结果导入到数据库中。
将可视化结果与统计数据结合,实时显示当前最新数据。
这次的作业与上次周的可视化可以整合成一个完整的代码,只需要在这次加上python爬取数据即可
本次爬取的是丁香医生网站的数据,网址为:https://ncov.dxy.cn/ncovh5/view/pneumonia
爬取的代码如下
1 from os import path 2 import requests 3 from bs4 import BeautifulSoup 4 import json 5 import pymysql 6 #import numpy as np 7 import time 8 from _ast import Try 9 10 url = 'https://ncov.dxy.cn/ncovh5/view/pneumonia?from=timeline&isappinstalled=0' #请求地址 11 headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'}#创建头部信息 12 response = requests.get(url,headers = headers) #发送网络请求 13 #print(response.content.decode('utf-8'))#以字节流形式打印网页源码 14 content = response.content.decode('utf-8') 15 #print(content) 16 soup = BeautifulSoup(content, 'html.parser') 17 listA = soup.find_all(name='script',attrs={"id":"getAreaStat"}) 18 #世界确诊 19 listB = soup.find_all(name='script',attrs={"id":"getListByCountryTypeService2"}) 20 #listA = soup.find_all(name='div',attrs={"class":"c-touchable-feedback c-touchable-feedback-no-default"}) 21 account = str(listA) 22 world_messages = str(listB)[87:-21] 23 messages = account[52:-21] 24 messages_json = json.loads(messages) 25 world_messages_json = json.loads(world_messages) 26 valuesList = [] 27 cityList = [] 28 worldList = [] 29 localtime = time.localtime(time.time()) 30 L=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 31 print(L) 32 for i in range(len(messages_json)): 33 #value = messages_json[i] 34 #value = (messages_json[i].get('provinceName'),messages_json[i].get('provinceShortName'),messages_json[i].get('currentConfirmedCount'),messages_json[i].get('confirmedCount'),messages_json[i].get('suspectedCount'),messages_json[i].get('curedCount'),messages_json[i].get('deadCount'),messages_json[i].get('comment'),messages_json[i].get('locationId')) 35 value = (messages_json[i].get('provinceName'),messages_json[i].get('confirmedCount'),messages_json[i].get('curedCount'),messages_json[i].get('deadCount'),messages_json[i].get('locationId')) 36 valuesList.append(value) 37 cityValue = messages_json[i].get('cities') 38 #print(cityValue) 一个省内没有划分开的值 39 for j in range(len(cityValue)): 40 #cityValueList = (cityValue[j].get('cityName'),cityValue[j].get('currentConfirmedCount'),cityValue[j].get('confirmedCount'),cityValue[j].get('suspectedCount'),cityValue[j].get('curedCount'),cityValue[j].get('deadCount'),cityValue[j].get('locationId'),messages_json[i].get('provinceShortName')) 41 cityValueList = (messages_json[i].get('provinceName'),cityValue[j].get('cityName'),cityValue[j].get('confirmedCount'),cityValue[j].get('curedCount'),cityValue[j].get('deadCount'),cityValue[j].get('locationId')) 42 #print(cityValueList) 省份内各个城市的值 43 cityList.append(cityValueList) 44 45 #print(cityList) #城市 46 #print(valuesList) #省份 47 db=pymysql.connect("localhost","root","123456","payiqing", charset='utf8') 48 cursor = db.cursor() 49 50 sql_city="insert into info_copy (Province,City,Confirmed_num,Cured_num,Dead_num,Code,Date) values (%s,%s,%s,%s,%s,%s,'"+L+"')" 51 sql_province="insert into info_copy (Province,Confirmed_num,Cured_num,Dead_num,Code,Date) values (%s,%s,%s,%s,%s,'"+L+"')" 52 #print(sql) 53 54 value_tuple= tuple(valuesList) 55 city_tuple=tuple(cityList) 56 57 try: 58 cursor.executemany(sql_province,valuesList) 59 cursor.executemany(sql_city,city_tuple) 60 db.commit() 61 except: 62 print('执行失败,进入回调4') 63 db.rollback() 64 65 66 67 db.close()
加上上次的代码,效果如下图所示:
psp表格
缺陷记录日志