《软件工程》个人作业3----网络爬虫爬取疫情数据
1.要求:
1.编程爬取每日最新的疫情统计数据。
2.并将爬取结果导入到数据库中。
3.将可视化结果与统计数据结合,实时显示当前最新数据。
2.截图:
# _*_ coding utf-8 _*_ #开发人员:MR-WildFire #开发时间:2020/3/12 18:45 #文件名称web_crawler.py #开发工具PyCharm import requests import time, json import sys import pymysql def get_wangyi_request(): url = 'https://c.m.163.com/ug/api/wuhan/app/data/list-total' headers = { 'accept': '*/*', 'accept-encoding': 'gzip,deflate,br', 'accept-language': 'en-US,en;q=0.9,zh-CN;q = 0.8,zh;q = 0.7', 'origin': 'https://wp.m.163.com', 'referer': 'https://wp.m.163.com/', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-ite', 'user-agent': 'Mozilla/5.0(WindowsNT10.0;Win64;x64) AppleWebKit/37.36 (KHTML, likeGecko) Chrome/82.0.4056.0 Safari/537.36 Edg/82.0.432.3' } result = requests.get(url, headers=headers) return result def print_mess1(string: str, dict1total: dict): sys.stdout.write(string + '确诊: ' + str(dict1total['confirm'] if dict1total['confirm'] != None else 0)) sys.stdout.write(' ') sys.stdout.write(string + '疑似: ' + str(dict1total['suspect'] if dict1total['suspect'] != None else 0)) sys.stdout.write(' ') sys.stdout.write(string + '治愈: ' + str(dict1total['heal'] if dict1total['heal'] != None else 0)) sys.stdout.write(' ') sys.stdout.write(string + '死亡: ' + str(dict1total['dead'] if dict1total['dead'] != None else 0)) if __name__ == '__main__': result = get_wangyi_request() json_str = json.loads(result.text)['data'] # print(json_str.keys()) # dict_keys(['chinaTotal', 'chinaDayList', 'lastUpdateTime', 'areaTree']) print(json_str['lastUpdateTime']) province_list = json_str['areaTree'][0]['children'] # 每个省份包含如下的键 # dict_keys(['today', 'total', 'extData', 'name', 'id', 'lastUpdateTime', 'children']) conn = pymysql.connect( host='localhost', # 我的IP地址 port=3306, # 不是字符串不需要加引号。 user='root', password='20000604', db='database', charset='utf8' ) cursor = conn.cursor() # 获取一个光标 id = 0; for dict in province_list: sql = 'insert into pachong1 (province,total_confirm,total_suspect,total_heal,total_dead,today_confirm,today_suspect,today_heal,today_dead,today_lastUpdate,id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);' province=dict['name'] total_confirm=dict['total']['confirm'] total_suspect=dict['total']['suspect'] total_heal=dict['total']['heal'] total_dead=dict['total']['dead'] today_confirm=dict['today']['confirm'] today_suspect=dict['today']['suspect'] today_heal=dict['today']['heal'] today_dead=dict['today']['dead'] today_lastUpdate= dict['lastUpdateTime'] id=id+1 sys.stdout.write( dict['name'] + ' ') cursor.execute(sql, [province,total_confirm, total_suspect,total_heal,total_dead,today_confirm,today_suspect,today_heal,today_dead,today_lastUpdate,id]) print() conn.commit() cursor.close() conn.close()