第二次作业
一、
(1)在中国气象网(http://www.weather.com.cn)给定城市集的7日天气预报,并保存在数据库。
from bs4 import BeautifulSoup from bs4 import UnicodeDammit import urllib.request import sqlite3 class WeatherDB: def openDB(self): self.con = sqlite3.connect("weathers.db") self.cursor = self.con.cursor() try: self.cursor.execute("create table weathers (wNum varchar(16),wCity varchar(16),wDate varchar(16),wWeather varchar(64),wTemp varchar(32),constraint pk_weather primary key (wCity,wDate))") except: self.cursor.execute("delete from weathers") def closeDB(self): self.con.commit() self.con.close() def insert(self,num,city,date,weather,temp): try: self.cursor.execute("insert into weathers (wNum,wCity,wDate,wWeather,wTemp) values(?,?,?,?,?)",(num,city,date,weather,temp)) except Exception as err: print(err) def show(self): self.cursor.execute("select * from weathers") rows = self.cursor.fetchall() print("%-16s%-16s%-16s%-32s%-16s"%("序号","地区","日期","天气信息","温度")) for row in rows: print("%-16s%-16s%-16s%-32s%-16s"%(row[0],row[1],row[2],row[3],row[4])) class WeatherForecast: def __init__(self): self.headers={ "User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre" } self.cityCode={"北京":"101010100","上海":"101020100","广州":"101280101","深圳":"101280601"} def forecastCity(self,city): if city not in self.cityCode.keys(): print(city+" code cannot be found") return url="http://www.weather.com.cn/weather/"+self.cityCode[city]+".shtml" try: req=urllib.request.Request(url,headers=self.headers) data=urllib.request.urlopen(req) data=data.read() dammit=UnicodeDammit(data,["utf-8","gbk"]) data=dammit.unicode_markup soup=BeautifulSoup(data,"lxml") lis=soup.select("ul[class='t clearfix'] li") num=0 for li in lis: try: num=num+1 data=li.select('h1')[0].text weather=li.select('p[class="wea"]')[0].text temp=li.select('p[class="tem"] span')[0].text+"/"+li.select('p[class="tem"] i')[0].text print(num,city,data,weather,temp) self.db.insert(num,city,data,weather,temp) except Exception as err: print(err) except Exception as err: print(err) def process(self,cities): self.db=WeatherDB() self.db.openDB() for city in cities: self.forecastCity(city) self.db.show() self.db.closeDB() ws=WeatherForecast() ws.process(["北京","上海","广州","深圳"]) print("完成")
心得体会
本次作业的代码主要是参考课本上的例题,对书本给出的代码进行理解和复现,约会了数据库的应用
作业二:
(1)用requests和BeautifulSoup库方法定向爬取股票相关信息。
import re import requests def getHtml(fs, pn):#fs控制股票号,pn控制页数 url = "http://56.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409968248217612661_1601548126340&pn=" + str(pn) + "&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=" + fs + "&fields=f12,f14,f2,f3,f4,f5,f6,f7" r = requests.get(url) pat = '"diff":\[\{(.*?)\}\]' data = re.compile(pat, re.S).findall(r.text) return data # 获取股票数据 print("序号\t代码\t名称\t最新价\t涨跌幅\t跌涨额\t成交量\t成交额\t涨幅") def getOnePageStock(sort, fs, pn): data = getHtml(fs, pn) datas = data[0].split("},{") for i in range(len(datas)): line = datas[i].replace('"', "").split(",") print(sort, line[6][4:], line[7][4:], line[0][3:], line[1][3:], line[2][3:], line[3][3:], line[4][3:], line[5][3:]) # 输出 sort += 1 return sort def main(): sort = 1# 代码序号 pn = 1 #页数 fs = { "沪深A股": "m:0+t:6,m:0+t:13,m:0+t:80,m:1+t:2,m:1+t:23", "上证A股": "m:1+t:2,m:1+t:23", "深证A股": "m:0+t:6,m:0+t:13,m:0+t:80", "新股": "m:0+f:8,m:1+f:8", "中小板": "m:0+t:13", "创业板": "m:0+t:80", "科技版": "m:1+t:23" } #要爬取的股票 for i in fs.keys(): sort = getOnePageStock(sort, fs[i], pn) main()
心得体会:与之前作业不同的是,本次作业需要抓取js信息,学会了使用抓包工具获取网页数据。
三、
(1)根据自选3位数+学号后3位选取股票,获取印股票信息。
import re import requests def getHtml(sort): url = "http://push2.eastmoney.com/api/qt/stock/get? ut=fa5fd1943c7b386f172d6893dbfba10b&invt=2&fltt=2&fields=f44,f45,f46,f57,f58&secid=0."+sort+"&cb=jQuery112409396991179940428_1601692476366" r = requests.get(url) data = re.findall('{"f.*?}', r.text) return data # 获取股票数据 print("代码\t名称\t今日开\t今日最高\t今日最低") def getOnePageStock(sort): data = getHtml(sort) datas = data[0].split("},{") for i in range(len(datas)): line = datas[i].replace('"', "").split(",") print(line[3][4:], line[4][4:8], line[2][4:], line[0][5:], line[1][4:]) def main(): sort = "002129" # 并不是每股股票都存在,事先选择一支存在的股票 try: getOnePageStock(sort) except: print("该股票不存在!") main()
心得体会:在上一次作业的基础上,手动选择股票代码