第五次作业
作业一:爬取京东手机信息
要求：
熟练掌握 Selenium 查找HTML元素、爬取Ajax网页数据、等待HTML元素等内容。
使用Selenium框架爬取京东商城某类商品信息及图片。
候选网站：http://www.jd.com/
代码内容：
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import urllib.request
import threading
import sqlite3
import os
import pymysql
import datetime
from selenium.webdriver.common.keys import Keys
import time
class MySpider:
headers = {
"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre"}
imagePath = "download"
def startUp(self, url, key):
# # Initializing Chrome browser
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
self.driver = webdriver.Chrome(chrome_options=chrome_options)
# Initializing variables
self.threads = []
self.No = 0
self.imgNo = 0
# Initializing database
try:
self.con = pymysql.connect(host="127.0.0.1", port=3306, user="root",
passwd="5322005abc", db="mysql", charset="utf8")
self.cursor = self.con.cursor(pymysql.cursors.DictCursor)
try:
# 如果有表就删除
self.cursor.execute("drop table phones")
except:
pass
try:
# 建立新的表
sql = "create table phones (mNo varchar(32) primary key, mMark varchar(256),mPrice varchar(32),mNote varchar(1024),mFile varchar(256))"
self.cursor.execute(sql)
except:
pass
except Exception as err:
print(err)
# Initializing images folder
try:
if not os.path.exists(MySpider.imagePath):
os.mkdir(MySpider.imagePath)
images = os.listdir(MySpider.imagePath)
for img in images:
s = os.path.join(MySpider.imagePath, img)
os.remove(s)
except Exception as err:
print(err)
self.driver.get(url)
keyInput = self.driver.find_element_by_id("key")
keyInput.send_keys(key)
keyInput.send_keys(Keys.ENTER)
def closeUp(self):
try:
self.con.commit()
self.con.close()
self.driver.close()
except Exception as err:
print(err);
def insertDB(self, mNo, mMark, mPrice, mNote, mFile):
try:
sql = "insert into phones (mNo,mMark,mPrice,mNote,mFile) values (?,?,?,?,?)"
self.cursor.execute(sql, (mNo, mMark, mPrice, mNote, mFile))
except Exception as err:
print(err)
def showDB(self):
try:
con = sqlite3.connect("phones.db")
cursor =con.cursor()
print("%-8s%-16s%-8s%-16s%s"%("No", "Mark", "Price", "Image", "Note"))
cursor.execute("select mNo,mMark,mPrice,mFile,mNote from phones order by mNo")
rows = cursor.fetchall()
for row in rows:
print("%-8s %-16s %-8s %-16s %s" % (row[0], row[1], row[2], row[3],row[4]))
con.close()
except Exception as err:
print(err)
def download(self, src1, src2, mFile):
data = None
if src1:
try:
req = urllib.request.Request(src1, headers=MySpider.headers)
resp = urllib.request.urlopen(req, timeout=10)
data = resp.read()
except:
pass
if not data and src2:
try:
req = urllib.request.Request(src2, headers=MySpider.headers)
resp = urllib.request.urlopen(req, timeout=10)
data = resp.read()
except:
pass
if data:
print("download begin", mFile)
fobj = open(MySpider.imagePath + "\\" + mFile, "wb")
fobj.write(data)
fobj.close()
print("download finish", mFile)
def processSpider(self):
try:
time.sleep(1)
print(self.driver.current_url)
lis =self.driver.find_elements_by_xpath("//div[@id='J_goodsList']//li[@class='gl-item']")
for li in lis:
# We find that the image is either in src or in data-lazy-img attribute
try:
src1 = li.find_element_by_xpath(".//div[@class='p-img']//a//img").get_attribute("src")
except:
src1 = ""
try:
src2 = li.find_element_by_xpath(".//div[@class='p-img']//a//img").get_attribute("data-lazy-img")
except:
src2 = ""
try:
price = li.find_element_by_xpath(".//div[@class='p-price']//i").text
except:
price = "0"
try:
note = li.find_element_by_xpath(".//div[@class='p-name p-name-type-2']//em").text
mark = note.split(" ")[0]
mark = mark.replace("爱心东东\n", "")
mark = mark.replace(",", "")
note = note.replace("爱心东东\n", "")
note = note.replace(",", "")
except:
note = ""
mark = ""
self.No = self.No + 1
no = str(self.No)
while len(no) < 6:
no = "0" + no
print(no, mark, price)
if src1:
src1 = urllib.request.urljoin(self.driver.current_url, src1)
p = src1.rfind(".")
mFile = no + src1[p:]
elif src2:
src2 = urllib.request.urljoin(self.driver.current_url, src2)
p = src2.rfind(".")
mFile = no + src2[p:]
if src1 or src2:
T = threading.Thread(target=self.download, args=(src1, src2, mFile))
T.setDaemon(False)
T.start()
self.threads.append(T)
else:
mFile = ""
self.insertDB(no, mark, price, note, mFile)
# 取下一页的数据，直到最后一页
try:
self.driver.find_element_by_xpath("//span[@class='p-num']//a[@class='pn-next disabled']")
except:
nextPage = self.driver.find_element_by_xpath("//span[@class='p-num']//a[@class='pn-next']")
time.sleep(10)
nextPage.click()
self.processSpider()
except Exception as err:
print(err)
def executeSpider(self, url, key):
starttime = datetime.datetime.now()
print("Spider starting......")
self.startUp(url, key)
print("Spider processing......")
self.processSpider()
print("Spider closing......")
self.closeUp()
for t in self.threads:
t.join()
print("Spider completed......")
endtime = datetime.datetime.now()
elapsed = (endtime - starttime).seconds
print("Total ", elapsed, " seconds elapsed")
url = "http://www.jd.com"
spider = MySpider()
while True:
print("1.爬取")
print("2.显示")
print("3.退出")
value = input("请选择(1,2,3):")
if value == "1":
spider.executeSpider(url, "笔记本电脑")
continue
elif value == "2":
spider.showDB()
continue
elif value == "3":
break
结果：
本次作业是复现课上的代码，加深对selenium的理解
作业二：selenium+Xpath+Mysql爬取东方财富网
要求：
熟练掌握 Selenium 查找HTML元素、爬取Ajax网页数据、等待HTML元素等内容。
使用Selenium框架+ MySQL数据库存储技术路线爬取“沪深A股”、“上证A股”、“深证A股”3个板块的股票数据信息。
候选网站：东方财富网：http://quote.eastmoney.com/center/gridlist.html#hs_a_board
代码内容：
import pymysql
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import datetime
import time
class MySpider:
def startup(self, url):
# 初始化谷歌浏览器
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
self.driver = webdriver.Chrome(chrome_options=chrome_options)
# 初始化变量
self.board_name = ["沪深A股", "上证A股", "深证A股"] # 要爬取的版块
self.board = ["nav_hs_a_board", "nav_sh_a_board", "nav_sz_a_board"] # 要点击的版块按钮的id属性
self.TAB_NO = 1
self.curr = 1
self.count = 0
# 初始化数据库
try:
self.con = pymysql.connect(host="127.0.0.1", port=3306, user="root",
passwd="yang6106", db="test", charset="utf8")
self.cursor = self.con.cursor(pymysql.cursors.DictCursor)
try:
# 如果有表就删除
self.cursor.execute("drop table ALLStocks")
self.cursor.execute("drop table HSAStocks")
self.cursor.execute("drop table SHAStocks")
self.cursor.execute("drop table SZAStocks")
except:
pass
try:
# 建立新的表
sql = "CREATE TABLE IF NOT EXISTS ALLStocks (id varchar(256), 股票代码 varchar(256), 板块 varchar(256), 股票名称 varchar(256), 最新报价 varchar(256), " \
"涨跌幅 varchar(256), 涨跌额 varchar(256), 成交量 varchar(256), 成交额 varchar(256), 振幅 varchar(256), " \
"最高 varchar(256), 最低 varchar(256), 今开 varchar(256) , 昨收 varchar(256),primary key (id,板块))"
sql1 = "create table HSAStocks (id varchar(256) primary key, 股票代码 varchar(256), 股票名称 varchar(256), 最新报价 varchar(256), " \
"涨跌幅 varchar(256), 涨跌额 varchar(256), 成交量 varchar(256), 成交额 varchar(256), 振幅 varchar(256), " \
"最高 varchar(256), 最低 varchar(256), 今开 varchar(256) , 昨收 varchar(256))"
sql2 = "create table SHAStocks (id varchar(256) primary key, 股票代码 varchar(256), 股票名称 varchar(256), 最新报价 varchar(256), " \
"涨跌幅 varchar(256), 涨跌额 varchar(256), 成交量 varchar(256), 成交额 varchar(256), 振幅 varchar(256), " \
"最高 varchar(256), 最低 varchar(256), 今开 varchar(256) , 昨收 varchar(256))"
sql3 = "create table SZAStocks (id varchar(256) primary key, 股票代码 varchar(256), 股票名称 varchar(256), 最新报价 varchar(256), " \
"涨跌幅 varchar(256), 涨跌额 varchar(256), 成交量 varchar(256), 成交额 varchar(256), 振幅 varchar(256), " \
"最高 varchar(256), 最低 varchar(256), 今开 varchar(256) , 昨收 varchar(256))"
self.cursor.execute(sql)
self.cursor.execute(sql1)
self.cursor.execute(sql2)
self.cursor.execute(sql3)
except:
pass
except Exception as err:
print(err)
# 打开网站并获取当前总页数
self.driver.get(url)
self.final = 3
def closeup(self):
try:
# 关闭数据库、断开与谷歌浏览器连接
self.con.commit()
self.con.close()
self.driver.close()
except Exception as err:
print(err)
def insertdb(self, table, id, board, Symbol, Name, Last_Trade, Chg, Change, Volume, Turnover, zhengfu, highest,
lowest, Today_Open, Prev_Close):
try:
sql = "insert into " + table + "(id,板块,股票代码,股票名称,最新报价,涨跌幅,涨跌额,成交量,成交额,振幅,最高,最低,今开,昨收) " \
"values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
self.cursor.execute(sql, (id, board, Symbol, Name, Last_Trade, Change, Chg, Volume, Turnover, zhengfu,
highest, lowest, Today_Open, Prev_Close))
except Exception as err:
print(err)
def insertDB(self, table, id, Symbol, Name, Last_Trade, Chg, Change, Volume, Turnover, zhengfu,
highest,
lowest, Today_Open, Prev_Close):
try:
sql = "insert into " + table + "(id,股票代码,股票名称,最新报价,涨跌幅,涨跌额,成交量,成交额,振幅,最高,最低,今开,昨收) " \
"values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
self.cursor.execute(sql,
(id, Symbol, Name, Last_Trade, Change, Chg, Volume, Turnover, zhengfu,
highest, lowest, Today_Open, Prev_Close))
except Exception as err:
print(err)
def processspider(self):
try:
print("Page", self.curr)
WebDriverWait(self.driver, 1000).until(
EC.presence_of_all_elements_located((By.XPATH, "//table[@id='table_wrapper-table']/tbody/tr")))
trs = self.driver.find_elements_by_xpath("//table[@id='table_wrapper-table']/tbody/tr")
for tr in trs:
id = tr.find_element_by_xpath(".//td[1]").text
Symbol = tr.find_element_by_xpath("./td[2]/a").text
Name = tr.find_element_by_xpath("./td[3]/a").text
Last_Trade = tr.find_element_by_xpath("./td[5]/span").text
Chg = tr.find_element_by_xpath("./td[6]/span").text
Change = tr.find_element_by_xpath("./td[7]/span").text
Volume = tr.find_element_by_xpath("./td[8]").text
Turnover = tr.find_element_by_xpath("./td[9]").text
amplitude = tr.find_element_by_xpath("./td[10]").text
highest = tr.find_element_by_xpath("./td[11]/span").text
lowest = tr.find_element_by_xpath("./td[12]/span").text
Today_Open = tr.find_element_by_xpath("./td[13]/span").text
Prev_Close = tr.find_element_by_xpath("./td[14]").text
board = self.board_name[self.TAB_NO]
self.count += 1
print(id, Symbol, Name, Last_Trade, Chg, Change, Volume, Turnover, amplitude, highest, lowest,
Today_Open, Prev_Close)
self.insertdb("ALLStocks", id, board, Symbol, Name, Last_Trade, Chg, Change, Volume, Turnover, amplitude,
highest, lowest, Today_Open, Prev_Close)
if self.TAB_NO == 1:
self.insertDB("HSAStocks", id, Symbol, Name, Last_Trade, Chg, Change, Volume, Turnover, amplitude,
highest, lowest, Today_Open, Prev_Close)
elif self.TAB_NO == 2:
self.insertDB("SHAStocks", id, Symbol, Name, Last_Trade, Chg, Change, Volume, Turnover, amplitude,
highest, lowest, Today_Open, Prev_Close)
else:
self.insertDB("SZAStocks", id, Symbol, Name, Last_Trade, Chg, Change, Volume, Turnover, amplitude,
highest, lowest, Today_Open, Prev_Close)
# 翻页直到最后一页
if self.curr < self.final:
WebDriverWait(self.driver, 1000).until(EC.presence_of_element_located(
(By.XPATH, "//div[@class='dataTables_paginate paging_input']/a[position()=2]")))
nextPage = self.driver.find_element_by_xpath(
"//div[@class='dataTables_paginate paging_input']/a[position()=2]")
self.driver.execute_script('arguments[0].click()', nextPage)
time.sleep(3)
self.curr += 1
self.processspider()
# 翻模块
if self.TAB_NO < 3:
self.TAB_NO += 1
nextTab = self.driver.find_element_by_xpath(
"//ul[@class='tab-list clearfix']/li[position()=" + str(self.TAB_NO) + "]/a")
self.driver.execute_script('arguments[0].click()', nextTab)
time.sleep(3)
WebDriverWait(self.driver, 1000).until(
EC.presence_of_element_located((By.XPATH, "//span[@class='paginate_page']/a[last()]")))
self.final = 3
self.curr = 1
print()
print(self.TAB_NO)
print()
self.processspider()
except Exception as err:
print(err)
def executespider(self, url):
starttime = datetime.datetime.now()
print("开始爬取")
self.startup(url)
print("处理中")
print()
self.processspider()
print("爬取结束")
self.closeup()
print("爬取完成")
endtime = datetime.datetime.now() # 计算爬虫耗时
elapsed = (endtime - starttime).seconds
print("Total ", elapsed, " seconds elapsed")
def main():
url = "http://quote.eastmoney.com/center/gridlist.html#hs_a_board"
spider = MySpider()
spider.executespider(url)
if __name__ == '__main__':
main()
结果：
之前数据库无法插入中文，然后按网上的方法导致了数据库无法联网，没办法打开，后面再试了试终于好了
作业三:爬取mooc课程资源信息
要求：
熟练掌握 Selenium 查找HTML元素、实现用户模拟登录、爬取Ajax网页数据、等待HTML元素等内容。
使用Selenium框架+MySQL爬取中国mooc网课程资源信息（课程号、课程名称、学校名称、主讲教师、团队成员、参加人数、课程进度、课程简介）
候选网站：中国mooc网：https://www.icourse163.org
代码：
import re
import pymysql
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import datetime
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait
class MySpider:
headers = {
"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre"}
def startUp(self, url):
# 实例化浏览器对象
chrome_options = Options()
# 不显示打开浏览器的界面
# 指定谷歌浏览器路径
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--no-sandbox')
self.driver = webdriver.Chrome(chrome_options=chrome_options)
self.driver.get(url)
self.count = 0 # 爬取信息数量
self.page = 1 # 爬取页数
locator = (By.XPATH, "//div[@id='app']/div/div/div[2]/div[2]/div/div[2]/div[1]")
# 等待网页数据加载
WebDriverWait(self.driver, 10, 0.5).until(expected_conditions.presence_of_element_located(locator))
# 连接 MySQL数据库，并创建操作游标self.cursor
try:
print("连接数据库")
self.con = pymysql.connect(host="127.0.0.1", port=3306, user="root",
passwd="yang6106", db="test", charset="utf8")
self.cursor = self.con.cursor(pymysql.cursors.DictCursor)
try:
# 如果有表就删除
self.cursor.execute("DROP TABLE IF EXISTS courses")
except:
pass
try:
# 建立新的表
sql = "CREATE TABLE courses(Id int,课程名称 VARCHAR (256),学校名称 VARCHAR(256)," \
"主讲教师 VARCHAR(256),团队成员 VARCHAR(256),参加人数 VARCHAR(256)," \
"课程进度 VARCHAR(256),课程简介 VARCHAR(1024),PRIMARY KEY (id))"
self.cursor.execute(sql)
except Exception as err:
print(err)
except Exception as err:
print("数据库连接失败")
exit()
def closeUp(self):
try:
self.con.commit()
self.driver.close()
print("总共爬取", self.count, "条信息")
except Exception as err:
print(err)
def processSpider(self):
try:
divs = self.driver.find_elements_by_xpath("//div[@class='_1aoKr']/div[@class='_1gBJC']/div[""@class='_2mbYw']")
for div in divs:
try:
course = div.find_element_by_xpath(".//div[@class='WFpCn']/h3").text
college = div.find_element_by_xpath(".//div[@class='WFpCn']/p").text
teacher = div.find_element_by_xpath(".//div[@class='WFpCn']/div").text
count = re.sub("\D", "", div.find_element_by_xpath(".//div[@class='jvxcQ']//span").text)
# 点击课程，打开课程详情新页面
self.driver.execute_script("arguments[0].click();", div.find_element_by_xpath(".//div[@class='_3KiL7']"))
# 跳转到新打开的页面
self.driver.switch_to.window(self.driver.window_handles[-1])
# 等待新页面加载完成
time.sleep(2)
process = self.driver.find_element_by_xpath(
"//div[@class='course-enroll-info_course-info_term-info_term-time']/span[2]").text
brief = self.driver.find_element_by_xpath("//div[@id='j-rectxt2']").text
team = self.driver.find_elements_by_xpath(
"//div[@class='um-list-slider_con_item']//h3[@class='f-fc3']")
cTeam=""
for t in team:
cTeam += " "+t.text
# 关闭新页面，返回原来的页面
self.driver.close()
self.driver.switch_to.window(self.driver.window_handles[0])
self.count += 1
id = self.count
self.insertDB(id, course, college, teacher, cTeam, count, process, brief)
print(id, course, college, teacher, cTeam, count, process, brief)
except:
pass
print("第", self.page, "页爬取完成")
# 爬取前4页
if self.page <= 4:
self.page += 1
nextPage = self.driver.find_element_by_xpath("//div[@id='app']/div/div/div[2]/div[2]/div/div[2]/div[2]/div/a[10]")
nextPage.click()
time.sleep(10)
self.processSpider()
except Exception as err:
print(err)
def insertDB(self, id, course, college, teacher, cTeam, count, process, brief):
try:
self.cursor.execute(
"insert into courses(Id,课程名称,学校名称,主讲教师,团队成员,参加人数,课程进度,课程简介) values(%s,%s,%s,%s,%s,%s,%s,%s)",
(id, course, college, teacher, cTeam, count, process, brief))
except Exception as err:
print(err)
print("数据插入失败")
def executeSpider(self, url):
starttime = datetime.datetime.now()
print("爬虫开始......")
self.startUp(url)
print("爬虫处理......")
self.processSpider()
print("爬虫结束......")
self.closeUp()
self.showDB()
endtime = datetime.datetime.now()
time = (endtime - starttime).seconds
print("一共花了", time, "秒")
def main():
URL = "https://www.icourse163.org/channel/3008.htm"
spider = MySpider()
spider.executeSpider(URL)
if __name__ == '__main__':
main()
结果：
第三个作业与第二个作业很相近
