数据采集与融合技术第四次实践作业

gitee链接：
https://gitee.com/zxbaixuexi/2024scrapy/tree/master/第四次实践

作业①:

1）

使用 Selenium 框架+ MySQL 数据库存储技术路线爬取“沪深 A 股”、“上证 A 股”、“深证 A 股”3 个板块的股票数据信息。

候选网站：东方财富网：
http://quote.eastmoney.com/center/gridlist.html#hs_a_board

代码：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
import time
from selenium.webdriver.chrome.service import Service as ChromeService
import pymysql

# 连接自己的数据库，根据自己情况修改
conn = pymysql.connect(host="127.0.0.1", port=3306, user='root', passwd='123456', charset='utf8', db='homework1')
cursor = conn.cursor(cursor=pymysql.cursors.DictCursor)

# 显式指定Chrome二进制文件路径（根据实际安装路径进行修改）
chrome_binary_path = r"D:\noneprogramsoft\Google\Chrome\Application\chrome.exe"
#ChromeDriver
CHROMEDRIVER_PATH = r"C:\Users\supermejane\Desktop\爬虫实践\test4\chromedriver.exe"

# 配置Chrome选项
options = webdriver.ChromeOptions()
options.binary_location = chrome_binary_path
#Service
service = ChromeService(executable_path=CHROMEDRIVER_PATH)

# 创建一个WebDriver对象，并指定驱动程序路径
driver = webdriver.Chrome(service=service, options=options)


urls = ["https://quote.eastmoney.com/center/gridlist.html#hs_a_board","https://quote.eastmoney.com/center/gridlist.html#sh_a_board","https://quote.eastmoney.com/center/gridlist.html#sz_a_board"]
tables = ["husheng","shangzheng","shengzheng"]
# 爬取前n页股票信息
def parseData(driver,n,tableName):
    for page in range(1, n + 1):  # 爬取n页
        print(f"正在爬取第 {page} 页股票信息")

        # 定位所有股票信息的元素
        stocks_list = driver.find_elements(By.XPATH, "//div[@class='listview full']//tbody//tr")

        # 打印股票信息
        for stock in stocks_list:
            stock_id = stock.find_element(By.XPATH, './/td[1]').text
            stock_code = stock.find_element(By.XPATH, './/td[2]/a').text
            stock_name = stock.find_element(By.XPATH, './/td[3]/a').text
            stock_latest_price = stock.find_element(By.XPATH, './/td[5]//span').text
            stock_change_percent = stock.find_element(By.XPATH, './/td[6]//span').text
            stock_change_amount = stock.find_element(By.XPATH, './/td[7]//span').text
            stock_volume = stock.find_element(By.XPATH, './/td[8]').text
            stock_turnover = stock.find_element(By.XPATH, './/td[9]').text
            stock_amplitude = stock.find_element(By.XPATH, './/td[10]').text
            stock_highest = stock.find_element(By.XPATH, './/td[11]//span').text
            stock_lowest = stock.find_element(By.XPATH, './/td[12]//span').text
            stock_open_price = stock.find_element(By.XPATH, './/td[13]//span').text
            stock_close_price = stock.find_element(By.XPATH, './/td[14]').text

            sql = f"INSERT INTO {tableName} VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
            cursor.execute(sql,
                           [stock_id, stock_code, stock_name, stock_latest_price, stock_change_percent.strip("%"),
                            stock_change_amount,
                            stock_volume[:-1], stock_turnover[:-1], stock_amplitude.strip("%"), stock_highest,
                            stock_lowest, stock_open_price,
                            stock_close_price])
        conn.commit()
        print("数据已存入数据库")

        # 点击下一页按钮
        next_page_button = driver.find_element(By.XPATH, '//*[@id="main-table_paginate"]/a[2]')
        action = ActionChains(driver)
        time.sleep(5)
        action.move_to_element(next_page_button).perform()
        next_page_button.click()

        time.sleep(5)

table_No=0
for url in urls:
    driver.get(url)
    parseData(driver,2,tables[table_No])
    table_No += 1

conn.close()
# 关闭浏览器
driver.quit()

运行结果
husheng(沪深)

shangzheng(上证)

shengzheng(深证)

2）心得体会

学会了使用selenium爬取网页信息1，使用get获取网页信息，并模拟用户翻页，并可以结合Xpath等方式实现解析

作业②:

1）

使用 Selenium 框架+MySQL 爬取中国 mooc 网课程资源信息（课程号、课程名
称、学校名称、主讲教师、团队成员、参加人数、课程进度、课程简介）

候选网站：中国 mooc 网：https://www.icourse163.org

代码

from time import sleep

from selenium import webdriver
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.chrome.service import Service as ChromeService
import pymysql
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from scrapy.selector import Selector
from selenium.webdriver.common.keys import Keys
"""
链接数据库
"""
conn = pymysql.connect(host="127.0.0.1", port=3306, user='root', passwd='123456', charset='utf8', db='homework1')
cursor = conn.cursor(cursor=pymysql.cursors.DictCursor)

"""
创建ChromeDriver
"""
# 显式指定Chrome二进制文件路径（根据实际安装路径进行修改）
chrome_binary_path = r"D:\noneprogramsoft\Google\Chrome\Application\chrome.exe"
#ChromeDriver
CHROMEDRIVER_PATH = r"C:\Users\supermejane\Desktop\爬虫实践\test4\chromedriver.exe"

# 配置Chrome选项
options = webdriver.ChromeOptions()
options.binary_location = chrome_binary_path
#Service
service = ChromeService(executable_path=CHROMEDRIVER_PATH)

# 创建一个WebDriver对象，并指定驱动程序路径
driver = webdriver.Chrome(service=service, options=options)


"""
请求首页
"""
#请求
driver.get("https://www.icourse163.org/")

"""
登陆
"""
#等待登录元素加载好再执行操作
WebDriverWait(driver, 10, 0.5).until(
    EC.presence_of_element_located((By.XPATH, '//a[@class="f-f0 navLoginBtn"]'))).click()

iframe = WebDriverWait(driver, 10, 0.5).until(EC.presence_of_element_located((By.XPATH, '//*[@frameborder="0"]')))
#切换到登录界面中
driver.switch_to.frame(iframe)
# 输入账号密码
driver.find_element(By.XPATH, '//*[@id="phoneipt"]').send_keys("19906912753")
time.sleep(3)
driver.find_element(By.XPATH, '//*[@class="j-inputtext dlemail"]').send_keys("Woshi2gouzi@")
time.sleep(3)
#点击登录按钮
driver.find_element(By.ID, 'submitBtn').click()

time.sleep(5)
"""
搜索
"""
driver.switch_to.default_content()

#等待输入框并输入’python‘
wait = WebDriverWait(driver, 10)
select_course = wait.until(EC.presence_of_element_located((By.XPATH,"//input[@class='ant-input']")))

select_course.send_keys("python")
#输入回车
select_course.send_keys(Keys.ENTER)


"""
解析数据
"""

time.sleep(5)

content = driver.page_source
print(content)
driver.quit()
selector = Selector(text=content)
rows = selector.xpath("//div[@class='m-course-list']/div/div")

xuhao=0
#清空数据库
#cursor.execute("delete from course")

# 爬取网站信息
for row in rows:
    # 课程名称
    course1= row.xpath(".//span[@class=' u-course-name f-thide']//text()").extract()
    course="".join(course1)
    # 学校
    college=row.xpath(".//a[@class='t21 f-fc9']/text()").extract_first()
    # 主讲教师
    teacher=row.xpath(".//a[@class='f-fc9']//text()").extract_first()
    # 团队成员
    team1 = row.xpath(".//a[@class='f-fc9']//text()").extract()
    team=",".join(team1)
    # 参加人数
    count = row.xpath(".//span[@class='hot']/text()").extract_first()
    # 课程进度
    process = row.xpath(".//span[@class='txt']/text()").extract_first()
    # 课程简介
    brief1=row.xpath(".//span[@class='p5 brief f-ib f-f0 f-cb']//text()").extract()
    brief=",".join(brief1)

    xuhao+=1

    #sql插入数据
    sql = "INSERT INTO course VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
    cursor.execute(sql,
                   [xuhao,course,college,teacher,team,count,process,brief])

#提交
conn.commit()
print("数据已存入数据库")

运行结果

2）心得体会

学会了使用selenium的send_key填写表单实现登陆以及搜素，另外需要注意登陆时要查找Ifarme并使用selenium的driver.switch_to.frame(iframe)切换到对应的iframe
还有就是使用sleep配合wait.until(EC.presence_of_element_located())可以有效避免元素未加载的情况