DAY-03

上午

selenium元素交互操作

——点击、清除

 1 from selenium import webdriver
 2 from selenium.webdriver import ActionChains  # 破解滑动验证码的时候用的 可以拖动图片
 3 from selenium.webdriver.common.keys import Keys  # 键盘按键操作
 4 import time
 5 driver = webdriver.Chrome()
 6 try:
 7     driver.implicitly_wait(10)
 8     driver.get('https://www.jd.com/')
 9 
10     #点击、清除
11     input = driver.find_element_by_id('key')
12     input.send_keys('人间失格')
13 
14     #通过class查找搜索
15     search = driver.find_element_by_class_name('button')
16     search.click(); #点击搜索按钮
17     time.sleep(3)
18 
19     input2 = driver.find_element_by_id('key')
20     input2.clear() #清空输入框
21     time.sleep(2)
22 
23     input2.send_keys('三体')
24     input2.send_keys(Keys.ENTER)
25     time.sleep(5)
26 
27 finally:
28     driver.close()

——Actions Chains

动作链对象，需要把driver驱动传给它。

动作链对象可以操作一系列设定好的动作行为

——frame的切换

1.方式一：起始目标向重点目标 瞬间移动

 1 from selenium import webdriver
 2 from selenium.webdriver import ActionChains  # 破解滑动验证码的时候用的 可以拖动图片
 3 from selenium.webdriver.common.keys import Keys  # 键盘按键操作
 4 import time
 5 driver = webdriver.Chrome()
 6 
 7 try:
 8     driver.implicitly_wait(5)
 9     driver.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
10     time.sleep(5)
11 
12     #frame切换
13     driver.switch_to.frame('ifframeResault')
14     time.sleep(1)
15 
16     #起始方块id：draggable
17     action = ActionChains(driver)
18 
19     #目标方块id：droppable
20     source = driver.find_element_by_id('draggable')
21 
22     target = driver.find_element_by_id('droppable')
23 
24     # 起始目标向终点目标 瞬间移动
25     action.drag_and_drop(source,target).perform()
26     time.sleep(10)
27 
28 finally:
29     driver.close()

2.方式二：缓慢移动

 1 from selenium import webdriver
 2 from selenium.webdriver import ActionChains  # 破解滑动验证码的时候用的 可以拖动图片
 3 from selenium.webdriver.common.keys import Keys  # 键盘按键操作
 4 import time
 5 
 6 driver = webdriver.Chrome()
 7 
 8 try:
 9     driver.implicitly_wait(10)
10     driver.get('https://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
11     time.sleep(5)
12 
13     # frame切换
14     driver.switch_to.frame('iframeResult')
15     time.sleep(1)
16 
17     # 起始方块id：draggable
18     source = driver.find_element_by_id('draggable')
19 
20     # 目标方块id：droppable
21     target = driver.find_element_by_id('droppable')
22     print(source.size) #大小
23     print(source.tag_name) #标签
24     print(source.text) #文本
25     print(source.location) #坐标：x，y
26 
27     # 缓慢移动
28     # 找到移动距离
29     distance = target.location['x'] - source.location['x']
30 
31     # 按住起始目标
32     ActionChains(driver).click_and_hold(source).perform()
33 
34     # 移动
35     s = 0
36     while s < distance:
37         ActionChains(driver).move_by_offset(xoffset=2, yoffset=0).perform()
38         s += 2
39         time.sleep(0.1)
40 
41     # 松开目标
42     ActionChains(driver).release().perform()
43     time.sleep(10)
44 
45 finally:
46     driver.close()

——执行js代码

 

 1 from selenium import webdriver
 2 import time
 3 
 4 driver = webdriver.Chrome()
 5 
 6 try:
 7     driver.implicitly_wait(10)
 8     driver.get('https://www.baidu.com/')
 9     driver.execute_script(
10         '''
11         alert("噜噜噜")
12         ''')
13     time.sleep(5)
14 
15 
16 finally:
17     driver.close()

 

前进后退

 

 

爬取京东商品信息

 1 from selenium import webdriver
 2 from selenium.webdriver.common.keys import Keys  # 键盘按键操作
 3 import time
 4 
 5 #
 6 def get_good(driver):
 7     num = 1
 8     try:
 9         time.sleep(5)
10 
11         #下拉滑动5000px
12         js_code = '''
13                    window.scrollTo(0,5000)
14                    '''
15         driver.execute_script(js_code)
16 
17         time.sleep(5) # 商品信息加载，等待5s
18         good_list = driver.find_elements_by_class_name('gl-item')
19         for good in good_list:
20             # 商品名称
21             good_name = good.find_element_by_css_selector('.p-name em').text
22             # 商品链接
23             good_url = good.find_element_by_css_selector('.p-name a').get_attribute('href')
24             # 商品价格
25             good_price = good.find_element_by_class_name('p-price').text
26             # 商品评价
27             good_commit = good.find_element_by_class_name('p-commit').text
28 
29             good_content = f'''
30                        num:{num}
31                        商品名称：{good_name}
32                        商品链接：{good_url}
33                        商品价格：{good_price}
34                        商品评论：{good_commit}
35                        \n
36                        '''
37             print(good_content)
38             # 保存数据写入文件
39             with open('京东商品信息爬取.txt', 'a', encoding='utf-8') as f:
40                 f.write(good_content)
41             num += 1
42 
43         # 找到页面下一页点击
44         next_tag = driver.find_element_by_class_name('pn-next')
45         next_tag.click()
46 
47         time.sleep(5)
48         #递归调用函数本身
49         get_good(driver)
50 
51     finally:
52         driver.close()
53 
54 if __name__ == '__main__':
55     driver = webdriver.Chrome()
56     try:
57         driver.implicitly_wait(10)
58         driver.get('https://www.jd.com/')
59 
60         input = driver.find_element_by_id('key')
61         input.send_keys('人间失格')
62         input.send_keys(Keys.ENTER)
63         get_good(driver)
64         print('商品信息写入完成')
65     finally:
66         driver.close()

 下午

BeautifulSoup4(BS4)

1.什么是bs4？

bs4是一个解析库，可以通过某种解析器来帮我们提取想要的数据。

2.为什么要使用bs4？

它可以通过简洁的语法快速提取用户想要的数据

3.解析器的分类

   首选：lxml XML

　次选：Python标准库

4.bs4的安装与使用

　　安装解析器：pip install lxml

　　安装解析库：pip install bs4

html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="sister"><b>$37</b></p>

<p class="story" id="p">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" >Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>

<p class="story">...</p>
"""
from bs4 import BeautifulSoup

# python自带的解析库
# soup = BeautifulSoup(html_doc,'html.parser')

#调用bs4得到一个对象
soup = BeautifulSoup(html_doc,'lxml')

#bs4对象
print(soup)

#bs4类型
print(type(soup))

#美化功能
html = soup.prettify()

bs4解析库--搜索文档树

 

''''''
'''
find：找第一个
find_all：找所有
标签查找与属性查找:

    name属性匹配：
            name 标签名
            attrs 属性查找匹配
            text 文本匹配
    标签:

        - 正则过滤器
            re模块匹配

        - 列表过滤器
            列表内的数据匹配

        - bool过滤器
            True匹配

        - 方法过滤器
            用于一些要的属性以及不需要的属性查找。
    属性:
        - class_
        - id
'''

html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="sister"><b>$37</b></p>

<p class="story" id="p">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" >Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>

<p class="story">...</p>
"""
from bs4 import BeautifulSoup

soup = BeautifulSoup(html_doc,'lxml')
# # name 标签名
# div = soup.find(name='div')
# div_s = soup.find_all(name='div')
# # attrs 属性查找匹配
# p = soup.find(attrs={"id":"p"})
# p = soup.find_all(attrs="id","p")
# # text 文本匹配
# # name+attrs
# p = soup.find(name='div',attrs={"id":"p"})
# # name+text
# p = soup.find(name='div',text='j')

# - 正则过滤器
import re
# re模块匹配
#根据re模块匹配所有带 a 的节点
a = soup.find(name=re.compile('a'))
print(a)

# - 列表过滤器
# 列表内的数据匹配
print(soup.find(name=['a','p',re.compile('a')]))
print(soup.find_all(name=['a','p',re.compile('a')]))

# - bool过滤器
# True匹配
print(soup.find(name=True,attrs={"id":True}))
print(soup.find_all(name=True,attrs={"id":True}))

# - 方法过滤器
# 用于一些要的属性以及不需要的属性查找。
# print(soup.find_all(name='函数对象'))

def have_id_not_class(tag):
    print(tag.name)
    if tag.name == 'p' and tag.has_attr("id") and not tag.has_attr("class"):
        return tag
print(soup.find_all(name=have_id_not_class))

#补充
a = soup.find(id='link2')
print(a)
p = soup.find(class_='sister')
print(p)

 

bs4解析库--遍历文档树

html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="sister"><b>$37</b></p>

<p class="story" id="p">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" >Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>

<p class="story">...</p>
"""
from bs4 import BeautifulSoup

soup = BeautifulSoup(html_doc,'lxml')

# 1、直接使用 √
# print(soup.html)

#寻找第一个a标签
# print(soup.a)

#寻找第一个p标签
# print(soup.p)

# 2、获取标签的名称
# print(soup.a.name)

# 3、获取标签的属性 √
# print(soup.a.attrs) #获取a标签中所有的属性
# print(soup.a.attrs['href']) #获取a标签里href属性

# 4、获取标签的内容 √
# print(soup.p.txt)

# 5、嵌套选择
# print(soup.html.body.p)

# 6、子节点、子孙节点
# print(soup.p.children) #返回迭代器对象
# print(list(soup.p.children))

# 7、父节点、祖先节点
# print(soup.b.parent)  #父节点
# print(soup.b.parents) #祖先节点 ：<generator object parents at 0x000002999E869308>

# 8、兄弟节点
# print(soup.a)
# # 获取下一个兄弟节点
# print(soup.a.next_sibling)
# # 获取下一个的所有兄弟节点,返回的是一个生成器
# print(soup.a.next_siblings)
# print(list(soup.a.next_siblings))
# # 获取上一个兄弟节点
# print(soup.a.previous_sibling)
# # 获取上一个的所有兄弟节点，返回的是一个生成器
# print(list(soup.a.previous_siblings))

补充知识点：　　

yeild值（把值放进生成器中）
def f():
# return 1
yield 1
yield 2
yield 3
g = f()
print(g)

for line in g:
print(line)

 

 

posted @ 2019-07-03 11:27  吼嘿嘿吼  阅读(122)  评论(0)  编辑  收藏  举报