爬虫--BeautifulSoup

什么是BeautifulSoup？

 

BeautifulSoup支持的一些解析库

 

基本使用

from bs4 import BeautifulSoup

html ="""
    <html><head><title> The Dormouse's story</title></head>
    <body>
    <p class="title" name="dromouse"> <b> The Dormouse's story</b></p>
    <p class="story">Once upon a time there were three little sisters;and their names were
    <a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
    <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
    <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
    and they lived at the bottom of a well.</p>
    <p class="story">..</p>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.prettify())  # .prettify() 格式化代码
print(soup.title.string) # .title.string
<html>
 <head>
  <title>
   The Dormouse's story
  </title>
 </head>
 <body>
  <p class="title" name="dromouse">
   <b>
    The Dormouse's story
   </b>
  </p>
  <p class="story">
   Once upon a time there were three little sisters;and their names were
   <a class="sister" href="http://example.com/elsie" id="link1">
    <!--Elsie-->
   </a>
   ,
   <a class="sister" href="http://example.com/lacie" id="link2">
    Lacie
   </a>
   and
   <a class="sister" href="http://example.com/tillie" id="link3">
    Tillie
   </a>
   ;
    and they lived at the bottom of a well.
  </p>
  <p class="story">
   ..
  </p>
 </body>
</html>
 The Dormouse's story
打印后的结果为：

标签选择器

选择元素

from bs4 import BeautifulSoup

html ="""
    <html><head><title> The Dormouse's story</title></head>
    <body>
    <p class="title" name="dromouse"> <b> The Dormouse's story</b></p>
    <p class="story">Once upon a time there were three little sisters;and their names were
    <a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
    <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
    <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
    and they lived at the bottom of a well.</p>
    <p class="story">..</p>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.title)
print(type(soup.title))
print(soup.head)
print(soup.p)
<title> The Dormouse's story</title>
<class 'bs4.element.Tag'>
<head><title> The Dormouse's story</title></head>
<p class="title" name="dromouse"> <b> The Dormouse's story</b></p>
打印的结果为：

获取名称

from bs4 import BeautifulSoup

html ="""
    <html><head><title> The Dormouse's story</title></head>
    <body>
    <p class="title" name="dromouse"> <b> The Dormouse's story</b></p>
    <p class="story">Once upon a time there were three little sisters;and their names were
    <a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
    <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
    <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
    and they lived at the bottom of a well.</p>
    <p class="story">..</p>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.title.name)
title
打印的结果为：

获取属性

from bs4 import BeautifulSoup

html ="""
    <html><head><title> The Dormouse's story</title></head>
    <body>
    <p class="title" name="dromouse"> <b> The Dormouse's story</b></p>
    <p class="story">Once upon a time there were three little sisters;and their names were
    <a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
    <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
    <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
    and they lived at the bottom of a well.</p>
    <p class="story">..</p>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.p.attrs["name"])
print(soup.p["name"])
dromouse
dromouse
打印的结果为:

获取内容

from bs4 import BeautifulSoup

html ="""
    <html><head><title> The Dormouse's story</title></head>
    <body>
    <p class="title" name="dromouse"> <b> The Dormouse's story</b></p>
    <p class="story">Once upon a time there were three little sisters;and their names were
    <a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
    <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
    <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
    and they lived at the bottom of a well.</p>
    <p class="story">..</p>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.title.string)
 The Dormouse's story
打印后的结果为：

嵌套选择

from bs4 import BeautifulSoup

html ="""
    <html><head><title> The Dormouse's story</title></head>
    <body>
    <p class="title" name="dromouse"> <b> The Dormouse's story</b></p>
    <p class="story">Once upon a time there were three little sisters;and their names were
    <a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
    <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
    <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
    and they lived at the bottom of a well.</p>
    <p class="story">..</p>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.head.title.string)
 The Dormouse's story
打印后的结果为：

子节点和子孙节点

from bs4 import BeautifulSoup

html ="""
    <html>
        <head>
            <title> The Dormouse's story</title>
        </head>
    <body>
    <p class="story">
        Once upon a time there were three little sisters;and their names were
        <a href="http://example.com/elsie" class="sister" id="link1"> 
            <!--Elsie-->
        </a>,
        <a href="http://example.com/lacie" class="sister" id="link2">
            Lacie
        </a>
        and
        <a href="http://example.com/tillie" class="sister" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>
    <p class="story">
        ..
    </p>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.p.contents)
['\n        Once upon a time there were three little sisters;and their names were\n        ', <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>, ',\n        ', <a class="sister" href="http://example.com/lacie" id="link2">
            Lacie
        </a>, '\n        and\n        ', <a class="sister" href="http://example.com/tillie" id="link3">
            Tillie
        </a>, ';\n        and \n        they lived at the bottom of a well.\n    ']
打印后的结果为：
from bs4 import BeautifulSoup

html ="""
    <html>
        <head>
            <title> The Dormouse's story</title>
        </head>
    <body>
    <p class="story">
        Once upon a time there were three little sisters;and their names were
        <a href="http://example.com/elsie" class="sister" id="link1"> 
            <!--Elsie-->
        </a>,
        <a href="http://example.com/lacie" class="sister" id="link2">
            Lacie
        </a>
        and
        <a href="http://example.com/tillie" class="sister" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>
    <p class="story">
        ..
    </p>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.p.children) # .children就相当于迭代器，需要循环的方式才能把内容取走
for i,child in enumerate(soup.p.children):
    print(i,child)
<list_iterator object at 0x00000000012C5E10>
0 
        Once upon a time there were three little sisters;and their names were
        
1 <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>
2 ,
        
3 <a class="sister" href="http://example.com/lacie" id="link2">
            Lacie
        </a>
4 
        and
        
5 <a class="sister" href="http://example.com/tillie" id="link3">
            Tillie
        </a>
6 ;
        and 
        they lived at the bottom of a well.
打印后的结果为：
from bs4 import BeautifulSoup

html ="""
    <html>
        <head>
            <title> The Dormouse's story</title>
        </head>
    <body>
    <p class="story">
        Once upon a time there were three little sisters;and their names were
        <a href="http://example.com/elsie" class="sister" id="link1"> 
            <!--Elsie-->
        </a>,
        <a href="http://example.com/lacie" class="sister" id="link2">
            Lacie
        </a>
        and
        <a href="http://example.com/tillie" class="sister" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>
    <p class="story">
        ..
    </p>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.p.descendants) # .descendants就相当于迭代器，获取所有的子孙节点，需要循环的方式才能把内容取走
for i,child in enumerate(soup.p.descendants):
    print(i,child)
Once upon a time there were three little sisters;and their names were
        
1 <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>
2 
3 Elsie
4 
5 ,
6 <a class="sister" href="http://example.com/lacie" id="link2">
            Lacie
        </a>
7 
            Lacie 
8 
        and
9 <a class="sister" href="http://example.com/tillie" id="link3">
            Tillie
        </a>
10 
            Tillie
11 ;
        and 
        they lived at the bottom of a well.
打印后的结果为：

父节点和祖先节点

from bs4 import BeautifulSoup

html ="""
    <html>
        <head>
            <title> The Dormouse's story</title>
        </head>
    <body>
    <p class="story">
        Once upon a time there were three little sisters;and their names were
        <a href="http://example.com/elsie" class="sister" id="link1"> 
            <!--Elsie-->
        </a>,
        <a href="http://example.com/lacie" class="sister" id="link2">
            Lacie
        </a>
        and
        <a href="http://example.com/tillie" class="sister" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>
    <p class="story">
        ..
    </p>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.a.parent) # .descendants就相当于迭代器，获取所有的父节点
<p class="story">
        Once upon a time there were three little sisters;and their names were
        <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>,
        <a class="sister" href="http://example.com/lacie" id="link2">
            Lacie
        </a>
        and
        <a class="sister" href="http://example.com/tillie" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>
打印后的结果为：
from bs4 import BeautifulSoup

html ="""
    <html>
        <head>
            <title> The Dormouse's story</title>
        </head>
    <body>
    <p class="story">
        Once upon a time there were three little sisters;and their names were
        <a href="http://example.com/elsie" class="sister" id="link1"> 
            <!--Elsie-->
        </a>,
        <a href="http://example.com/lacie" class="sister" id="link2">
            Lacie
        </a>
        and
        <a href="http://example.com/tillie" class="sister" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>
    <p class="story">
        ..
    </p>
"""

soup=BeautifulSoup(html,"lxml")
print(list(enumerate(soup.a.parents))) # .descendants就相当于迭代器，获取所有的祖先节点)
[(0, <p class="story">
        Once upon a time there were three little sisters;and their names were
        <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>,
        <a class="sister" href="http://example.com/lacie" id="link2">
            Lacie
        </a>
        and
        <a class="sister" href="http://example.com/tillie" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>), (1, <body>
<p class="story">
        Once upon a time there were three little sisters;and their names were
        <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>,
        <a class="sister" href="http://example.com/lacie" id="link2">
            Lacie
        </a>
        and
        <a class="sister" href="http://example.com/tillie" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>
<p class="story">
        ..
    </p>
</body>), (2, <html>
<head>
<title> The Dormouse's story</title>
</head>
<body>
<p class="story">
        Once upon a time there were three little sisters;and their names were
        <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>,
        <a class="sister" href="http://example.com/lacie" id="link2">
            Lacie
        </a>
        and
        <a class="sister" href="http://example.com/tillie" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>
<p class="story">
        ..
    </p>
</body></html>), (3, <html>
<head>
<title> The Dormouse's story</title>
</head>
<body>
<p class="story">
        Once upon a time there were three little sisters;and their names were
        <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>,
        <a class="sister" href="http://example.com/lacie" id="link2">
            Lacie
        </a>
        and
        <a class="sister" href="http://example.com/tillie" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>
<p class="story">
        ..
    </p>
</body></html>)]
打印后的结果为：

兄弟节点

from bs4 import BeautifulSoup

html ="""
    <html>
        <head>
            <title> The Dormouse's story</title>
        </head>
    <body>
    <p class="story">
        Once upon a time there were three little sisters;and their names were
        <a href="http://example.com/elsie" class="sister" id="link1"> 
            <!--Elsie-->
        </a>,
        <a href="http://example.com/lacie" class="sister" id="link2">
            Lacie
        </a>
        and
        <a href="http://example.com/tillie" class="sister" id="link3">
            Tillie
        </a>;
        and 
        they lived at the bottom of a well.
    </p>
    <p class="story">
        ..
    </p>
"""

soup=BeautifulSoup(html,"lxml")
print(list(enumerate(soup.a.next_siblings))) # .descendants就相当于迭代器，获取兄弟节点
print(list(enumerate(soup.a.previous_siblings))) # .descendants就相当于迭代器，获取兄弟节点
[(0, ',\n        '), (1, <a class="sister" href="http://example.com/lacie" id="link2">
            Lacie
        </a>), (2, '\n        and\n        '), (3, <a class="sister" href="http://example.com/tillie" id="link3">
            Tillie
        </a>), (4, ';\n        and \n        they lived at the bottom of a well.\n    ')]
[(0, '\n        Once upon a time there were three little sisters;and their names were\n        ')]
打印后的结果为：

标准选择器

find_all(name,attrs,recursive,text,**kwargs)

可根据标签名、属性、内容查找文档

from bs4 import BeautifulSoup

html="""
    <div class="panel">
        <div class="panel-heading">
            <h4>Hello</h4>
        </div>
        <div class="panel-body">
            <ul class="list" id="list-1>
                <li class="element">Foo</li>
                <li class="element">Bar</li>
                <li class="element">Jay</ll>
            </ul>
            <ul class="list list-small" id="list-2">
                <li class="element">Foo</ll>
                <ll class="element">Bar</l>
            </ul>
        </div>
    </div>
"""

soup=BeautifulSoup(html,"lxml")
print(soup.find_all("ul"))
print(type(soup.find_all("ul")[0]))
[<ul class="list" element="" id="list-1&gt;&lt;li class=">Foo
<li class="element">Bar</li>
<li class="element">Jay
</li></ul>, 
<ul class="list list-small" id="list-2">
<li class="element">Foo
<ll class="element">Bar
</ll></li></ul>]
<class 'bs4.element.Tag'>
打印后的结果为：
from bs4 import BeautifulSoup

html="""
    <div class="panel">
        <div class="panel-heading">
            <h4>Hello</h4>
        </div>
        <div class="panel-body">
            <ul class="list" id="list-1>
                <li class="element">Foo</li>
                <li class="element">Bar</li>
                <li class="element">Jay</ll>
            </ul>
            <ul class="list list-small" id="list-2">
                <li class="element">Foo</ll>
                <ll class="element">Bar</l>
            </ul>
        </div>
    </div>
"""

soup=BeautifulSoup(html,"lxml")
for UL in soup.find_all("ul"):
    print(UL.find_all("li"))
[<li class="element">Bar</li>, <li class="element">Jay</li>]
[<li class="element">Foo<ll class="element">Bar</ll></li>]
打印后的结果为：

attrs

from bs4 import BeautifulSoup

html="""
    <div class="panel">
        <div class="panel-heading">
            <h4>Hello</h4>
        </div>
        <div class="panel-body">
            <ul class="list" id="list-1 name="elements">
                <li class="element">Foo</li>
                <li class="element">Bar</li>
                <li class="element">Jay</ll>
            </ul>
            <ul class="list list-small" id="list-2">
                <li class="element">Foo</ll>
                <ll class="element">Bar</l>
            </ul>
        </div>
    </div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.find_all(attrs={"id":"list-1"}))
print(soup.find_all(attrs={"name":"elements"}))
[]
[]
打印后的结果为：
from bs4 import BeautifulSoup

html="""
    <div class="panel">
        <div class="panel-heading">
            <h4>Hello</h4>
        </div>
        <div class="panel-body">
            <ul class="list" id="list-1 name="elements">
                <li class="element">Foo</li>
                <li class="element">Bar</li>
                <li class="element">Jay</ll>
            </ul>
            <ul class="list list-small" id="list-2">
                <li class="element">Foo</ll>
                <ll class="element">Bar</l>
            </ul>
        </div>
    </div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.find_all(id="list-1"))
print(soup.find_all(class_="element"))
[<ul class="list list-small" id="list-2"><li class="element">Foo<ll class="element">Bar</ll></li></ul>]
[<li class="element">Foo</li>, <li class="element">Bar</li>, <li class="element">Jay</li>, <li class="element">Foo<llclass="element">Bar</ll></li>, <ll class="element">Bar</ll>]
打印后的结果为：

text

from bs4 import BeautifulSoup

html="""
    <div class="panel">
        <div class="panel-heading">
            <h4>Hello</h4>
        </div>
        <div class="panel-body">
            <ul class="list" id="list-1 name="elements">
                <li class="element">Foo</li>
                <li class="element">Bar</li>
                <li class="element">Jay</ll>
            </ul>
            <ul class="list list-small" id="list-2">
                <li class="element">Foo</ll>
                <ll class="element">Bar</l>
            </ul>
        </div>
    </div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.find_all(text="Foo"))
['Foo']
打印后的结果为：

find(name,attrs,recursive,text,**kwargs)

find返回单个元素，find_all返回所有元素

from bs4 import BeautifulSoup

html="""
    <div class="panel">
        <div class="panel-heading">
            <h4>Hello</h4>
        </div>
        <div class="panel-body">
            <ul class="list" id="list-1 name="elements">
                <li class="element">Foo</li>
                <li class="element">Bar</li>
                <li class="element">Jay</ll>
            </ul>
            <ul class="list list-small" id="list-2">
                <li class="element">Foo</ll>
                <ll class="element">Bar</l>
            </ul>
        </div>
    </div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.find("ul"))
print(type(soup.find("ul")))
print(soup.find("h4"))
<ul class="list" elements="" id="list-1 name=">
     <li class="element">Foo</li>
     <li class="element">Bar</li>
     <li class="element">Jay</li>
</ul>
<class 'bs4.element.Tag'>
<h4>Hello</h4>
打印后的结果为：

find_parents() , find_parent()
find_parents()返回所有祖先节点，find_parent()返回直接父节点。
find_next_siblings() , find_next_sibling()
find_next_siblings()返回后面所有兄弟节点，findnext_sibling0返回后面第一个兄弟节点。
find_previous_ siblings() , find_previous_sibling()
find_previous_siblings0返回前面所有兄弟节点，find_previous_sibling0返回前面第一个兄弟节点。
find_all_next() , find_next()
find_all_next()返回节点后所有符合条件的节点，find_next()返回第一个符合条件的节点
find_all_previous()和find_previous()
find_all_previous()返回节点后所有符合条件的节点，find_previous()返回第一个符合条件的节点

CSS选择器

通过select()直接传入CSS选择器即可完成选择

from bs4 import BeautifulSoup

html="""
    <div class="panel">
        <div class="panel-heading">
            <h4>Hello</h4>
        </div>
        <div class="panel-body">
            <ul class="list" id="list-1 name="elements">
                <li class="element">Foo</li>
                <li class="element">Bar</li>
                <li class="element">Jay</ll>
            </ul>
            <ul class="list list-small" id="list-2">
                <li class="element">Foo</ll>
                <ll class="element">Bar</l>
            </ul>
        </div>
    </div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.select(".panel .panel-heading"))
print(soup.select("ul li"))
print(soup.select("#list-2 .element"))
print(type(soup.select("ul")[0]))
[<div class="panel-heading"><h4>Hello</h4></div>]
[<li class="element">Foo</li>, <li class="element">Bar</li>, <li class="element">Jay</li>, <li class="element">Foo<ll class="element">Bar</ll></li>]
[<li class="element">Foo<li class="element">Bar</li></li>, <li class="element">Bar</li>]
<class 'bs4.element.Tag'>
打印后的结果为：
from bs4 import BeautifulSoup

html="""
    <div class="panel">
        <div class="panel-heading">
            <h4>Hello</h4>
        </div>
        <div class="panel-body">
            <ul class="list" id="list-1 name="elements">
                <li class="element">Foo</li>
                <li class="element">Bar</li>
                <li class="element">Jay</ll>
            </ul>
            <ul class="list list-small" id="list-2">
                <li class="element">Foo</ll>
                <ll class="element">Bar</l>
            </ul>
        </div>
    </div>
"""
soup=BeautifulSoup(html,"lxml")
for ul in soup.select("ul"):
    print(ul.select("li"))
[<li class="element">Foo</li>, <li class="element">Bar</li>, <li class="element">Jay </li>]
[<li class="element">Foo<ll class="element">Bar</ll></li>]
打印后的结果为：

获取属性

from bs4 import BeautifulSoup

html="""
    <div class="panel">
        <div class="panel-heading">
            <h4>Hello</h4>
        </div>
        <div class="panel-body">
            <ul class="list" id="list-1" name="elements">
                <li class="element">Foo</li>
                <li class="element">Bar</li>
                <li class="element">Jay</ll>
            </ul>
            <ul class="list list-small" id="list-2">
                <li class="element">Foo</li>
                <li class="element">Bar</li>
            </ul>
        </div>
    </div>
"""
soup=BeautifulSoup(html,"lxml")

for ul in soup.select("ul"):
    print(ul["id"])
    print(ul.attrs["id"])
list-1
list-1
list-2
list-2
打印的结果为：

获取内容

from bs4 import BeautifulSoup

html="""
    <div class="panel">
        <div class="panel-heading">
            <h4>Hello</h4>
        </div>
        <div class="panel-body">
            <ul class="list" id="list-1" name="elements">
                <li class="element">Foo</li>
                <li class="element">Bar</li>
                <li class="element">Jay</ll>
            </ul>
            <ul class="list list-small" id="list-2">
                <li class="element">Foo</li>
                <li class="element">Bar</li>
            </ul>
        </div>
    </div>
"""
soup=BeautifulSoup(html,"lxml")
for ul in soup.select("li"):
    print(ul.get_text())
Foo
Bar
Jay
            
Foo
Bar
打印后的结果为：

总结：
       ●推荐使用lxml解析库，必要时使用html.parser
       ●标签选择筛选功能弱但是速度快
       ●建议使用find()、find_all()查询匹配单个结果或者多个结果
       ●如果对CSS选择器熟悉建议使用select()
       ●记住常用的获取属性和文本值的方法

 

posted @ 2018-09-21 13:43  追风的小蚂蚁  阅读(281)  评论(0)  编辑  收藏  举报