Beautifulsoup的使用
from bs4 import BeautifulSoup # html_doc = """ # <html><head><title>The Dormouse's story</title></head> # <body> # # # <p class="story">Once upon a time there were three little sisters; and their names were # <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>, # <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and # <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>; # and they lived at the bottom of a well.</p> # # <p class="story">...</p> # """ # soup=BeautifulSoup(html_doc,'lxml') # print(soup) #会将不完整的html文本补充完成 # print(soup.prettify()) #将html文本进行结构划分 # print(soup.p.b) #查找文本的第一个标签 <p class="title"><b>The Dormouse's story</b></p> #查找子孙节点 # print(list(soup.p.descendants)) # print(soup.p.a.text) #显示标签内的文本 # print(soup.p.children) #<list_iterator object at 0x00000181ABD6A978> ,迭代器使用list()转为列表形式 # print(list(soup.p.children)) #['Once upon a time there were three little sisters; and their names were\n', <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>, ',\n', <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>, ' and\n', <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>, ';\nand they lived at the bottom of a well.'] # print(soup.p.contents) # print(soup.a.parent) #查找标签a的父标签，包括子标签和内容 # print(soup.a.parents) #<generator object parents at 0x000001D5FC242A40> # print(list(soup.a.parents)) #结果类型为列表，查找所有父标签包括父标签的内容，直到document
find和findall
html_doc = """ <html><head><title>The Dormouse's story</title></head> <body> <p class="title"><b>The Dormouse's story</b></p> <p class="story">Once upon a time there were three little sisters; and their names were <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>, <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>; and they lived at the bottom of a well.</p> <p class="story">...</p> """ soup=BeautifulSoup(html_doc,'lxml') # print(soup.find_all('a')) #[<a class="sister" hCSSref="http://example.com/elsie" id="link1">Elsie</a>, <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>, <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>] # print(soup.find_all('a',id='link2')) #[<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>] # print(soup.find('a',id='link2')) #<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a> # print(soup.find(id='link2').attrs['href']) #http://example.C/lacie # print(soup.find_all(attrs={'class':'sister'})) #[<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>, <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>, <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>] # print(soup.find('p').find('b')) #<b>The Dormouse's story</b>
CSS选择器
from bs4 import BeautifulSoup html_doc = """ <html><head><title>The Dormouse's story</title></head> <body> <p class="title"><b>The Dormouse's story</b></p> <p class="story">Once upon a time there were three little sisters; and their names were <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>, <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>; and they lived at the bottom of a well.</p> <p class="story">...</p> """ soup=BeautifulSoup(html_doc,'lxml') # print(soup.select('p b')) #[<b>The Dormouse's story</b>] 查找p标签下的b标签 # print(soup.select('p')[1].select('#link2')[0].attrs['href']) #http://example.com/lacie 获取标签的属性 # print(soup.select('p')[1].get_text()) #获取标签的文本内容
