爬虫之lxml - etree - xpath的使用

# 解析原理:
# - 获取页面源码数据
# - 实例化一个etree对象,并且将页面源码数据加载到该对象中
# - 调用该对象的xpath方法进行指定标签定位
# - xpath函数必须结合着xpath表达式进行标签定位和内容捕获

# xpath表达式:
# - 属性定位: //div[@class="song"] 找到class属性值为song的div 返回一个列表
# - 索引层级定位: //div[@class="tang"]/ul/li[2]/a
# - 逻辑运算: //a[@href="" and @class="du"] 并且
# - 模糊匹配: //div[contains(@class, 'ng')] class包含 ng 的div
#            //div[startwith(@class, 'ta')] class以 ta 开头的div
# - 取文本: //div[@class="song"]/p[1]/text() div下的文本内容
#          //div[@class="tang"]//text() div下以及字标签下的文本内容 返回列表
# - 取属性: // div[@class="tang"]//a[1]/@href

下面上几个小案例:

import requests
from lxml import etree

url = 'https://bj.58.com/ershoufang/?utm_source=sem-sales-baidu-pc&spm=85077276202.21974091622&utm_campaign=sell&utm_medium=cpc&showpjs=pc_fg'

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}

page_text = requests.get(url=url, headers=headers).text

tree = etree.HTML(page_text)
li_list = tree.xpath('//ul[@class="house-list-wrap"]/li') # 返回的是Element对象
fp = open('58.csv', 'w', encoding='utf8')
for li in li_list:
    title = li.xpath('./div[2]/h2/a/text()')[0] # 局部页面解析要加'.'
    price1 = li.xpath('./div[3]//text()')
    price = ''.join(price1)
    fp.write(title+":"+price+'\n')
fp.close()
print('over')

xpath 解析图片资源

import requests
from lxml import etree

url = "http://pic.netbian.com/4kmeinv/"

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
page_text = requests.get(url=url, headers=headers).text

tree = etree.HTML(page_text)
# etree.parse(page_text) 解析本地文件推荐使用
li_list = tree.xpath('//div[@class="slist"]/ul/li')
for li in li_list:
    image_name = li.xpath('./a/b/text()')[0]
    image_name = image_name.encode('iso-8859-1').decode('gbk')
    image_url = 'http://pic.netbian.com'+li.xpath('./a/img/@src')[0]
    image_path = './img/'+image_name+'.jpg'
    img = requests.get(image_url).content
    with open(image_path, 'wb') as f:
        f.write(img)
    print(image_path+'下载成功')

import requests
import base64
from lxml import etree

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
url = 'http://jandan.net/top'
response = requests.get(url=url, headers=headers)
page_text = response.text
tree = etree.HTML(page_text)
code_list = tree.xpath('//span[@class="img-hash"]/text()')
for img_code in code_list:
    img_url = 'http:'+base64.b64decode(img_code).decode()
    img_name = img_url.split('/')[-1]
    img_path = f'./jd_img/{img_name}'
    print(img_url)
    content = requests.get(img_url).content
    with open(img_path, 'wb') as f:
        f.write(content)
    print(img_name+'成功')
print('over')

上面是煎蛋网采用了js的方法对图片链接地址进行了base64的加密

# 简历模板爬取(ip禁用问题)
# 解决方法:
#       ip代理,
#       请求头中添加Connection字段:close
import requests
import random
from lxml import etree

url = 'http://sc.chinaz.com/jianli/free.html'

headers = {
          'Connection': 'close', # 每次请求成功之后,发马上断开连接(修改后有几率无法立即生效,出现Httppool...错误- 重新运行)
          'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
page_text = requests.get(url=url, headers=headers).text
tree = etree.HTML(page_text)
a_list = tree.xpath('//div[@id="container"]/div/a[1]')
for a in a_list:
    title = a.xpath('./img/@alt')[0].encode('iso-8859-1').decode('utf-8')
    detail_url = a.xpath('./@href')[0]
    detail_text = requests.get(url=detail_url, headers=headers).text
    d_tree = etree.HTML(detail_text)
    down_url_list = d_tree.xpath('//div[@class="down_wrap"]//li/a/@href')
    down_url = random.choice(down_url_list)
    data = requests.get(down_url,headers=headers).content
    with open(f'./简历模板/{title}.rar', 'wb') as f:
        f.write(data)
    print(title+'完成')
print('over')