爬虫Xpath使用

Xpath使用案例

准备工作

pip install lxml

代码如下:

点击查看代码


from lxml import etree
#定义字符串
wb_data = """
        <div>
            <ul>
                 <li class="item-0"><a href="link1.html">first item</a></li>
                 <li class="item-1"><a href="link2.html">second item</a></li>
                 <li class="item-inactive"><a href="link3.html">third item</a></li>
                 <li class="item-1"><a href="link4.html">fourth item</a></li>
                 <li class="item-0"><a href="link5.html">fifth item</a>
             </ul>
         </div>
        """
#讲变量读取为html对象，自动添加<html><body>... ...</body></html>
html=etree.HTML(wb_data)
#解析数据,/是跟节点
data1=html.xpath('/html')
for i in data1:
    print(i)
#//后面跟子孙节点，后面的节点可以在任意位置
data2=html.xpath('//li')
for i in data2:
    print(i)
#写具体路径
data3=html.xpath('/html/body/div/ul/li/a')
for i in data3:
    print(i.text)
#循环某个属性
data4=html.xpath('/html/body/div/ul/li/@class')
for i in data4:
    print(i)
#.代表当前节点，..代表父节点
data5=html.xpath('/html/body/div/ul/li/a')
for i in data5:
    a=i.xpath('./@href')
    print(a)

#读取某一个节点
data6=html.xpath('/html/body/div/ul')
for i in data6:
    li_1=i.xpath('./li[1]')
    print(li_1)
    li_last=i.xpath('./li[last()]')
    print(li_last)
#带有某属性的标签
data7=html.xpath('//ul/li/a[@href="link2.html"]')
for i in data7:
    print(i.text)

#*位置节点
data8=html.xpath('//ul/li/*')
for i in data8:
    if hasattr(i,'text'):
        print(i.text)
    else:
        print(i)