爬取博客文章

1、去除置顶的旧文章内容：

import requests
from lxml import etree

url = "https://www.cnblogs.com/chingho/"

r = requests.get(url)
selector = etree.HTML(r.text)
divs = selector.xpath('//*[@role="article"]')

articles = []
for d in divs:
    day = d.xpath('./*[@class="dayTitle"]/a')[0]
    title = d.xpath('normalize-space(./*[@class="postTitle"]/a/span)')
    if '置顶' not in title:
        href = d.xpath('./*[@class="postTitle"]/a/@href')[0]
        article = day.text.strip() + ' ' + title.strip() + ' ' + href
        print(article)
        articles.append(article)

输出：

 

2、若要置顶的文章：

html_str = '''
<div class="postTitle" role="heading" aria-level="2">
    <a class="postTitle2 vertical-middle pinned-post" href="https://www.cnblogs.com/chingho/archive/2010/09/05/1818373.html">
        <span>
            <span class="pinned-post-mark">[置顶]</span>
        如何管理庞大的Ajax请求？
        </span>
    </a>
</div>
'''

html = etree.HTML(html_str)
# 置顶
result = html.xpath('//div[@class="postTitle"]/a/span/span/text()')
text_0 = result[0]
# 标题
result = html.xpath('//div[@class="postTitle"]/a/span/span/following-sibling::text()')  # 获取当前节点之后的所有同级节点
text = result[0].strip()

print(text_0 + text)

输出：