【python爬虫】1.我爬我自己

初步学习尝试爬了一下自己的博客

不得不感叹python的各种库和函数是真多 代码量是真少

import time
import requests
import re
from bs4 import BeautifulSoup

session = requests.Session()

i1 = session.get(
    url='https://www.cnblogs.com/gaters/',
    headers={
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36',
    }
)
soup = BeautifulSoup(i1.text, 'lxml')
# print(soup.title)
# print(type(soup.title))
# print(soup.title.string)
# print(soup.head)
for i in soup.find_all(class_='postTitle2 vertical-middle'):
    i_name=i.find('span').get_text()
    i_url=i.get('href')
    print(i_name+i_url)
    i2 = session.get(
        url=i_url,
        headers={
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36',
        }
    )
    soup2 = BeautifulSoup(i2.text, 'lxml')
    for j in soup2.find_all(class_='blogpost-body blogpost-body-html'):
        if j.a :
            j_url=j.a.get('href')
            print(j_url)