爬取抽屉热搜榜文章
import os import requests from bs4 import BeautifulSoup # 登陆, 模仿用户浏览器 r1 = requests.get( # 要爬取的网页 url='https://dig.chouti.com/', # 浏览器的信息 headers={ 'user-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36' } ) # 编码 # r1.encoding = 'gbk' # 获取第一条cookie r1_cookie_dict = r1.cookies.get_dict() # 登陆信息,需要的时候再加上 # r2 = requests.post( # url='https://dig.chouti.com/login', # # 加入浏览器信息 # headers={ # 'user-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36' # }, # # 填写账号密码 # data={ # 'phone':'861352039****', # 'password':'******', # 'oneMonth':1 # }, # # 抽屉网还要用第一次那个cookies值 # cookies=r1_cookie_dict # ) # 去响应体中j解析我们想要的数据 soup = BeautifulSoup(r1.text, 'html.parser') # 按照规则找名字: div标签且id=content-list 找到匹配的第一个 container = soup.find(name='div', attrs={'id':'content-list'}) # 去container找到所有的class=item的div标签 div_list = container.find_all(name='div',attrs={'class':'item'}) # 循环所有的标签 for tag in div_list: # 找到class='part2'的所有div标签 articles = tag.find(name='div', attrs={'class': 'part2'}) # 如果为空就跳过 if not articles: continue # 找到class='summary'的所有span标签 summay = tag.find(name='span', attrs={'class': 'summary'}) if not summay: continue # 找到class='show-content color-chag'的所有a标签 articles_addr = tag.find(name='a', attrs='show-content color-chag') if not articles_addr: continue print('标题:',articles['share-title']) print('简介:',summay.text) print('文章地址:', articles_addr['href']) print('图片地址:', articles['share-pic']) print('------------------------------------------------') # 下载图片 # 找到图片地址https://img.jandan.net/news/2018/07/D5BA3729EF7F435B8BD5BC7DC8DCDA8C_W550H698.jpg img_addr = articles['share-pic'] # 切割出文件名D5BA3729EF7F435B8BD5BC7DC8DCDA8C_W550H698.jpg file_name = img_addr.rsplit('/', maxsplit=1)[1] # 创建一个文件夹来存要下载的图片 root = "F:/简单项目在这里/pacongfirst/img/" # join连接 拼接一个完整文件名 path_filename = os.path.join('root', file_name) # 或 path_filename = root + file_name try: # 如果文件夹不存在就创建 if not os.path.exists('root'): os.mkdir('root') # 如果文件不存在就下载 if not os.path.exists(path_filename): # 拿到图片 r3 = requests.get( url=img_addr ) with open(path_filename, 'wb') as f: f.write(r3.content) else: print('文件已存在') except: print('爬取失败')