Python爬虫爬取ECCV Conference Papers（一）

爬取到2020年所有论文标题

代码：

import re
import requests
from bs4 import BeautifulSoup
import lxml
import traceback
import time
import json
from lxml import etree
def get_paper():
    #https://www.ecva.net/papers/eccv_2020/papers_ECCV/html/267_ECCV_2020_paper.php
    #https://www.ecva.net/papers/eccv_2020/papers_ECCV/html/283_ECCV_2020_paper.php
    #https://www.ecva.net/papers/eccv_2020/papers_ECCV/html/343_ECCV_2020_paper.php
    url='https://www.ecva.net/papers.php'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36'
    }
    response=requests.get(url,headers)
    response.encoding='utf-8'
    page_text=response.text
    #输出页面html
    # print(page_text)
    soup = BeautifulSoup(page_text,'lxml')
    all_dt=soup.find_all('dt',class_='ptitle')
    for dt in all_dt:
        single_dt=str(dt)
        single_soup=BeautifulSoup(single_dt,'lxml')
        title=single_soup.find('a').text
        print(title)
    return
if (__name__=='__main__'):
    get_paper()