抓取猎聘网 职位与职位网址

import requests
from fake_useragent import UserAgent
from lxml import etree
import time

start = time.time()
Title = []
Href = []

for num in range(0,51):
    ua = UserAgent()
    headers = {'User-Agent': ua.random}
    url = "https://www.liepin.com/zhaopin/?ckId=lqquzhl92u2xj926ospwblclsa3i8wqa&fkId=yf2ci851pzp10g2zxbuffq7oi5e7vxha&skId=yf2ci851pzp10g2zxbuffq7oi5e7vxha&sfrom=search_job_pc&key=%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90%E5%B8%88&scene=input&currentPage="+(str(num))
    response =requests.get(url,headers=headers)
    response.encoding = "utf-8"
    #print(response.text)

    html = etree.HTML(response.text)
    titles = html.xpath('/html/body/div[1]/div/section[1]/div/ul/li/div/div/div[1]/div/a[1]/div[1]/div/div[1]/text()')
    href_title = html.xpath('/html/body/div[1]/div/section[1]/div/ul/li/div/div/div[1]/div/a[1]/@href')
    Title.extend(titles)
    Href.extend(href_title)
    print("正在加载数据--{}".format(str(num)))
    time.sleep(0.2)

print(Title)
print(Href)

print(len(Title))
print(len(Href))

end = time.time()
print(start - end)