from selenium import webdriver
from lxml import etree
driver = webdriver.Chrome()
driver.get(url='https://www.ixigua.com/home/2515337690485792/?source=pgc_author_name&list_entrance=anyVideo')
# 页面最大
driver.maximize_window()
# 定义一个初始值
temp_height = 0
time.sleep(2)
while True:
# 循环将滚动条下拉
driver.execute_script("window.scrollBy(0,1000)")
# sleep一下让滚动条反应一下
time.sleep(2)
# 获取当前滚动条距离顶部的距离
check_height = driver.execute_script(
"return document.documentElement.scrollTop || window.pageYOffset || document.body.scrollTop;")
# 如果两者相等说明到底了
if check_height == temp_height:
# driver.refresh()
# 在这里获取网页源代码
html_data = driver.page_source
# time.sleep(1)
# print(html_data)
break
temp_height = check_height
# print(check_height)
driver.close
#提取内容
html = etree.HTML(html_data)
address_list=html.xpath('//a[@class="HorizontalFeedCard__coverWrapper disableZoomAnimation"]/@href')