use selenium in scrapy webdriver
https://pypi.python.org/pypi/selenium
from selenium import webdriver from selenium.webdriver.common.keys import Keys browser = webdriver.Firefox() browser.get('http://www.yahoo.com') assert 'Yahoo' in browser.title elem = browser.find_element_by_name('p') # Find the search box elem.send_keys('seleniumhq' + Keys.RETURN) browser.quit()
//do get all contents
driver = webdriver.Chrome() driver.get("http://www.ask.com") print driver.title inputElement = driver.find_element_by_name("q") # type in the search inputElement.send_keys("baseball hats") # submit the form (although google automatically searches now without submitting) inputElement.submit() try: WebDriverWait(driver, 10).until(EC.title_contains("baseball")) print driver.title output = driver.page_source print(output) finally: driver.quit()
iframes = driver.find_elements_by_tag_name("iframe") for iframe in iframes driver.switch_to_default_content() driver.switch_to_frame(iframe) output = driver.page_source print(output)
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】凌霞软件回馈社区,博客园 & 1Panel & Halo 联合会员上线
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】博客园社区专享云产品让利特惠,阿里云新客6.5折上折
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步