Selenium+PhantomJS自动化登录爬取博客文章

selenium采集页面元素

phantomjs主要是模拟登录

也没多少说的，上代码吧

 

from selenium import webdriver
import selenium.webdriver.support.ui as ui
import time

def crawl_cnblogs(blog_url,username,pwd):

    driver = webdriver.PhantomJS()

    driver.get("http://passport.cnblogs.com/user/signin?ReturnUrl=http%3A%2F%2Fwww.cnblogs.com%2F")
    wait = ui.WebDriverWait(driver, 10)
    wait.until(lambda dr: dr.find_element_by_id('signin').is_displayed())
    driver.find_element_by_id("input1").send_keys(username)
    driver.find_element_by_id("input2").send_keys(pwd)
    driver.find_element_by_id("signin").click()
    wait.until(lambda dr: dr.find_element_by_id('login_area').is_displayed()) #登录成功

    driver.get(blog_url)
    wait.until(lambda dr: dr.find_element_by_id('mainContent').is_displayed())
    time.sleep(3)
    #articles = driver.find_element_by_xpath('//div[@class="postTitle"]/a') #为啥不成功
    articles = driver.find_elements_by_class_name("postTitle")
    for article in articles:
        print article
　　　　 #print article.text
        #print article.text.decode("utf-8", "ignore")
        # scrapy爬虫之爬取汽车信息 编码居然错误 

    urls = driver.find_elements_by_class_name("postTitle2")
    for url in urls:
        print url.get_attribute("href")

    driver.save_screenshot('screen.png')
    driver.quit()

if __name__ == '__main__':
    crawl_cnblogs("http://www.cnblogs.com/xiaoyy3/", "xiaoyaoyou3", "------password---------")

　　

运行结果

编码错误，需要改成 print article.text.encode('gb18030')

运行结果为