提升600百万数据的处理速度 禁止图片下载

 

 

firefox_profile = webdriver.FirefoxProfile()
firefox_profile.set_preference('permissions.default.image', 2)#某些firefox只需要这个
firefox_profile.set_preference('browser.migration.version', 9001)#部分需要加上这个
#禁用css
firefox_profile.set_preference('permissions.default.stylesheet', 2)
#禁用flash
firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
# #禁用js
# firefox_profile.set_preference('javascript.enabled', 'false')

browser = webdriver.Firefox(firefox_profile=firefox_profile)

# browser = webdriver.Firefox()

for i in url_l:
    c+=1
    with open('DISTINCT_url.break.log', 'r', encoding='utf-8') as fc:
        for breaker in fc:
            break
    if c<int(breaker):
        continue