python+selenium爬取关键字搜索google图片
1 # -*- coding: utf-8 -*- 2 3 import json 4 import os 5 import time 6 from multiprocessing import Pool 7 import multiprocessing 8 import requests 9 from selenium import webdriver 10 11 12 def get_image_links(keyword, num_requested = 1000): 13 """get image links with selenium 14 """ 15 number_of_scrolls = int(num_requested/400) + 1 16 img_urls = set()#设置为集合,自动去除重复链接 17 chrome_options = webdriver.ChromeOptions() 18 # chrome_options.add_argument('--headless')#设置无头浏览器 19 # chrome_options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"') 20 # chrome_options.add_argument("lang=en_US")#设置语言 21 # prefs = {"profile.managed_default_content_settings.images":2} 22 # chrome_options.add_experimental_option("prefs",prefs)#配置不加载图片 23 driver = webdriver.Chrome(chrome_options=chrome_options) 24 driver.maximize_window() 25 search_query = keyword 26 url = "https://www.google.com/search?q="+search_query+"&source=lnms&tbm=isch" 27 driver.get(url) 28 for _ in range(number_of_scrolls): 29 for i in range(5): 30 # multiple scrolls needed to show all 400 images 31 driver.execute_script("window.scrollBy(0, 100000)") 32 time.sleep(1) 33 time.sleep(5)#等待页面刷新,否则有可能元素不可见 34 try: 35 # driver.find_element_by_xpath("//input[@value='Show more results']").click()#浏览器的中英文版本不同 36 driver.find_element_by_xpath("//input[@value='显示更多结果']").click() 37 except Exception as e: 38 print("reach the end of page ") 39 break 40 41 # with open('page.html','w') as f: 42 # f.write(driver.page_source) 43 imgs = driver.find_elements_by_xpath('//div[contains(@class,"rg_meta")]')#模糊定位 44 for i,img in enumerate(imgs): 45 img_url = json.loads(img.get_attribute('innerHTML'))["ou"] 46 img_urls.add(img_url) 47 driver.quit() 48 print("finish getting all image urls!") 49 50 return img_urls 51 52 def download(urls,download_dir): 53 '''download images 54 ''' 55 print("start downloading images!") 56 for url in urls: 57 filename=os.path.join(download_dir,os.path.basename(url)) 58 try: 59 r = requests.get(url, stream=True, timeout=60) 60 r.raise_for_status() 61 with open(filename, 'wb') as f: 62 f.write(r.content) 63 except Exception: 64 continue 65 print("finish downloading images!") 66 67 keywords = ['girl','boy'] 68 download_dir = './images/' 69 download_dirs = [] 70 for keyword in keywords: 71 path = os.path.join(download_dir,keyword) 72 download_dirs.append(path) 73 if not os.path.exists(path): 74 os.makedirs(path) 75 76 # for keyword in main_keywords: 77 # image_urls = get_image_links(keyword) 78 # download(image_urls,download_dir) 79 80 81 ################################### 82 # get image links/MultiProcess 83 ################################### 84 img_urls=[] 85 multiprocessing.freeze_support() 86 p = Pool(4) # default number of process is the number of cores of your CPU, change it by yourself 87 for keyword in keywords: 88 img_urls.append(p.apply_async(get_image_links, (keyword,))) 89 #img_urls:[<multiprocessing.pool.ApplyResult object at 0x7f536925fcc0>, <multiprocessing.pool.ApplyResult object at 0x7f536925fd68>] 90 for i,urls in enumerate(img_urls): 91 img_urls[i]=urls.get() 92 p.close() 93 p.join() 94 95 96 # # ################################### 97 # # # download images/MultiProcess 98 # # ################################### 99 p = Pool(4) # default number of process is the number of cores of your CPU, change it by yourself 100 for i,urls in enumerate(img_urls): 101 p.apply_async(download, [urls,download_dirs[i]]) 102 p.close() 103 p.join()
桔桔桔桔桔桔桔桔桔桔