搜狗图片抓取,主要以图集类进行抓取
搜狗图集抓取,类型比较多,但是数据量比较少
import os import time from concurrent.futures.thread import ThreadPoolExecutor import requests import re import json from urllib import parse from pymongo import MongoClient class Save: def __init__(self, host): self.client = MongoClient(host=host, port=27017) self.db = self.client.ImageSet def _save_data_mongodb(self, collect_name, data): self.collect_name = self.db[collect_name] history_record = self.collect_name.find_one({"_id": data['id']}) if history_record: return True else: self.collect_name.update_one({'_id': data['id']}, {'$set': data}, upsert=True) return True class SouHu: def __init__(self, category): self.category_name = category self.category = parse.quote(category) self.image_url_temp = "https://pic.sogou.com/pics/imageddetail2013.jsp?k="+self.category+"&tc=&t=&id=0&d={}" self.start_url = "https://pic.sogou.com/pics?query="+self.category+"&mode=8&dm=11&leftp=44230502&cwidth=1024&cheight=768&st=0&start={}&reqType=ajax&reqFrom=result&tn=0" self.headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36" } def get_title_id_grpdocs(self, url): print(url) response = requests.get(url=url, headers=self.headers) json_response = json.loads(response.text) json_dict = json_response.get('items') image_content = [] for i in json_dict: item={} try: id = i['mf_id'] title = i['title'] grpdocs = i['grpdocs'] rule = re.compile(r'\s+') grpdocs = rule.sub(',', grpdocs) item['id'] = id item['title'] = title item['ImageUrl'] = self.image_url_temp.format(grpdocs) image_content.append(item) except Exception as e: print(e) continue print(image_content) return image_content def get_save_content(self, image_content): save_content = [] for image in image_content: item={} url = image['ImageUrl'] response = requests.get(url=url, headers=self.headers) json_response = json.loads(response.text) image_list = [] for i in json_response: try: image_url = i['pic_url'] except: try: image_url = i['ori_pic_url'] except: continue image_list.append(image_url) item['id'] = image['id'] item['title'] = image['title'] item['url'] = image_list save_content.append(item) print(save_content) return save_content def save_(self,save_content): upload_time = time.strftime("%Y-%m-%d", time.localtime()) print("开始写入") for i in save_content: if len(i['url']) < 3: continue collect_name = "搜狗图片" result = Save("localhost")._save_data_mongodb(collect_name, data=i) if result: try: rule = re.compile(r'\s*', re.S) rule2 = re.compile(r'\W*', re.S) title = rule.sub('', i['title']) title = rule2.sub('', title) path = 'D:/搜狗/'+self.category_name+'/' + str(upload_time) + '/' + title except Exception as e: print(e) continue if os.path.exists(path): continue else: os.makedirs(path) try: with open(path + '/content.txt', 'w', encoding='utf8')as fb: fb.write(str([i['title']])) for s in i['url']: a = i['url'].index(s) with open(path + '/{}.jpg'.format(str(a)), 'wb') as f: print(s) response = requests.get(url=s) f.write(response.content) except Exception as e: print(e) continue print(title+" 写入完成") else: continue def run(self, num): url = self.start_url.format(num) image_content = self.get_title_id_grpdocs(url) save_content = self.get_save_content(image_content) self.save_(save_content) if __name__ == '__main__': category = input("输入分类名称:") with ThreadPoolExecutor(10) as executor: sh = SouHu(category) for num in range(2400): executor.submit(sh.run, num)