开发笔记 -- MongoDB 单个数据集合拆分脚本
场景描述：
数据清洗过程中发现，Mongodb中单个集合(表)数据量比较大(千万)，定时任务处理加工数据时，由于服务器内存有限，会在处理过程中，出现内存不足的异常，于是希望把单个集合进行拆分成多个，在网上找了相关文章，发现处理这种的场景的案例比较少，基本上考虑了如下3中方案来实现：
1. mongodb sharding 分片 ---还未研究透彻
2. mongodb 聚合查询，将结果封装成变量，循环遍历，写入新的集合 ---测试了下，效率较低 --文末会说明该方案，适用于小数据量
3. 自定义python脚本 ---自己动手，丰衣足食
处理方式：
这里采用了自己写脚本处理的方式，好了！逻辑比较简单，就是将源数据对象读取，循环遍历，到达自己指定的count计数后，开多线程，将符合条件的数据写入新集合(表)。
脚本内容：小伙伴们，可以根据自己实际需求修改脚本内相应的条件，有更好的实现方案，欢迎交流。---脚本运行依赖包 pymongo
---说明：由于pool.map(thread_insert_001, source_rows_list) map方法，没有找到传递多个参数的方式，因此脚本中if else和重复函数较多，封装的不好。
# -*- coding: utf-8 -*- import datetime import json # from bson import ObjectId from pymongo import MongoClient from multiprocessing.pool import ThreadPool # 将3千万条数据拆分成多个集合，每400万一组 # 大概拆分成8组 # mongodb settings settings = { "ip": '127.0.0.1', # ip "port": 27007, # 端口 "db_name": "add_your_db_name", # 数据库名字 "set_name": "add_your_collection_name" , # 集合名字 "new_db_name": "add_your_target_db_name", "target_set_name_001": "coll_001", "target_set_name_002": "coll_002", "target_set_name_003": "coll_003", "target_set_name_004": "coll_004", "target_set_name_005": "coll_005", "target_set_name_006": "coll_006", "target_set_name_007": "coll_007", "target_set_name_008": "coll_008", } class MyMongoDB(): def __init__(self): try: self.conn = MongoClient(settings["ip"], settings["port"]) except Exception as e: print(e) self.db = self.conn[settings["db_name"]] self.my_set = self.db[settings["set_name"]] def insert(self, dic): print("inser...") if dic: self.my_set.insert(dic) else: pass def update(self, dic, newdic): print("update...") self.my_set.update(dic, newdic) def delete(self, dic): print("delete...") self.my_set.remove(dic) def dbfind(self, dic=None): print("find...") current_set = self.my_set data = current_set.find() return data def thread_insert_001(rows_dic): conn_mongodb = MongoClient(settings["ip"], settings["port"]) new_db = conn_mongodb[settings["new_db_name"]] new_collect_set = new_db[settings["target_set_name_001"]] new_collect_set.insert(rows_dic) def thread_insert_002(rows_dic): conn_mongodb = MongoClient(settings["ip"], settings["port"]) new_db = conn_mongodb[settings["new_db_name"]] new_collect_set = new_db[settings["target_set_name_002"]] new_collect_set.insert(rows_dic) def thread_insert_003(rows_dic): conn_mongodb = MongoClient(settings["ip"], settings["port"]) new_db = conn_mongodb[settings["new_db_name"]] new_collect_set = new_db[settings["target_set_name_003"]] new_collect_set.insert(rows_dic) def thread_insert_004(rows_dic): conn_mongodb = MongoClient(settings["ip"], settings["port"]) new_db = conn_mongodb[settings["new_db_name"]] new_collect_set = new_db[settings["target_set_name_004"]] new_collect_set.insert(rows_dic) def thread_insert_005(rows_dic): conn_mongodb = MongoClient(settings["ip"], settings["port"]) new_db = conn_mongodb[settings["new_db_name"]] new_collect_set = new_db[settings["target_set_name_005"]] new_collect_set.insert(rows_dic) def thread_insert_006(rows_dic): conn_mongodb = MongoClient(settings["ip"], settings["port"]) new_db = conn_mongodb[settings["new_db_name"]] new_collect_set = new_db[settings["target_set_name_006"]] new_collect_set.insert(rows_dic) def thread_insert_007(rows_dic): conn_mongodb = MongoClient(settings["ip"], settings["port"]) new_db = conn_mongodb[settings["new_db_name"]] new_collect_set = new_db[settings["target_set_name_007"]] new_collect_set.insert(rows_dic) def thread_insert_008(rows_dic): conn_mongodb = MongoClient(settings["ip"], settings["port"]) new_db = conn_mongodb[settings["new_db_name"]] new_collect_set = new_db[settings["target_set_name_008"]] new_collect_set.insert(rows_dic) def split_mg_dt_col_to_new_col(): # mongodb 旧集合数据 拆分 写入新集合 my_mongodb_obj = MyMongoDB() data = my_mongodb_obj.dbfind() # 开10个线程，根据服务器配置修改 pool = ThreadPool(10) count = 1 source_rows_list = [] for result in data: count += 1 source_rows_list.append(result) # print(source_rows_list) if count < 4000000: if len(source_rows_list) == 2000: print('11111111111111111') print(count) print(len(source_rows_list)) insert_dic = pool.map(thread_insert_001, source_rows_list) del source_rows_list[:] elif 4000000 <= count < 8000000: if len(source_rows_list) == 2000: print('222222222222222222') print(count) print(len(source_rows_list)) insert_dic = pool.map(thread_insert_002, source_rows_list) del source_rows_list[:] elif 8000000 <= count < 12000000: if len(source_rows_list) == 2000: print('333333333333333333') print(count) print(len(source_rows_list)) insert_dic = pool.map(thread_insert_003, source_rows_list) del source_rows_list[:] elif 12000000 <= count < 16000000: if len(source_rows_list) == 2000: print('44444444444444444444') print(count) print(len(source_rows_list)) insert_dic = pool.map(thread_insert_004, source_rows_list) del source_rows_list[:] elif 16000000 <= count < 20000000: if len(source_rows_list) == 2000: print('5555555555555555555555') print(count) print(len(source_rows_list)) insert_dic = pool.map(thread_insert_005, source_rows_list) del source_rows_list[:] elif 20000000 <= count < 24000000: if len(source_rows_list) == 2000: print('666666666666666666666') print(count) print(len(source_rows_list)) insert_dic = pool.map(thread_insert_006, source_rows_list) del source_rows_list[:] elif 24000000 <= count < 28000000: if len(source_rows_list) == 2000: print('7777777777777777777') print(count) print(len(source_rows_list)) insert_dic = pool.map(thread_insert_007, source_rows_list) del source_rows_list[:] elif 28000000 <= count < 32000000: if len(source_rows_list) == 2000: print('88888888888888888') print(count) print(len(source_rows_list)) insert_dic = pool.map(thread_insert_008, source_rows_list) del source_rows_list[:] pool.close() if __name__ == '__main__': split_mg_dt_col_to_new_col()
---补充：
mongodb 查询结果，循环遍历写入，边读边取，不是一次直接返回所有数据集，类似mysql的游标或者指针
var result = db.csmdR.find({"查询条件"}) while(result.hasNext()) db.新集合(表)名.insert(result.next())
