mongodb批量删除数据(python)

 

 

#!/usr/bin/env python
#coding=utf-8
import os, json, urllib, datetime, shutil,random,uuid
from random import choice
import pymysql
import traceback
import time
from pymongo import MongoClient
from pymongo import InsertOne
from pymongo import DeleteMany
import time
import random



gl_mysql_server = "192.168.1.134"
gl_user_name = "root"
gl_password = "mysql"
gl_db_name_msg="db_pushmsg"
gl_port=4000

def insert_data():
    #连接数据库
    conn = MongoClient("192.168.1.135:28001",maxPoolSize=None)
    my_db = conn['db_pushmsg']
    my_collection = my_db['app_message_all_01']

    ip_list = ['192.168.1.1', '192.168.1.2', '192.168.1.3', '192.168.1.4', '192.168.1.5', '192.168.1.6', '192.168.1.7','192.168.1.8', '192.168.1.9', '192.168.1.10', '192.168.1.11', '192.168.1.12', '192.168.1.13', '192.168.1.14', '192.168.1.15']
    message_type=[0,2,3,6,8,9,16,4096,8196,8448,12292,16385,16386,16387,32770,65536,69632,73728,77824]
    massive_type=[0, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007]
    status = [1000,2000]

    # 批量写
    t0 = time.time()
    data =[]
    for i in range(1, 10301):

        str_i = str(i)
        curr_time = datetime.datetime.now()
        l_user_id = random.randint(10000000, 99999999)
        l_message_type = choice(message_type)
        l_massive_type = choice(massive_type)
        l_status = choice(status)
        l_create_time = str(curr_time)
        l_message_id = random.randint(1000000000000000, 9999999999999999)
        l_sender_seq_no = str(uuid.uuid4())
        l_title = 'test，请点击查看' + str_i
        l_message = 'test'
        l_extra = 'test'
        l_send_date = str(curr_time)
        l_deleted = 0

        data.append(InsertOne({"message_id": l_message_id,
                               "user_id": l_user_id,
                               "sender_seq_no": l_sender_seq_no,
                               "title": l_title,
                               "message": l_message,
                               "extra": l_extra,
                               "send_date": l_send_date,
                               "status": l_status,
                               "message_type": l_message_type,
                               "massive_type": l_massive_type,
                               "deleted": l_deleted,
                               "create_time": l_create_time
                               }))
        i+=1
        if (i % 1000) == 0:
            my_collection.bulk_write(data)
            data = []
            i += 1
        else:
            continue
            ##my_collection.bulk_write(data) ##处理不能取mod的数据

    my_collection.bulk_write(data) ##处理不能取mod的数据
    return 0

##delete_many
def delete_data01():
    #连接数据库
    conn = MongoClient("192.168.1.135:28001",maxPoolSize=None)
    my_db = conn['db_pushmsg']
    my_collection = my_db['app_message_all']

    data = []
    i = 1
    for x in my_collection.find({"massive_type":0}, {"_id": 1}, no_cursor_timeout=True).batch_size(5000):
        ##print(x)
        data.append(x["_id"])
        i = i + 1
        if len(data) == 1000:
            my_collection.delete_many({'_id': {"$in": data}})
            data = []
    my_collection.delete_many({'_id': {"$in": data}})


##bulk_write +delete_many
def delete_data02():
    #连接数据库
    conn = MongoClient("192.168.1.135:28001",maxPoolSize=None)
    my_db = conn['db_pushmsg']
    my_collection = my_db['app_message_all']

    data = []
    i = 1
    data01 = []
    for x in my_collection.find({"massive_type": 0}, {"_id": 1},no_cursor_timeout=True).batch_size(5000):
        ##print(x)
        data.append(x["_id"])
        i = i + 1
        if len(data) == 1000:
            data01.append(DeleteMany({'_id': {"$in": data}}))
            my_collection.bulk_write(data01)
            data = []
            data01 = []

    data01.append(DeleteMany({'_id': {"$in": data}}))
    my_collection.bulk_write(data01)

    #my_collection.delete_many({'_id': {"$in": data}})






if __name__ == '__main__':
    print("开始时间:"+time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
    ##insert_data()
    delete_data01()
    delete_data02()

    print("结束时间:"+time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))