使用pyinotify监控目录变化

 

关于线程池：python线程池(ThreadPoolExecutor)与进程池(ProcessPoolExecutor)的简单用法 - 知乎 (zhihu.com)

关于与pyinotify的基本使用：使用 Python 监控文件系统 - 知乎 (zhihu.com)

关于pyinotify的使用：Python 模块学习 - pyinotify | We all are data.

 

修改参数，防止队列及用户监控数不够

fs.inotify.max_user_watches = 8192000
fs.inotify.max_queued_events = 1638400

 

对于目录下文件，完成上传后，对文件进行移动或复制

代码基本实现：

#coding:utf-8
import os
import sys
import time
import shutil
import datetime
import pyinotify
import logging
import logging.handlers
from concurrent.futures import ThreadPoolExecutor

exec = ThreadPoolExecutor(max_workers=10)

def copy_files(path):
    logger.info('我将拷贝文件{}'.format(path))
    # shutil.copy("src","dst")
    logger.info('我将拷贝文件{}'.format(path))


class CreteLog(object):
    def __init__(self):
        self.logger = logging.getLogger('w3logger')
        self.logger.setLevel(logging.INFO)
    
    def statup(self,path,stream=True):
        write_handler = logging.handlers.TimedRotatingFileHandler(path, when='midnight', interval=1, backupCount=7, atTime=datetime.time(0, 0, 0, 0))
        write_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(module)s[:%(lineno)d] - %(message)s"))
        
        if stream:
            stream_handler = logging.StreamHandler()
            stream_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(module)s[:%(lineno)d] - %(message)s"))
            self.logger.addHandler(stream_handler)
        self.logger.addHandler(write_handler)
        
        return self.logger



class MonitorEvent(pyinotify.ProcessEvent):
    def __init__(self,func):
        super().__init__()
        self.func = func
        self.flag = time.time()
    
    # 文件被创建
    def process_IN_CREATE(self,event):
        if time.time() - self.flag > 0:
            file_full_path = os.path.join(event.path,event.name)
            print()
            logger.info("{} 被创建！".format(file_full_path))
            self.flag = time.time()
            # 这里就可以做想做的事情了
    
    # 文件被修改
    def process_IN_MODIFY(self,event):
        if time.time() - self.flag > 3:
            file_full_path = os.path.join(event.path,event.name)
            logger.info("{} 被修改！".format(file_full_path))
            self.flag = time.time()
            # 这里就可以做想做的事情了
    
    # 文件被删除
    def process_IN_DELETE(self,event):
        if time.time() - self.flag > 0:
            file_full_path = os.path.join(event.path,event.name)
            logger.info("{} 被删除！".format(file_full_path))
            self.flag = time.time()
            # 这里就可以做想做的事情了
    
    # 文件写入完毕
    def process_IN_CLOSE_WRITE(self,event):
        if time.time() - self.flag > 0:
            file_full_path = os.path.join(event.path,event.name)
            logger.info("{} 写入完毕！".format(file_full_path))
            self.flag = time.time()
            # 这里就可以做想做的事情了
            exec.submit(self.func,file_full_path)



def main():
    
    try:
        path = sys.argv[1]
    except:
        raise(AttributeError("has no attribute path"))

    vm = pyinotify.WatchManager()
    vm.add_watch(path,pyinotify.ALL_EVENTS,rec = True)
    event = MonitorEvent(func=copy_files)
    notifier = pyinotify.ThreadedNotifier(vm,event)
    notifier.loop()

if __name__ == '__main__':
    createlog = CreteLog()
    logger = createlog.statup("test.log",stream=True)
    main()

　　

#coding:utf-8

import os

import sys

import time

import shutil

import datetime

import pyinotify

import logging

import logging.handlers

from concurrent.futures import ThreadPoolExecutor

exec = ThreadPoolExecutor(max_workers=10)

def copy_files(path):

logger.info('我将拷贝文件{}'.format(path))

# shutil.copy("src","dst")

logger.info('我将拷贝文件{}'.format(path))

class CreteLog(object):

def __init__(self):

self.logger = logging.getLogger('w3logger')

self.logger.setLevel(logging.INFO)

 

def statup(self,path,stream=True):

write_handler = logging.handlers.TimedRotatingFileHandler(path, when='midnight', interval=1, backupCount=7, atTime=datetime.time(0, 0, 0, 0))

write_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(module)s[:%(lineno)d] - %(message)s"))

 

if stream:

stream_handler = logging.StreamHandler()

stream_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(module)s[:%(lineno)d] - %(message)s"))

self.logger.addHandler(stream_handler)

self.logger.addHandler(write_handler)

 

return self.logger

class MonitorEvent(pyinotify.ProcessEvent):

def __init__(self,func):

super().__init__()

self.func = func

self.flag = time.time()

 

# 文件被创建

def process_IN_CREATE(self,event):

if time.time() - self.flag > 0:

file_full_path = os.path.join(event.path,event.name)

print()

logger.info("{} 被创建！".format(file_full_path))

self.flag = time.time()

# 这里就可以做想做的事情了

 

# 文件被修改

def process_IN_MODIFY(self,event):

if time.time() - self.flag > 3:

file_full_path = os.path.join(event.path,event.name)

logger.info("{} 被修改！".format(file_full_path))

self.flag = time.time()

# 这里就可以做想做的事情了

 

# 文件被删除

def process_IN_DELETE(self,event):

if time.time() - self.flag > 0:

file_full_path = os.path.join(event.path,event.name)

logger.info("{} 被删除！".format(file_full_path))

self.flag = time.time()

# 这里就可以做想做的事情了

 

# 文件写入完毕

def process_IN_CLOSE_WRITE(self,event):

if time.time() - self.flag > 0:

file_full_path = os.path.join(event.path,event.name)

logger.info("{} 写入完毕！".format(file_full_path))

self.flag = time.time()

# 这里就可以做想做的事情了

exec.submit(self.func,file_full_path)

def main():

 

try:

path = sys.argv[1]

except:

raise(AttributeError("has no attribute path"))

vm = pyinotify.WatchManager()

vm.add_watch(path,pyinotify.ALL_EVENTS,rec = True)

event = MonitorEvent(func=copy_files)

notifier = pyinotify.ThreadedNotifier(vm,event)

notifier.loop()

if __name__ == '__main__':

createlog = CreteLog()

logger = createlog.statup("test.log",stream=True)

main()