Python Watchdog PollingObserverVFS Повышение производительности - PullRequest
Новая
       75

Python Watchdog PollingObserverVFS Повышение производительности

0 голосов
/ 19 апреля 2020

Я использую PollingObserverVFS в каталоге в хранилище Dell ISILON с более чем 2 миллионами файлов, и для обнаружения изменений требуется около 2 часов 30 минут. Есть ли способ повысить производительность, чтобы быстрее обнаруживать изменения, такие как параллельная обработка или любые другие изменения настроек.

Код объединяет хранилище dell isilon для обнаружения изменений папки. 

import datetime
import time 
import os
from watchdog.observers.polling import PollingObserverVFS
from watchdog.events import FileSystemEventHandler


if __name__ == "__main__": 

 # Set format for displaying path 
my_event_handler = FileSystemEventHandler()

path = "\\\\dd\\ARCHIVE\\"

print(path)
# Initialize logging event handler 
#event_handler = LoggingEventHandler() 
def on_any_event(event):

    print(datetime.datetime.now())

    if event.is_directory: 
        if event.event_type == 'created':
             print("Watchdog received directory create event - % s." % event.src_path) 

        elif event.event_type == 'modified': 
            # Event is modified, you can process it now 
            print("Watchdog received directory modified event - % s." % event.src_path) 

        elif event.event_type == 'moved': 
            # Event is modified, you can process it now 
           print("Watchdog received directory moved event - % s." % event.src_path) 
           print("Moved or Renamed directory path - % s." %event.dest_path )

        elif event.event_type == 'deleted': 
            # Event is modified, you can process it now 
           print("Watchdog received deleted event - % s." % event.src_path)


    elif event.event_type == 'created': 
            # Event is modified, you can process it now 
            print("Watchdog received file created event - % s." % event.src_path) 

    elif event.event_type == 'modified': 
            # Event is modified, you can process it now 
            print("Watchdog received modified file event - % s." % event.src_path) 

    elif event.event_type == 'moved': 
            # Event is modified, you can process it now 
           print("Watchdog received moved file event - % s." % event.src_path) 
           print("Moved or Renamed path - % s." %event.dest_path )

    elif event.event_type == 'deleted': 
            # Event is modified, you can process it now 
           print("Watchdog received deleted file event - % s." % event.src_path)        


my_event_handler.on_any_event = on_any_event

observer = PollingObserverVFS(stat=os.stat, listdir=os.listdir,polling_interval=60)
observer.schedule(my_event_handler, path, recursive=True) 

# Start the observer 
observer.start() 
try: 
    while True: 
        # Set the thread sleep time 
        time.sleep(60) 
except KeyboardInterrupt: 
    observer.stop() 
observer.join()
...