I try to monitor the contents of a directory tree, which my contain a huge ammount of files (many directories with 9000 files per directory as an example).
Synchron mode:
I first tryied using ReadDirectoryChangesW in blocking mode (synchronous), but when I delete the watched directory I end up in a deadlock which I can't detect nor exit.
#
# Monitors a directory for changes and pass the changes to the queue
#
def MonitorDirectory(self, out_queue):
print("Monitoring instance \'{0}\' is watching directory: {1}".format(self.name, self.path))
# File monitor
FILE_LIST_DIRECTORY = 0x0001
buffer = win32file.AllocateReadBuffer(1024 * 64)
hDir = win32file.CreateFile(self.path,
FILE_LIST_DIRECTORY,
win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE | win32con.FILE_SHARE_DELETE,
None,
win32con.OPEN_EXISTING,
win32con.FILE_FLAG_BACKUP_SEMANTICS,
None)
# Monitor directory for changes
while not self._shutdown.is_set():
# Create handle to directory if missing
#if os.path.isdir(self.path):
self.fh.write("ReOpen Exists {0}\n".format(os.path.isdir(self.path)))
self.fh.flush()
try:
hDir = win32file.CreateFile(self.path,
FILE_LIST_DIRECTORY,
win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE | win32con.FILE_SHARE_DELETE,
None,
win32con.OPEN_EXISTING,
win32con.FILE_FLAG_BACKUP_SEMANTICS,
None)
except:
self.fh.write("Handle is dead\n")
self.fh.flush()
try:
self.fh.write("{0}\n".format(newH))
self.fh.flush()
except:
self.fh.write("Write failed\n")
self.fh.flush()
self.fh.write("Check Changes\n")
self.fh.flush()
results = win32file.ReadDirectoryChangesW(hDir,
1024 * 64,
True,
win32con.FILE_NOTIFY_CHANGE_FILE_NAME |
win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
win32con.FILE_NOTIFY_CHANGE_SIZE |
win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
win32con.FILE_NOTIFY_CHANGE_SECURITY,
None,
None)
# Add all changes to queue
for action, file in results:
self.fh.write("Action: {0} on {1}\n".format(action, file))
out_queue.put((action, time.time(), os.path.join(self.path, file)))
self.fh.flush()
#else:
# Done main loop
print("Monitoring instance \'{0}\' has finished watching directory: {1}".format(self.name, self.path))
there just seemed to be no way to avoid the call from blocking when the watched directory is removed?
Also as the function is running in a thread, I cannot kill it when deadlocked, from a "supervisor" thread which would monitor the parent directory for DELETE actions on the watched directory and I dont really like that being a good solution as it involves much more code.
ASynchron mode:
I then tried the overlapped mode (async) which does not block in a deadlock, but I can't detect when the directory handle becomes void as the diorectory is deleted. The WaitForSingleObject call just time's out, and checking if the directory is present with os.path.isdir does not help because if the directory is recreated in the mean time, it will not return False, but the old directory handle is still invalid and will not detect the changes in the newly created directory with the same name.
Afer days of trying various approaches, I finnaly got to this code, which however does not work flawlessly bacause it still does not detect the removel of the watched directory and it also does miss a few files when mass deleting files rapidly. A thing which the sync mode did not.
#
# Monitors a directory for changes and pass the changes to the queue
#
def MonitorDirectory(self, out_queue):
print("Monitoring instance \'{0}\' is watching directory: {1}".format(self.name, self.path))
# File monitor
FILE_LIST_DIRECTORY = 0x0001
overlapped = pywintypes.OVERLAPPED()
overlapped.hEvent = win32event.CreateEvent(None, False, 0, None)
buffer = win32file.AllocateReadBuffer(1024 * 64)
# Main loop to keep watching active
while not self._shutdown.is_set():
# Open directory
try:
hDir = win32file.CreateFile(self.path,
FILE_LIST_DIRECTORY,
win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE | win32con.FILE_SHARE_DELETE,
None,
win32con.OPEN_EXISTING,
win32con.FILE_FLAG_BACKUP_SEMANTICS | win32con.FILE_FLAG_OVERLAPPED,
None)
except:
# Wait before retry
time.sleep(1)
else:
# Monitor directory for changes
while not self._shutdown.is_set():
win32file.ReadDirectoryChangesW(hDir,
buffer,
True,
win32con.FILE_NOTIFY_CHANGE_FILE_NAME |
win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
win32con.FILE_NOTIFY_CHANGE_SIZE |
win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
win32con.FILE_NOTIFY_CHANGE_SECURITY,
overlapped,
None)
# Wait for the changes
rc = win32event.WaitForSingleObject(overlapped.hEvent, 10000)
if rc == win32event.WAIT_OBJECT_0:
try:
bytes_returned = win32file.GetOverlappedResult(hDir, overlapped, True)
except:
raise Exception("Error: handle invalid?")
else:
# Get the changes
for action, file in win32file.FILE_NOTIFY_INFORMATION(buffer, bytes_returned):
out_queue.put((action, time.time(), os.path.join(self.path, file)))
elif rc == win32event.WAIT_TIMEOUT:
print("Monitoring instance \'{0}\': Timeout, no actions")
else:
raise Exception("Error?! RC = {0}".format(rc))
# Done main loop
print("Monitoring instance \'{0}\' has finished watching directory: {1}".format(self.name, self.path))
Is there a way to handle the detection of the removal of the watched directory, instead of just removing the win32con.FILE_SHARE_DELETE flag?
