Asynchronous ReadDirectoryChangesW call blocks the thread from exiting

Question

Introduction:

I am writing a small app that monitors a certain directory for newly added files.

I would like to put the monitoring code in a separate thread, so I can leave the main thread free for other stuff and cancel the monitoring thread when I need to.

Relevant Information:

• I am using ReadDirectoryChangesW to do the monitoring
• I am using raw WIN32 API for thread creation/synchronization
• I am trying to support Windows XP onward.

Problem:

I was able to code everything properly, except for one thing:

I cannot exit the monitoring thread properly, hence this post.

I am signaling an event object in the main thread, wait for the thread to exit, and then do clean up.

The problem lies in my usage of ReadDirectoryChangesW since everything works fine after I comment out that piece of code.

Once the event handle is signaled, ReadDirectoryChangesW blocks the thread which prevents it to "catch" the event and exit. If I add a new file in the directory it "unblocks" ReadDirectoryChangesW, thread "catches" the event and exits.

To help further, I have made a small MVCE below, which illustrates what I have stated so far.

MVCE:

#include <iostream>
#include <Windows.h>
#include <map>

struct SThreadParams
{
    HANDLE hEvent;
    HANDLE hDir;
    int processDirectoryChanges(const char *buffer)
    {
        if (NULL == buffer) return -1;

        DWORD offset = 0;
        char fileName[MAX_PATH] = "";
        FILE_NOTIFY_INFORMATION *fni = NULL;

        do
        {
            fni = (FILE_NOTIFY_INFORMATION*)(&buffer[offset]);
            // since we do not use UNICODE, 
            // we must convert fni->FileName from UNICODE to multibyte
            int ret = ::WideCharToMultiByte(CP_ACP, 0, fni->FileName,
                fni->FileNameLength / sizeof(WCHAR),
                fileName, sizeof(fileName), NULL, NULL);

            switch (fni->Action)
            {
            case FILE_ACTION_ADDED:     
            {
                std::cout << "FILE_ACTION_ADDED " << fileName << std::endl;
            }
            break;
            case FILE_ACTION_REMOVED:
            {
                std::cout << "FILE_ACTION_REMOVED " << fileName << std::endl;
            }
            break;
            case FILE_ACTION_MODIFIED:
            {
                std::cout << "FILE_ACTION_MODIFIED " << fileName << std::endl;
            }
            break;
            case FILE_ACTION_RENAMED_OLD_NAME:
            {
                std::cout << "FILE_ACTION_RENAMED_OLD_NAME " << fileName << std::endl;
            }
            break;
            case FILE_ACTION_RENAMED_NEW_NAME:
            {
                std::cout << "FILE_ACTION_RENAMED_NEW_NAME " << fileName << std::endl;
            }
            break;
            default:
                break;
            }
            // clear string so we can reuse it
            ::memset(fileName, '\0', sizeof(fileName));
            // advance to next entry
            offset += fni->NextEntryOffset;

        } while (fni->NextEntryOffset != 0);

        return 0;
    }
};

DWORD WINAPI thread(LPVOID arg)
{
    SThreadParams p = *((SThreadParams *)arg);
    OVERLAPPED ovl = { 0 };
    DWORD bytesTransferred = 0, error = 0;
    char buffer[1024];

    if (NULL == (ovl.hEvent = ::CreateEvent(NULL, TRUE, FALSE, NULL)))
    {
        std::cout << "CreateEvent error = " << ::GetLastError() << std::endl;
        return ::GetLastError();
    };

    do {

        if (::ReadDirectoryChangesW(p.hDir, buffer, sizeof(buffer), FALSE,
            FILE_NOTIFY_CHANGE_FILE_NAME,
            NULL, &ovl, NULL))
        {
            if (::GetOverlappedResult(p.hDir, &ovl, &bytesTransferred, TRUE))
            {
                for (int i = 0; i < 5; ++i) std::cout << '=';
                std::cout << std::endl;

                if (-1 == p.processDirectoryChanges(buffer))
                    std::cout << "processDirectoryChanges error = " << std::endl;
            }
            else
            { 
                bytesTransferred = 0;
                std::cout << "GetOverlappedResult error = " << ::GetLastError() << std::endl;
            }

            if (0 == ::ResetEvent(ovl.hEvent))
            {
                std::cout << "ResetEvent error = " << ::GetLastError() << std::endl;
                ::CloseHandle(ovl.hEvent);
                return ::GetLastError();
            }
        }
        else
        {
            // we shall just output the error, and try again...
            std::cout << "ReadDirectoryChangesW error =  " << ::GetLastError() << std::endl;
        }

        error = ::WaitForSingleObject(p.hEvent, 2000);

    } while (WAIT_TIMEOUT == error);

    ::CloseHandle(ovl.hEvent);

    return 0;
}

int main()
{
    SThreadParams s;
    
    s.hDir = ::CreateFile(SOME_DIRECTORY,
            FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
            NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
    
    if (INVALID_HANDLE_VALUE == s.hDir)
    {
        std::cout << "CreateFile error = " << ::GetLastError() << std::endl;
        return 1;
    }

    s.hEvent = ::CreateEvent(NULL, TRUE, FALSE, NULL);
    
    if (NULL == s.hEvent)
    {
        std::cout << "CreateEvent error = " << ::GetLastError() << std::endl;
        ::CloseHandle(s.hDir);
        return 1;
    }
    
    HANDLE hThread = ::CreateThread(NULL, 0, thread, (LPVOID)&s, 0, NULL);
    
    if (NULL == hThread)
    {
        std::cout << "CreateThread error = " << ::GetLastError() << std::endl;
        ::CloseHandle(s.hDir);
        ::CloseHandle(s.hEvent);
        return 1;
    }

    std::cout << "press any key to close program..." << std::endl;
    std::cin.get();
    
    if (0 == ::CancelIoEx(s.hDir, NULL))
    {
        std::cout << "CancelIoEx error = " << ::GetLastError() << std::endl;
        ::CloseHandle(s.hDir);
        ::CloseHandle(s.hEvent);
        return 1;
    }

    if (0 == ::SetEvent(s.hEvent))
    {
        std::cout << "SetEvent error = " << ::GetLastError() << std::endl;
        ::CloseHandle(s.hDir);
        ::CloseHandle(s.hEvent);
        return 1;
    }

    // wait for thread to exit
    DWORD error = ::WaitForSingleObject(hThread, INFINITE);
    std::cout << "Thread exited with error code = " << error << std::endl;

    ::CloseHandle(s.hEvent);
    ::CloseHandle(s.hDir);
    ::CloseHandle(hThread);

    return 0;
}

My Efforts to Solve:

• I have moved out OVERLAPPED structure out of thread into structure that was passed to thread. Then I set OVERLAPPED.hEvent to forcibly "unblock" ReadDirectoryChangesW. This seems to work, but scares me because I think it is not safe/error prone since it is undocumented.

• I have tried to use completion routines but got no success since I am new with all this. I was able to receive notifications, but content of the buffer (the one filled with ReadDirectoryChangesW) was not read properly after the first pass. I am still trying to make this work on my own, but could use help.

• I could use I/o completion port, but since I will monitor only one directory I think this is a bit of an overkill. If I am mistaken, please instruct me how to use I/o completion port for my case, I would love to try them out.

Question:

Given the MVCE above, can you instruct me on how to modify the code in the thread procedure, so it exits properly (without ReadDirectoryChangesW blocking).

I have a feeling that i will have to use completion routines. In that case I would humbly ask for some pseudo code or written instructions since this would be my first time using them.

Each time I made a progress, I will update this post with relevant data accordingly.

Answer 1

exist 3 ways to do asynchronous operation with file:

• use ApcRoutine
• use IoCompletionPort
• use Event - worst

you select worst variant. i be use IoCompletionPort on your place. in this case you not need create events, threads, call GetOverlappedResult, not need any loops..

all what need call BindIoCompletionCallback (or RtlSetIoCompletionCallback) on file and all !

about cancelation - CancelIoEx not exist in XP ("I am trying to support Windows XP" ), but you can simply close directory handle - in this case IO will be canceled with STATUS_NOTIFY_CLEANUP. so code can look like this:

RUNDOWN_REF_EVENT g_rundown; // Run-Down Protection

class SPYDATA : 
#ifdef _USE_NT_VERSION_
    IO_STATUS_BLOCK
#else
    OVERLAPPED 
#endif
{
    HANDLE _hFile;
    LONG _dwRef;
    union {
        FILE_NOTIFY_INFORMATION _fni;
        UCHAR _buf[PAGE_SIZE];
    };

    void DumpDirectoryChanges()
    {
        union {
            PVOID buf;
            PBYTE pb;
            PFILE_NOTIFY_INFORMATION pfni;
        };

        buf = _buf;

        for (;;)
        {
            DbgPrint("%x <%.*S>\n", pfni->Action, pfni->FileNameLength >> 1, pfni->FileName);

            ULONG NextEntryOffset = pfni->NextEntryOffset;

            if (!NextEntryOffset)
            {
                break;
            }

            pb += NextEntryOffset;
        }
    }

#ifdef _USE_NT_VERSION_
    static VOID WINAPI _OvCompRoutine(
        _In_    NTSTATUS dwErrorCode,
        _In_    ULONG_PTR dwNumberOfBytesTransfered,
        _Inout_ PIO_STATUS_BLOCK Iosb
        )
    {
        static_cast<SPYDATA*>(Iosb)->OvCompRoutine(dwErrorCode, (ULONG)dwNumberOfBytesTransfered);
    }
#else
    static VOID WINAPI _OvCompRoutine(
        _In_    DWORD dwErrorCode, // really this is NTSTATUS
        _In_    DWORD dwNumberOfBytesTransfered,
        _Inout_ LPOVERLAPPED lpOverlapped
        )
    {
        static_cast<SPYDATA*>(lpOverlapped)->OvCompRoutine(dwErrorCode, dwNumberOfBytesTransfered);
    }
#endif

    VOID OvCompRoutine(NTSTATUS status, DWORD dwNumberOfBytesTransfered)
    {
        DbgPrint("[%x,%x]\n", status, dwNumberOfBytesTransfered);

        if (0 <= status) 
        {
            if (status != STATUS_NOTIFY_CLEANUP)
            {
                if (dwNumberOfBytesTransfered) DumpDirectoryChanges();
                DoRead();
            }
            else
            {
                DbgPrint("\n---- NOTIFY_CLEANUP -----\n");
            }
        }

        Release();
        g_rundown.ReleaseRundownProtection();
    }

    ~SPYDATA()
    {
        Cancel();
    }

public:

    void DoRead()
    {
        if (g_rundown.AcquireRundownProtection())
        {
            AddRef();
#ifdef _USE_NT_VERSION_
            NTSTATUS status = ZwNotifyChangeDirectoryFile(_hFile, 0, 0, this, this, &_fni, sizeof(_buf), FILE_NOTIFY_VALID_MASK, TRUE);
            if (NT_ERROR(status))
            {
                OvCompRoutine(status, 0);
            }
#else
            if (!ReadDirectoryChangesW(_hFile, _buf, sizeof(_buf), TRUE, FILE_NOTIFY_VALID_MASK, (PDWORD)&InternalHigh, this, 0))
            {
                OvCompRoutine(RtlGetLastNtStatus(), 0);
            }
#endif
        }
    }

    SPYDATA()
    {
        _hFile = 0;// ! not INVALID_HANDLE_VALUE because use ntapi for open file
        _dwRef = 1;
#ifndef _USE_NT_VERSION_
        RtlZeroMemory(static_cast<OVERLAPPED*>(this), sizeof(OVERLAPPED));
#endif
    }

    void AddRef()
    {
        InterlockedIncrement(&_dwRef);
    }

    void Release()
    {
        if (!InterlockedDecrement(&_dwRef))
        {
            delete this;
        }
    }

    BOOL Create(POBJECT_ATTRIBUTES poa)
    {
        IO_STATUS_BLOCK iosb;
        NTSTATUS status = ZwOpenFile(&_hFile, FILE_GENERIC_READ, poa, &iosb, FILE_SHARE_VALID_FLAGS, FILE_DIRECTORY_FILE);
        if (0 <= status)
        {
            return
#ifdef _USE_NT_VERSION_
            0 <= RtlSetIoCompletionCallback(_hFile, _OvCompRoutine, 0);
#else
            BindIoCompletionCallback(_hFile, _OvCompRoutine, 0);
#endif
        }
        return FALSE;
    }

    void Cancel()
    {
        if (HANDLE hFile = InterlockedExchangePointer(&_hFile, 0))
        {
            NtClose(hFile);
        }
    }
};

void DemoF()
{
    if (g_rundown.Create())
    {
        STATIC_OBJECT_ATTRIBUTES(oa, "\\systemroot\\tmp");//SOME_DIRECTORY

        if (SPYDATA* p = new SPYDATA)
        {
            if (p->Create(&oa))
            {
                p->DoRead();
            }

            MessageBoxW(0, L"wait close program...", L"", MB_OK);

            p->Cancel();

            p->Release();
        }

        g_rundown.ReleaseRundownProtection();
        g_rundown.WaitForRundown();
    }
}

for wait when all IO finished i use Run-Down Protection. unfortunately this is not implemented in user mode, but not hard implement this very useful feature yourself. my implementation:

class __declspec(novtable) RUNDOWN_REF
{
    LONG _LockCount;

protected:

    virtual void RundownCompleted() = 0;

public:

    RUNDOWN_REF()
    {
        _LockCount = 1;
    }

    BOOL AcquireRundownProtection()
    {
        LONG LockCount = _LockCount, prevLockCount;

        do 
        {
            if (!LockCount)
            {
                return FALSE;
            }

            LockCount = InterlockedCompareExchange(&_LockCount, LockCount + 1, prevLockCount = LockCount);

        } while (LockCount != prevLockCount);

        return TRUE;
    }

    void ReleaseRundownProtection()
    {
        if (!InterlockedDecrement(&_LockCount))
        {
            RundownCompleted();
        }
    }
};

class RUNDOWN_REF_EVENT : public RUNDOWN_REF
{
    HANDLE _hEvent;

    virtual void RundownCompleted()
    {
        SetEvent(_hEvent);
    }

public:

    BOOL Create()
    {
        return (_hEvent = CreateEvent(0, TRUE, FALSE, 0)) != 0;
    }

    RUNDOWN_REF_EVENT()
    {
        _hEvent = 0;
    }

    ~RUNDOWN_REF_EVENT()
    {
        if (_hEvent) CloseHandle(_hEvent);
    }

    void WaitForRundown()
    {
        if (WaitForSingleObject(_hEvent, INFINITE) != WAIT_OBJECT_0) __debugbreak();
    }
};