How to detect if a File buffer has been flushed

Question

I'm trying to create a simple logging tool to monitor file changes. I have used the FileSystemWatcher to detect changes to the file, but I've discovered that the events are only triggered when the file is closed, not when the buffer is flushed. This means that if multiple lines are added before the file is closed, I will only see that when the file is closed.

here is my test example.

[TestClass]
public class FileWriteTests
{

    [TestMethod]
    public void TestMethodAfterClose()
    {
        var currentDir = Environment.CurrentDirectory;
        var fileToMonitor = "test.txt";
        List<string> output = new List<string>();
        var watcherTest = new FileWatcherTest(fileToMonitor, currentDir, output);

        File.Delete(Path.Combine(currentDir, fileToMonitor));
        using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
        {
            writer.WriteLine($"test");
            writer.Flush();
        }
        System.Threading.Thread.Sleep(10);
        Assert.AreEqual(1, output.Count);
        Assert.AreEqual("test", output[0]);

    }

    [TestMethod]
    public void TestMethodAfterFlush()
    {
        var currentDir = Environment.CurrentDirectory;
        var fileToMonitor = "test.txt";
        List<string> output = new List<string>();
        var watcherTest = new FileWatcherTest(fileToMonitor, currentDir, output);

        File.Delete(Path.Combine(currentDir, fileToMonitor));

        using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
        {
            try
            {
                writer.WriteLine($"test");
                writer.Flush();
                System.Threading.Thread.Sleep(1000);
                // add break point here for BareTail
                Assert.AreEqual(1, output.Count);
                Assert.AreEqual("test", output[0]);
            }
            catch
            {
                Assert.Fail("Test failed");
            }
        }
    }

    public class FileWatcherTest
    {
        public string FileName { get; set; }
        public string Directory { get; set; }
        private List<string> linesRead;
        private FileSystemWatcher watcher;
        public FileWatcherTest(string fileName, string directory, List<string> output)
        {
            FileName = fileName;
            Directory = directory;
            linesRead = output;
            watcher = new FileSystemWatcher();
            watcher.Path = directory;
            watcher.Filter = FileName;
            watcher.Changed += Watcher_Changed;
            watcher.EnableRaisingEvents = true;
            watcher.NotifyFilter =  NotifyFilters.Attributes |
                                    NotifyFilters.CreationTime |
                                    NotifyFilters.DirectoryName |
                                    NotifyFilters.FileName |
                                    NotifyFilters.LastAccess |
                                    NotifyFilters.LastWrite |
                                    NotifyFilters.Security |
                                    NotifyFilters.Size;
        }

        private void Watcher_Changed(object sender, FileSystemEventArgs e)
        {
            using (var fileStream = File.Open(Path.Combine(Directory, FileName), FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete | FileShare.Inheritable))
            {
                using (var reader = new StreamReader(fileStream))
                {
                    string line;
                    while ((line = reader.ReadLine()) != null)
                    {
                        linesRead.Add(line);
                    }
                }
            }
        }
    }
}

right now TestMethodAfterClose succeeds and TestMethodAfterFlush fails. When I use the program BareTail and wait at the breakpoint, I see that it updates the display before the file is closed. So that gives me an indication that it's possible. I don't know if it's possible in C# and I might need to import some native functions using dllimport. The problem is I don't know where to look

How do I make both tests succeed, while not using a timer?

EDIT: updated the FileWatcherTest class

Answer 1

Unfortunately Flush doesn't flush the thing you want. I find a lot articles to explain it, for example:

https://blogs.msdn.microsoft.com/alejacma/2011/03/23/filesystemwatcher-class-does-not-fire-change-events-when-notifyfilters-size-is-used/

There is a solution since .net 4, use another overload method of FileStream: Flush(bool)

var fs = writer.BaseStream as FileStream;
fs.Flush(true);

And you only give disk 10ms to react, maybe this is another problem.

Answer 2

After some searching, I discovered that the FileSystemWatcher only triggers an event after a file is closed as shown in this article. The article only mentions the date modified NotifyFilter, but in my testing I found that all Notifyfilters trigger after the file is closed and never while it's still open.

For that reason, it looks like tailing a file is only possible with a looped function that continuously monitors the file for extra rows. I used the code on this link as an example.

Here is my code working:

[TestClass]
public class FileWriteTests
{

    [TestMethod]
    public void TestMethodAfterClose_filetailing()
    {

        var currentDir = Environment.CurrentDirectory;
        var fileToMonitor = "test.txt";
        File.Delete(Path.Combine(currentDir, fileToMonitor));
        List<string> output = new List<string>();
        using (var watcherTest = new PersonalFileTail(currentDir, fileToMonitor))
        {
            watcherTest.StartTail(delegate (string line) { output.Add(line); });
            using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
            {
                writer.WriteLine($"test");
                writer.Flush();
            }
            System.Threading.Thread.Sleep(200);
            watcherTest.StopTail();
        }
        System.Threading.Thread.Sleep(10);
        Assert.AreEqual(1, output.Count);
        Assert.AreEqual("test", output[0]);
    }

    [TestMethod]
    public void TestMethodAfterFlush_filetailing()
    {
        // initiate file
        var currentDir = Environment.CurrentDirectory;
        var fileToMonitor = "test.txt";
        File.Delete(Path.Combine(currentDir, fileToMonitor));
        FileInfo info = new FileInfo(Path.Combine(currentDir, fileToMonitor));

        List<string> output = new List<string>();
        using (var watcherTest = new PersonalFileTail(currentDir, fileToMonitor))
        {
            watcherTest.StartTail(delegate (string line) { output.Add(line); });
            using (var writer = new StreamWriter(Path.Combine(currentDir, fileToMonitor), true))
            {
                try
                {
                    writer.WriteLine($"test");
                    writer.Flush();
                    System.Threading.Thread.Sleep(1000);
                    Assert.AreEqual(1, output.Count);
                    Assert.AreEqual("test", output[0]);
                }
                catch
                {
                    Assert.Fail("Test failed");
                }
            }
            watcherTest.StopTail();
        }
    }

    public class PersonalFileTail : IDisposable
    {
        private string filename;
        private string directory;
        private Task fileTailTask;
        private Action<string> handleResults;
        private volatile bool runTask;
        private long lastFilePosition;
        public string FileName
        {
            get { return Path.Combine(directory, filename); }
        }
        public PersonalFileTail(string directory, string filename)
        {
            this.directory = directory;
            this.filename = filename;
            this.runTask = false;
            lastFilePosition = 0;
        }

        public void StartTail(Action<string> handleResults)
        {
            this.handleResults = handleResults;
            runTask = true;
            fileTailTask = Task.Run(() => MonitorFileTask());
        }

        public void StopTail()
        {
            runTask = false;
            fileTailTask.Wait();
        }

        public IEnumerable<string> ReadLinesFromFile()
        {
            using (StreamReader reader = new StreamReader(new FileStream(FileName,
            FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
            {
                string line = "";
                while ((line = reader.ReadLine()) != null)
                {
                    yield return line;
                }
                lastFilePosition = reader.BaseStream.Length;
            }
        }

        public void MonitorFileTask()
        {
            StreamReader reader = null;
            FileStream stream = null;
            try
            {
                using(stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                using (reader = new StreamReader(stream))
                {
                    do
                    {
                        //if the file size has increased do something
                        if (reader.BaseStream.Length > lastFilePosition)
                        {
                            //seek to the last max offset
                            reader.BaseStream.Seek(lastFilePosition, SeekOrigin.Begin);

                            //read out of the file until the EOF
                            string line = "";
                            while ((line = reader.ReadLine()) != null)
                            {
                                handleResults(line);
                            }

                            //update the last max offset
                            lastFilePosition = reader.BaseStream.Position;
                        }
                        // sleep task for 100 ms 
                        System.Threading.Thread.Sleep(100);
                    }
                    while (runTask);
                }
            }
            catch
            {
                if (reader != null)
                    reader.Dispose();
                if (stream != null)
                    stream.Dispose();
            }
        }

        public void Dispose()
        {
            if(runTask)
            {
                runTask = false;
                fileTailTask.Wait();
            }
        }
    }
}

If anyone knows a way in which tailing can be done without using a timed function, I will accept that as the answer. Until that time, i feel that my answer is the only possible way to do this.