Observable not reacting to queue changed on different thread

Question

I have the following code:

static void Main()
    {
        var holderQueue = new ConcurrentQueue<int>(GetInitialElements());

        Action<ConcurrentQueue<int>> addToQueueAction = AddToQueue;
        var observableQueue = holderQueue.ToObservable();
        IScheduler newThreadScheduler = new NewThreadScheduler();

        IObservable<Timestamped<int>> myQueueTimestamped = observableQueue.Timestamp();

        var bufferedTimestampedQueue = myQueueTimestamped.Buffer(TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(3), newThreadScheduler);

        var t = new TaskFactory();
        t.StartNew(() => addToQueueAction(holderQueue));

        using(bufferedTimestampedQueue.SubscribeOn(newThreadScheduler).Subscribe(currentQueue =>
        {
            Console.WriteLine("buffer time elapsed, current queue contents is: {0} items.", currentQueue.Count);
            foreach(var item in currentQueue)
                Console.WriteLine("item {0} at {1}", item.Value, item.Timestamp);

            Console.WriteLine("holderqueue has: {0}", currentQueue.Count);
        }))
        {
            Console.WriteLine("started observing queue");

            Console.ReadLine();
        }
    }

    private static void AddToQueue(ConcurrentQueue<int> concurrentQueue)
    {
        while(true)
        {
            var x = new Random().Next(1, 10);
            concurrentQueue.Enqueue(x);
            Console.WriteLine("added {0}", x);
            Console.WriteLine("crtcount is: {0}", concurrentQueue.Count);
            Thread.Sleep(1000);
        }
    }

    private static IEnumerable<int> GetInitialElements()
    {
        var random = new Random();
        var items = new List<int>();
        for (int i = 0; i < 10; i++)
            items.Add(random.Next(1, 10));

        return items;
    }

The intention is as follows:

The holderQueue object is populated initially with a few elements (GetInitialElements) then changed on a different thread with further elements (by the method AddToQueue), and the observable is supposed to detect this change, and react accordingly when its time is elapsed (so each 3 seconds) by executing the method in its subscription.

So in short, what I expect is to have the code in the Subscribe body to execute each 3 seconds, and show me the changes in the queue (which was changed on a different thread). Instead the Subscribe body executes only once. Why?

Thanks

Answer 1

The ToObservable method takes an IEnumerable<T> and converts it into an observable. As a result, it will take your concurrent queue and enumerate it immediately, running through all available items. The fact that you later modify the queue to add additional items has no impact on the already enumerated IEnumerable<T> that is returned from the concurrent queue's GetEnumerator() implementation.

Answer 2

As per David Pfeffer's answer, just using .ToObserverable() will not get you what you need.

However, when I look at your code, I see several things:

1. You are using a NewThreadScheduler
2. You are adding to the queue via a Task
3. You are using a ConcurrentQueue<T>

I think you can achieve what you set out to do here if you just change a few things. First I think you are actually looking for a BlockingCollection<T>. I know it seems unlikely, but you can get it to act just like a thread-safe queue.

Next you are already dedicating a thread to doing something with the NewThreadScheduler, why not make that do the polling/pulling from the queue?

Lastly, if you use the BlockingCollection<T>.GetConsumingEnumerable(CancellationToken) method, you actually can go back and use that .ToObservable() method!

So lets look at the rewritten code:

static void Main()
{
    //The processing thread. I try to set the the thread name as these tend to be long lived. This helps logs and debugging.
    IScheduler newThreadScheduler = new NewThreadScheduler(ts=>{
        var t =  new Thread(ts);
        t.Name = "QueueReader";
        t.IsBackground = true;
        return t;
    });

    //Provide the ability to cancel our work
    var cts = new CancellationTokenSource();

    //Use a BlockingCollection<T> instead of a ConcurrentQueue<T>
    var holderQueue = new BlockingCollection<int>();
    foreach (var element in GetInitialElements())
    {
        holderQueue.Add(element);
    }

    //The Action that periodically adds items to the queue. Now has cancellation support
    Action<BlockingCollection<int>,CancellationToken> addToQueueAction = AddToQueue;
    var tf = new TaskFactory();
    tf.StartNew(() => addToQueueAction(holderQueue, cts.Token));

    //Get a consuming enumerable. MoveNext on this will remove the item from the BlockingCollection<T> effectively making it a queue. 
    //  Calling MoveNext on an empty queue will block until cancelled or an item is added.
    var consumingEnumerable = holderQueue.GetConsumingEnumerable(cts.Token);

    //Now we can make this Observable, as the underlying IEnumerbale<T> is a blocking consumer.
    //  Run on the QueueReader/newThreadScheduler thread.
    //  Use CancelationToken instead of IDisposable for single method of cancellation.
    consumingEnumerable.ToObservable(newThreadScheduler)
        .Timestamp()
        .Buffer(TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(3), newThreadScheduler)
        .Subscribe(buffer =>
            {
                Console.WriteLine("buffer time elapsed, current queue contents is: {0} items.", buffer.Count);
                foreach(var item in buffer)
                    Console.WriteLine("item {0} at {1}", item.Value, item.Timestamp);

                Console.WriteLine("holderqueue has: {0}", holderQueue.Count);
            },
            cts.Token);


    Console.WriteLine("started observing queue");

    //Run until [Enter] is pressed by user.
    Console.ReadLine();

    //Cancel the production of values, the wait on the consuming enumerable and the subscription.
    cts.Cancel();
    Console.WriteLine("Cancelled");
}

private static void AddToQueue(BlockingCollection<int> input, CancellationToken cancellationToken)
{
    while(!cancellationToken.IsCancellationRequested)
    {
        var x = new Random().Next(1, 10);
        input.Add(x);
        Console.WriteLine("added '{0}'. Count={1}", x, input.Count);
        Thread.Sleep(1000);
    }
}

private static IEnumerable<int> GetInitialElements()
{
    var random = new Random();
    var items = new List<int>();
    for (int i = 0; i < 10; i++)
        items.Add(random.Next(1, 10));

    return items;
}

Now I think you will get the results you were expecting:

added '9'. Count=11
started observing queue
added '4'. Count=1
added '8'. Count=1
added '3'. Count=1
buffer time elapsed, current queue contents is: 14 items.
item 9 at 25/01/2015 22:25:35 +00:00
item 5 at 25/01/2015 22:25:35 +00:00
item 5 at 25/01/2015 22:25:35 +00:00
item 9 at 25/01/2015 22:25:35 +00:00
item 7 at 25/01/2015 22:25:35 +00:00
item 6 at 25/01/2015 22:25:35 +00:00
item 2 at 25/01/2015 22:25:35 +00:00
item 2 at 25/01/2015 22:25:35 +00:00
item 9 at 25/01/2015 22:25:35 +00:00
item 3 at 25/01/2015 22:25:35 +00:00
item 9 at 25/01/2015 22:25:35 +00:00
item 4 at 25/01/2015 22:25:36 +00:00
item 8 at 25/01/2015 22:25:37 +00:00
item 3 at 25/01/2015 22:25:38 +00:00
holderqueue has: 0
added '7'. Count=1
added '2'. Count=1
added '5'. Count=1
buffer time elapsed, current queue contents is: 3 items.
item 7 at 25/01/2015 22:25:39 +00:00
item 2 at 25/01/2015 22:25:40 +00:00
item 5 at 25/01/2015 22:25:41 +00:00
holderqueue has: 0
Cancelled