How can I route Observable values to different Subscribers?

Question

This is all just pseudo code...

Ok here is my scenario, I have an incoming data stream that gets parsed into packets.

I have an IObservable<Packets> Packets

Each packet has a Packet ID, i.e. 1, 2, 3, 4

I want to create observables that only receive a specific ID.

so I do:

Packets.Where(p=>p.Id == 1)

for example... that gives me an IObservable<Packets> that only gives me packets of Id 1.

I may have several of these:

Packets.Where(p=>p.Id == 2)
Packets.Where(p=>p.Id == 3)
Packets.Where(p=>p.Id == 4)
Packets.Where(p=>p.Id == 5)

This essentially works, but the more Ids I want to select the more processing is required, i.e. the p=>p.Id will be run for every single Id, even after a destination Observable has been found.

How can I do the routing so that it is more efficient, something analogous:

Dictionary listeners;

listeners.GetValue(packet.Id).OnDataReceived(packet)

so that as soon as an id is picked up by one of my IObservables, then none of the others get to see it?

Updates

Added an extension based on Lee Campbell's groupby suggestion:

public static class IObservableExtensions
{
    class RouteTable<TKey, TSource>
    {
        public static readonly ConditionalWeakTable<IObservable<TSource>, IObservable<IGroupedObservable<TKey, TSource>>> s_routes = new ConditionalWeakTable<IObservable<TSource>, IObservable<IGroupedObservable<TKey, TSource>>>();
    }

    public static IObservable<TSource> Route<TKey, TSource>(this IObservable<TSource> source, Func<TSource, TKey> selector, TKey id)
    {
        var grouped = RouteTable<TKey, TSource>.s_routes.GetValue(source, s => s.GroupBy(p => selector(p)).Replay().RefCount());
        return grouped.Where(e => e.Key.Equals(id)).SelectMany(e => e);
    }
}

It would be used like this:

Subject<Packet> packetSubject = new Subject<Packet>();

        var packets = packetSubject.AsObservable();

        packets.Route((p) => p.Id, 5).Subscribe((p) =>
        {
            Console.WriteLine("5");
        });

        packets.Route((p) => p.Id, 4).Subscribe((p) =>
        {
            Console.WriteLine("4");
        });

        packets.Route((p) => p.Id, 3).Subscribe((p) =>
        {
            Console.WriteLine("3");
        });

        packetSubject.OnNext(new Packet() { Id = 1 });
        packetSubject.OnNext(new Packet() { Id = 2 });
        packetSubject.OnNext(new Packet() { Id = 3 });
        packetSubject.OnNext(new Packet() { Id = 4 });
        packetSubject.OnNext(new Packet() { Id = 5 });
        packetSubject.OnNext(new Packet() { Id = 4 });
        packetSubject.OnNext(new Packet() { Id = 3 });

output is: 3, 4, 5, 4, 3

It only checks the Id for every group when it sees a new packet id.

Answer 1

I would suggest looking at GroupBy and then checking if there is a performance pay off. I assume there is, but is it significant?

Packets.GroupBy(p=>p.Id)

Example code with tests on how to use GroupBy as a type of router

var scheduler = new TestScheduler();
var source = scheduler.CreateColdObservable(
    ReactiveTest.OnNext(100, 1),
    ReactiveTest.OnNext(200, 2),
    ReactiveTest.OnNext(300, 3),
    ReactiveTest.OnNext(400, 4),
    ReactiveTest.OnNext(500, 5),
    ReactiveTest.OnNext(600, 6),
    ReactiveTest.OnNext(700, 7),
    ReactiveTest.OnNext(800, 8),
    ReactiveTest.OnNext(900, 9),
    ReactiveTest.OnNext(1000, 10),
    ReactiveTest.OnNext(1100, 11)
    );

var router = source.GroupBy(i=>i%4)
    .Publish()
    .RefCount();

var zerosObserver = scheduler.CreateObserver<int>();
router.Where(grp=>grp.Key == 0)
    .Take(1)
    .SelectMany(grp=>grp)
    .Subscribe(zerosObserver);

var onesObserver = scheduler.CreateObserver<int>();
router.Where(grp => grp.Key == 1)
    .Take(1)
    .SelectMany(grp => grp)
    .Subscribe(onesObserver);

var twosObserver = scheduler.CreateObserver<int>();
router.Where(grp => grp.Key == 2)
        .Take(1)
        .SelectMany(grp => grp)
        .Subscribe(twosObserver);

var threesObserver = scheduler.CreateObserver<int>();
router.Where(grp => grp.Key == 3)
        .Take(1)
        .SelectMany(grp => grp)
        .Subscribe(threesObserver);

scheduler.Start();

ReactiveAssert.AreElementsEqual(new[] { ReactiveTest.OnNext(400, 4), ReactiveTest.OnNext(800, 8)}, zerosObserver.Messages);
ReactiveAssert.AreElementsEqual(new[] { ReactiveTest.OnNext(100, 1), ReactiveTest.OnNext(500, 5), ReactiveTest.OnNext(900, 9)}, onesObserver.Messages);
ReactiveAssert.AreElementsEqual(new[] { ReactiveTest.OnNext(200, 2), ReactiveTest.OnNext(600, 6), ReactiveTest.OnNext(1000, 10) }, twosObserver.Messages);
ReactiveAssert.AreElementsEqual(new[] { ReactiveTest.OnNext(300, 3), ReactiveTest.OnNext(700, 7), ReactiveTest.OnNext(1100, 11)}, threesObserver.Messages);

Answer 2

Here's an operator that I wrote quite some time ago, but I think it does what you're after. I still think that a simple .Where is probably better - even with multiple subscribers.

Nevertheless, I wanted a .ToLookup for observables that operates like the same operator for enumerables.

It isn't memory efficient, but it implements IDisposable so that it can be cleaned up afterwards. It also isn't thread-safe so a little hardening might be required.

Here it is:

public static class ObservableEx
{
    public static IObservableLookup<K, V> ToLookup<T, K, V>(this IObservable<T> source, Func<T, K> keySelector, Func<T, V> valueSelector, IScheduler scheduler)
    {
        return new ObservableLookup<T, K, V>(source, keySelector, valueSelector, scheduler);
    }

    internal class ObservableLookup<T, K, V> : IDisposable, IObservableLookup<K, V>
    {
        private IDisposable _subscription = null; 
        private readonly Dictionary<K, ReplaySubject<V>> _lookups = new Dictionary<K, ReplaySubject<V>>();

        internal ObservableLookup(IObservable<T> source, Func<T, K> keySelector, Func<T, V> valueSelector, IScheduler scheduler)
        {
            _subscription = source.ObserveOn(scheduler).Subscribe(
                t => this.GetReplaySubject(keySelector(t)).OnNext(valueSelector(t)),
                ex => _lookups.Values.ForEach(rs => rs.OnError(ex)),
                () => _lookups.Values.ForEach(rs => rs.OnCompleted()));
        }

        public void Dispose()
        {
            if (_subscription != null)
            {
                _subscription.Dispose();
                _subscription = null;
                _lookups.Values.ForEach(rs => rs.Dispose());
                _lookups.Clear();
            }
        }

        private ReplaySubject<V> GetReplaySubject(K key)
        {
            if (!_lookups.ContainsKey(key))
            {
                _lookups.Add(key, new ReplaySubject<V>());
            }
            return _lookups[key];
        }

        public IObservable<V> this[K key]
        {
            get
            {
                if (_subscription == null) throw new ObjectDisposedException("ObservableLookup");
                return this.GetReplaySubject(key).AsObservable();
            }
        }
    }
}

public interface IObservableLookup<K, V> : IDisposable
{
    IObservable<V> this[K key] { get; }
}

You would use it like this:

IObservable<Packets> Packets = ...

IObservableLookup<int, Packets> lookup = Packets.ToLookup(p => p.Id, p => p, Scheduler.Default);

lookup[1].Subscribe(p => { });
lookup[2].Subscribe(p => { });
// etc

The nice thing with this is that you can subscribe to values by key before a value with that key has been produced by the source observable.

Don't forget to call lookup.Dispose() when done to clean up the resources.

Answer 3

You can use GroupBy to split the data. I would suggest you set up all subscriptions first and then activate your source. Doing so would result in one huge nested GroupBy query, but it is also possible to multi-cast your groups and subscribe to them individually. I wrote a small helper utility to do so below.

Because you still might want to add new routes after the source has been activated (done trough Connect), we use Replay to replay the groups. Replay is also a multi-cast operator so we wont need Publish to multi-cast.

public sealed class RouteData<TKey, TSource>
{
    private IConnectableObservable<IGroupedObservable<TKey, TSource>> myRoutes;

    public RouteData(IObservable<TSource> source, Func<TSource, TKey> keySelector)
    {
        this.myRoutes = source.GroupBy(keySelector).Replay();
    }

    public IDisposable Connect()
    {
        return this.myRoutes.Connect();
    }

    public IObservable<TSource> Get(TKey id)
    {
        return myRoutes.FirstAsync(e => e.Key.Equals(id)).Merge();
    }
}

public static class myExtension
{
    public static RouteData<TKey, TSource> RouteData<TKey, TSource>(this IObservable<TSource> source, Func<TSource, TKey> keySelector)
    {
        return new RouteData<TKey, TSource>(source, keySelector);
    }
}

Example usage:

public class myPackage
{
    public int Id;

    public myPackage(int id)
    {
        this.Id = id;
    }
}

class program
{
    static void Main()
    {
        var source = new[] { 0, 1, 2, 3, 4, 5, 4, 3 }.ToObservable().Select(i => new myPackage(i));
        var routes = source.RouteData(e => e.Id);
        var subscription = new CompositeDisposable(
            routes.Get(5).Subscribe(Console.WriteLine),
            routes.Get(4).Subscribe(Console.WriteLine),
            routes.Get(3).Subscribe(Console.WriteLine),
            routes.Connect());
        Console.ReadLine();
    }
}

Answer 4

You may want to consider writing a custom IObserver that does your bidding. I've included an example below.

void Main()
{
    var source = Observable.Range(1, 10);
    var switcher = new Switch<int, int>(i => i % 3);
    switcher[0] = Observer.Create<int>(val => Console.WriteLine($"{val} Divisible by three"));
    source.Subscribe(switcher);
}

class Switch<TKey,TValue> : IObserver<TValue>
{
    private readonly IDictionary<TKey, IObserver<TValue>> cases;
    private readonly Func<TValue,TKey> idExtractor;

    public IObserver<TValue> this[TKey decision]
    {
        get
        {
            return cases[decision];
        }
        set
        {
            cases[decision] = value;
        }
    }

    public Switch(Func<TValue,TKey> idExtractor)
    {
        this.cases = new Dictionary<TKey, IObserver<TValue>>();
        this.idExtractor = idExtractor;
    }

    public void OnNext(TValue next)
    {
        IObserver<TValue> nextCase;
        if (cases.TryGetValue(idExtractor(next), out nextCase))
        {
            nextCase.OnNext(next);
        }
    }

    public void OnError(Exception e)
    {
        foreach (var successor in cases.Values)
        {
            successor.OnError(e);
        }
    }

    public void OnCompleted()
    {
        foreach (var successor in cases.Values)
        {
            successor.OnCompleted();
        }
    }
}

You would obviously need to implement idExtractor to extract the ids from your packet.