I'm doing batching of messages in stream want to do it by 3 conditions:
- if batcher can't add incoming message (for whatever internal logic of batcher)
- if there were messages and then no messages within X seconds (basically what throttle does)
- if there is continious stream of messages more often then every X seconds, then after Y seconds still close batch (cap on throttle)
I need to be able to change X and Y seconds in runtime without losing current batch (doesn't matter if it is closed immediately on config change or by closing conditions). Condition function and batch process function should not run in parallel threads.
I'm using Rx-Main 2.2.5. So far I came up with solution below and it seems to work, but I think there may be much simpler solution with reactive extensions? Also with this solution capTimer doesn't restart if closing condition is "batcher can't add this message".
Extension:
public static class ObservableExtensions
{
public static IDisposable ConditionalCappedThrottle<T>(this IObservable<T> observable,
int throttleInSeconds,
int capTimeInSeconds,
Func<T, bool> conditionFunc,
Action capOrThrottleAction,
Action<T, Exception> onException,
T fakeInstance = default(T))
{
Subject<T> buffer = new Subject<T>();
var capTimerObservable = new Subject<long>();
var throttleTimerObservable = observable.Throttle(TimeSpan.FromSeconds(throttleInSeconds)).Select(c => 1L);
IDisposable maxBufferTimer = null;
var bufferTicks = observable
.Do(c =>
{
if (maxBufferTimer == null)
maxBufferTimer = Observable.Timer(TimeSpan.FromSeconds(capTimeInSeconds))
.Subscribe(x => capTimerObservable.OnNext(1));
})
.Buffer(() => Observable.Amb(
capTimerObservable
.Do(c => Console.WriteLine($"{DateTime.Now:mm:ss.fff} cap time tick closing buffer")),
throttleTimerObservable
.Do(c => Console.WriteLine($"{DateTime.Now:mm:ss.fff} throttle time tick closing buffer"))
))
.Do(c =>
{
maxBufferTimer?.Dispose();
maxBufferTimer = null;
})
.Where(changes => changes.Any())
.Subscribe(dataChanges =>
{
buffer.OnNext(fakeInstance);
});
var observableSubscriber = observable.Merge(buffer)
.Subscribe(subject =>
{
try
{
if (!subject.Equals(fakeInstance)) {
if (conditionFunc(subject))
return;
Console.WriteLine($"{DateTime.Now:mm:ss.fff} condition false closing buffer");
maxBufferTimer?.Dispose();
}
capOrThrottleAction();
if (!subject.Equals(fakeInstance))
conditionFunc(subject);
}
catch (Exception ex)
{
onException(subject, ex);
}
});
return new CompositeDisposable(maxBufferTimer, observableSubscriber);
}
}
And usage:
class Program
{
static void Main(string[] args)
{
messagesObs = new Subject<Message>();
new Thread(() =>
{
while (true)
{
Thread.Sleep(random.Next(3) * 1000);
(messagesObs as Subject<Message>).OnNext(new Message());
}
}).Start();
while (true)
{
throttleTime = random.Next(8) + 2;
maxThrottleTime = random.Next(10) + 20;
Console.WriteLine($"{DateTime.Now:mm:ss.fff} resubscribing with {throttleTime} - {maxThrottleTime}");
Subscribe();
Thread.Sleep((random.Next(10) + 60) * 1000);
}
}
static Random random = new Random();
static int throttleTime = 3;
static int maxThrottleTime = 10;
static IDisposable messagesSub;
static IObservable<Message> messagesObs;
static void Subscribe()
{
messagesSub?.Dispose();
BatchProcess();
messagesSub = messagesObs.ConditionalCappedThrottle(
throttleTime,
maxThrottleTime,
TryAddToBatch,
BatchProcess,
(msg, ex) => { },
new FakeMessage());
}
static bool TryAddToBatch(Message msg)
{
if (random.Next(100) > 85)
{
Console.WriteLine($"{DateTime.Now:mm:ss.fff} can't add to batch");
return false;
}
else
{
Console.WriteLine($"{DateTime.Now:mm:ss.fff} added to batch");
return true;
}
}
static void BatchProcess()
{
Console.WriteLine($"{DateTime.Now:mm:ss.fff} Processing");
Thread.Sleep(2000);
Console.WriteLine($"{DateTime.Now:mm:ss.fff} Done Processing");
}
}
public class Message { }
public class FakeMessage : Message { }
Tests I want to work:
public class Test
{
static Subject<Base> sub = new Subject<Base>();
static int maxTime = 19;
static int throttleTime = 6;
// Batcher.Process must be always waited before calling any next Batcher.Add
static void MaxTime()
{
// foreach on next Batcher.Add must be called
sub.OnNext(new A());
Thread.Sleep(6 * 1000 - 100);
sub.OnNext(new A());
Thread.Sleep(6 * 1000 - 100);
sub.OnNext(new A());
Thread.Sleep(6 * 1000 - 100);
// Process must be called after 19 seconds = maxTime
}
static void Throttle()
{
// foreach on next Batcher.Add must be called
sub.OnNext(new A());
Thread.Sleep(6 * 1000 - 100);
sub.OnNext(new A());
Thread.Sleep(6 * 1000 - 100);
// Process must be called after 5.9+5.9+6 seconds = throttleTime
}
static void Condition()
{
// foreach on next Batcher.Add must be called
sub.OnNext(new A());
Thread.Sleep(6 * 1000 - 100);
sub.OnNext(new B());
// Process must be called because Batcher.Add will return false
// Batcher.Add(B) must be called after Process
}
static void MaxTimeOrThorttleNotTickingRandomly()
{
sub.OnNext(new A());
// Process called by throttle condition in 6 seconds
Thread.Sleep(1000 * 100);
// Process is not called for remaining 94 seconds
sub.OnNext(new A());
// Process called by throttle condition in 6 seconds
}
static void Resub()
{
sub.OnNext(new A());
sub.OnNext(new A());
sub.OnNext(new A());
sub.OnNext(new A());
sub.OnNext(new A());
maxTime = 15;
throttleTime = 3;
// Process is called
// Resubs with new timinig conditions
sub.OnNext(new A());
// Process called by throttle condition in 3 seconds
}
}
public class Batcher
{
private Type batchingType;
public bool Add(Base element)
{
if (batchingType == null || element.GetType() == batchingType)
{
batchingType = element.GetType();
return true;
}
return false;
}
public void Process()
{
batchingType = null;
}
}
public class Base{}
public class A : Base { }
public class B : Base { }
