We're running RavenDB locally in docker for automated integration tests and want it to not timeout quickly, wait for stale indices before query, wait for them after writing, just do whatever it takes to not have those tests flake.
RavenDB image: ravendb/ravendb:5.3.104-ubuntu.20.04-x64
C# Client: RavenDB.Client Version="5.4.1"
The problem: tests fail sporadically on stale indices in miliseconds
event though the DocumentStore is configured to await for stale indices for a long time.
Here's everything that I tried so far and have in this test DocumentStore setup
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(10);
private IDocumentStore CreateDocumentStore()
{
var store = new DocumentStore()
{
Urls = new[] { RavenUrl },
Database = TestDatabaseName,
Conventions = new DocumentConventions
{
WaitForIndexesAfterSaveChangesTimeout = Constants.DefaultTimeout,
WaitForNonStaleResultsTimeout = Constants.DefaultTimeout,
}
};
ConfigureDocumentStoreConventions(store);
store.OnBeforeQuery += (_, beforeQueryExecutedArgs) =>
{
beforeQueryExecutedArgs.QueryCustomization.WaitForNonStaleResults(Constants.DefaultTimeout);
};
store.OnSessionCreated += (_, b) =>
{
b.Session.WaitForIndexesAfterSaveChanges(Constants.DefaultTimeout);
// inspired by
// https://stackoverflow.com/a/10317242/3608449
while (b.Session.DocumentStore.Maintenance.ForDatabase(TestDatabaseName)
.Send(new GetStatisticsOperation()).StaleIndexes.Length != 0)
{
Thread.Sleep(500);
}
};
store.Initialize();
_newDatabaseWasCreated = EnsureDatabaseExists(store, TestDatabaseName);
if (_newDatabaseWasCreated)
{
SeedDefaultDocuments(store);
Thread.Sleep(2000);
}
return store;
}
private static void ConfigureDocumentStoreConventions( IDocumentStore store)
{
// not sure if any of this is relevant but didn't want to leave it out in case it is
var inflector = new Inflector.Inflector(new CultureInfo("en"));
store.Conventions.FindCollectionName = type =>
{
return type.Name.EndsWith(EntityPostfix)
? inflector.Pluralize(type.Name.Replace(EntityPostfix, string.Empty))
: DocumentConventions.DefaultGetCollectionName(type);
};
store.OnBeforeStore += (sender, args) =>
{
if (args.Entity is BaseStorageEntity entity)
{
entity.Id = Guid.Parse(args.DocumentId);
}
if (args.Entity is IRavenDbStoreOperations operations)
{
operations.OnBeforeStore();
}
};
store.OnAfterConversionToEntity += (sender, args) =>
{
if (args.Entity is BaseStorageEntity entity)
{
entity.Id = Guid.Parse(args.Id);
}
};
var defaultDocumentIdGenerator = store.Conventions.AsyncDocumentIdGenerator;
store.Conventions.AsyncDocumentIdGenerator = (dbName, ent) =>
{
if (ent is BaseStorageEntity entity)
{
entity.Id = entity.Id == Guid.Empty ? Guid.NewGuid() : entity.Id;
return Task.FromResult(entity.Id.ToString());
}
return defaultDocumentIdGenerator(dbName, ent);
};
var defaultIdentityProperty = store.Conventions.FindIdentityProperty;
store.Conventions.FindIdentityProperty = info =>
{
if (info.DeclaringType.IsAssignableFrom(typeof(BaseStorageEntity)))
return false;
return defaultIdentityProperty(info);
};
}
The exception from the failing test that failed after 339ms. How do I make DocumentStore respect these configured timeouts?
System.OperationCanceledException: The operation was canceled.
at System.Threading.CancellationToken.ThrowOperationCanceledException()
at Sparrow.Json.JsonContextPoolBase`1.AllocateOperationContext(T& context) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Sparrow\\Json\\JsonContextPoolBase.cs:line 76
at Raven.Server.Documents.Indexes.Index.IsStale(QueryOperationContext queryContext, Nullable`1 cutoff, List`1 stalenessReasons) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Documents\\Indexes\\Index.cs:line 1192
at Raven.Server.Documents.Handlers.BatchHandler.WaitForIndexesAsync(DocumentsContextPool contextPool, DocumentDatabase database, TimeSpan timeout, List`1 specifiedIndexesQueryString, Boolean throwOnTimeout, String lastChangeVector, Int64 lastTombstoneEtag, HashSet`1 modifiedCollections) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Documents\\Handlers\\BatchHandler.cs:line 404
at Raven.Server.Documents.Handlers.BatchHandler.BulkDocs() in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Documents\\Handlers\\BatchHandler.cs:line 142
at Raven.Server.Routing.RequestRouter.HandlePath(RequestHandlerContext reqCtx) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Routing\\RequestRouter.cs:line 365
at Raven.Server.RavenServerStartup.RequestHandler(HttpContext context) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\RavenServerStartup.cs:line 240","data":{},"innerException":{"ClassName":"System.OperationCanceledException","Message":"System.OperationCanceledException: The operation was canceled.
at System.Threading.CancellationToken.ThrowOperationCanceledException()
at Sparrow.Json.JsonContextPoolBase`1.AllocateOperationContext(T& context) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Sparrow\\Json\\JsonContextPoolBase.cs:line 76
at Raven.Server.Documents.Indexes.Index.IsStale(QueryOperationContext queryContext, Nullable`1 cutoff, List`1 stalenessReasons) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Documents\\Indexes\\Index.cs:line 1192
at Raven.Server.Documents.Handlers.BatchHandler.WaitForIndexesAsync(DocumentsContextPool contextPool, DocumentDatabase database, TimeSpan timeout, List`1 specifiedIndexesQueryString, Boolean throwOnTimeout, String lastChangeVector, Int64 lastTombstoneEtag, HashSet`1 modifiedCollections) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Documents\\Handlers\\BatchHandler.cs:line 404
at Raven.Server.Documents.Handlers.BatchHandler.BulkDocs() in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Documents\\Handlers\\BatchHandler.cs:line 142
at Raven.Server.Routing.RequestRouter.HandlePath(RequestHandlerContext reqCtx) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Routing\\RequestRouter.cs:line 365
at Raven.Server.RavenServerStartup.RequestHandler(HttpContext context) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\RavenServerStartup.cs:line 240","Data":null,"InnerException":null,"HelpURL":null,"StackTraceString":null,"RemoteStackTraceString":null,"RemoteStackIndex":0,"ExceptionMethod":null,"HResult":-2146233029,"Source":null,"WatsonBuckets":null},"helpLink":null,"source":"Raven.Client","hResult":-2146233088,"stackTrace":" at Raven.Client.Exceptions.ExceptionDispatcher.Throw(JsonOperationContext context, HttpResponseMessage response, Action`1 additionalErrorInfo) in C:\\Builds\\RavenDB-Stable-5.4\\54001\\src\\Raven.Client\\Exceptions\\ExceptionDispatcher.cs:line 119
at Raven.Client.Http.RequestExecutor.HandleUnsuccessfulResponse[TResult](ServerNode chosenNode, Nullable`1 nodeIndex, JsonOperationContext context, RavenCommand`1 command, HttpRequestMessage request, HttpResponseMessage response, String url, SessionInfo sessionInfo, Boolean shouldRetry, CancellationToken token) in C:\\Builds\\RavenDB-Stable-5.4\\54001\\src\\Raven.Client\\Http\\RequestExecutor.cs:line 1457
at Raven.Client.Http.RequestExecutor.ExecuteAsync[TResult](ServerNode chosenNode, Nullable`1 nodeIndex, JsonOperationContext context, RavenCommand`1 command, Boolean shouldRetry, SessionInfo sessionInfo, CancellationToken token) in C:\\Builds\\RavenDB-Stable-5.4\\54001\\src\\Raven.Client\\Http\\RequestExecutor.cs:line 903
at Raven.Client.Http.RequestExecutor.ExecuteAsync[TResult](ServerNode chosenNode, Nullable`1 nodeIndex, JsonOperationContext context, RavenCommand`1 command, Boolean shouldRetry, SessionInfo sessionInfo, CancellationToken token) in C:\\Builds\\RavenDB-Stable-5.4\\54001\\src\\Raven.Client\\Http\\RequestExecutor.cs:line 903
at Raven.Client.Documents.Session.AsyncDocumentSession.SaveChangesAsync(CancellationToken token) in C:\\Builds\\RavenDB-Stable-5.4\\54001\\src\\Raven.Client\\Documents\\Session\\AsyncDocumentSession.cs:line 171
at VendorCenter.Relationships.ItSystemRelationship.Repository.ItSystemRelationshipRepository.Update(Guid entityId, Action`1 applyChanges) in /home/runner/work/Cirrus/Cirrus/backend/src/VendorCenter/VendorCenter/VendorCenter/Relationships/ItSystemRelationship/Repository/ItSystemRelationshipRepository.cs:line 83
at VendorCenter.Relationships.ItSystemRelationship.Features.Commands.UpdateItSystemRelationshipConsequenceCommandHandler.Handle(UpdateItSystemRelationshipConsequenceCommand request, CancellationToken cancellationToken) in /home/runner/work/Cirrus/Cirrus/backend/src/VendorCenter/VendorCenter/VendorCenter/Relationships/ItSystemRelationship/Features/Commands/UpdateItSystemRelationshipConsequenceCommand.cs:line 36
at MediatR.Pipeline.RequestExceptionProcessorBehavior`2.Handle(TRequest request, RequestHandlerDelegate`1 next, CancellationToken cancellationToken)
at MediatR.Pipeline.RequestExceptionProcessorBehavior`2.Handle(TRequest request, RequestHandlerDelegate`1 next, CancellationToken cancellationToken)
at MediatR.Pipeline.RequestExceptionActionProcessorBehavior`2.Handle(TRequest request, RequestHandlerDelegate`1 next, CancellationToken cancellationToken)
at MediatR.Pipeline.RequestExceptionActionProcessorBehavior`2.Handle(TRequest request, RequestHandlerDelegate`1 next, CancellationToken cancellationToken)
at MediatR.Pipeline.RequestPostProcessorBehavior`2.Handle(TRequest request, RequestHandlerDelegate`1 next, CancellationToken cancellationToken)
at MediatR.Pipeline.RequestPreProcessorBehavior`2.Handle(TRequest request, RequestHandlerDelegate`1 next, CancellationToken cancellationToken)
at CC.Core.Shared.Mediator.Mediator.Pipelines.LoggingBehavior`2.Handle(TRequest request, RequestHandlerDelegate`1 next, CancellationToken cancellationToken) in /home/runner/work/Cirrus/Cirrus/backend/CC.Core.Shared/Mediator/Mediator/Pipelines/LoggingBehaviour.cs:line 22
at CC.Core.Shared.Mediator.Validation.ValidationBehavior`2.Handle(TRequest request, RequestHandlerDelegate`1 next, CancellationToken cancellationToken) in /home/runner/work/Cirrus/Cirrus/backend/CC.Core.Shared/Mediator/Validation/ValidationBehavior.cs:line 33
at VendorCenter.Relationships.ItSystemRelationship.ItSystemRelationshipsController.UpdateRelationshipConsequence(Guid relationshipId, Consequence consequence) in /home/runner/work/Cirrus/Cirrus/backend/src/VendorCenter/VendorCenter/VendorCenter/Relationships/ItSystemRelationship/ItSystemRelationshipsController.cs:line 88
at lambda_method4045(Closure, Object)
at Microsoft.AspNetCore.Mvc.Infrastructure.ActionMethodExecutor.AwaitableObjectResultExecutor.Execute(ActionContext actionContext, IActionResultTypeMapper mapper, ObjectMethodExecutor executor, Object controller, Object[] arguments)
at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.<InvokeActionMethodAsync>g__Awaited|12_0(ControllerActionInvoker invoker, ValueTask`1 actionResultValueTask)
at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.<InvokeNextActionFilterAsync>g__Awaited|10_0(ControllerActionInvoker invoker, Task lastTask, State next, Scope scope, Object state, Boolean isCompleted)
at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.Rethrow(ActionExecutedContextSealed context)
at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.Next(State& next, Scope& scope, Object& state, Boolean& isCompleted)
at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.<InvokeInnerFilterAsync>g__Awaited|13_0(ControllerActionInvoker invoker, Task lastTask, State next, Scope scope, Object state, Boolean isCompleted)
at Microsoft.AspNetCore.Mvc.Infrastructure.ResourceInvoker.<InvokeNextExceptionFilterAsync>g__Awaited|26_0(ResourceInvoker invoker, Task lastTask, State next, Scope scope, Object state, Boolean isCompleted)"},"detail":"System.OperationCanceledException: The operation was canceled.
at System.Threading.CancellationToken.ThrowOperationCanceledException()
at Sparrow.Json.JsonContextPoolBase`1.AllocateOperationContext(T& context) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Sparrow\\Json\\JsonContextPoolBase.cs:line 76
at Raven.Server.Documents.Indexes.Index.IsStale(QueryOperationContext queryContext, Nullable`1 cutoff, List`1 stalenessReasons) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Documents\\Indexes\\Index.cs:line 1192
at Raven.Server.Documents.Handlers.BatchHandler.WaitForIndexesAsync(DocumentsContextPool contextPool, DocumentDatabase database, TimeSpan timeout, List`1 specifiedIndexesQueryString, Boolean throwOnTimeout, String lastChangeVector, Int64 lastTombstoneEtag, HashSet`1 modifiedCollections) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Documents\\Handlers\\BatchHandler.cs:line 404
at Raven.Server.Documents.Handlers.BatchHandler.BulkDocs() in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Documents\\Handlers\\BatchHandler.cs:line 142
at Raven.Server.Routing.RequestRouter.HandlePath(RequestHandlerContext reqCtx) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\Routing\\RequestRouter.cs:line 365
at Raven.Server.RavenServerStartup.RequestHandler(HttpContext context) in C:\\Builds\\RavenDB-Stable-5.3\\53029\\src\\Raven.Server\\RavenServerStartup.cs:line 240
