Poor Performance BackgroundService dequeuing from Queue

Question

I have a HostedService which inherits from BackgroundService. It loops and dequeues items from a Queue.

These items are placed on the queue via a Controller from a http request.

If I set Postman runner to fire an item every 500 miliseconds, for up to 60 items. Some early items are dequeued after seconds, but later, it can take up to 10 seconds to dequeue.

I have tried: Queue, ConcurrentQueue and BlockingCollection. All with the same results.

Any ideas? Is this a valid use case for any of the Queue types I mentioned?

Here are the implementation details:

Registration:

    services.AddSimpleInjector(_container, options =>
    {
        options.AddAspNetCore().AddControllerActivation();
        options.AddHostedService<QueuedHostedService>();
    });

The Background service:

public class QueuedHostedService : BackgroundService
{
    private readonly IApiLog _apiLog;

    public QueuedHostedService(IBackgroundTaskQueue taskQueue, 
        IApiLog apiLog)
    {
        TaskQueue = taskQueue;
        _apiLog = apiLog;
    }

    public IBackgroundTaskQueue TaskQueue { get; }

    protected override async Task ExecuteAsync(
        CancellationToken cancellationToken)
    {
        _apiLog.Log(new LogEntry("Queued Hosted Service is starting."));


        while (!cancellationToken.IsCancellationRequested)
        {
            var workItem = await TaskQueue.Dequeue(cancellationToken);

            _apiLog.Log(new LogEntry($"Dequeuing work-item: {nameof(workItem)}"));

            try
            {
                await workItem(cancellationToken);
            }
            catch (Exception exception)
            {
                _apiLog.Log(new LogEntry(exception, $"Error occurred executing {nameof(workItem)}."));
            }

        }

        _apiLog.Log(new LogEntry("Queued Hosted Service is stopping."));
    }
}

The queue:

    public class BackgroundTaskQueue : IBackgroundTaskQueue
{
    private readonly SemaphoreSlim _signal = new SemaphoreSlim(0);
    private readonly IApiLog _apiLog;
    private readonly Queue<Func<CancellationToken, Task>> _items = new Queue<Func<CancellationToken, Task>>();


    public BackgroundTaskQueue(IApiLog apiLog)
    {
        _apiLog = apiLog;
    }

    public void QueueBackgroundWorkItem(Func<CancellationToken, Task> workItem, string eventId, string correlationId)
    {
        try
        {
            if (workItem == null)
            {
                throw new ArgumentNullException(nameof(workItem));
            }

            _items.Enqueue(workItem);
            _apiLog.Log(new LogEntry($"BackgroundWorkItem has been enqueued for EventId={eventId}, CorrelationId={correlationId}"));
        }
        catch (Exception exception)
        {
            _apiLog.Log(new LogEntry(exception, exception.Message));
        }
        finally
        {
            _signal.Release();
        }
    }

    public async Task<Func<CancellationToken, Task>> Dequeue(CancellationToken cancellationToken)
    {
        await _signal.WaitAsync(cancellationToken);

        _items.TryDequeue(out var workItem);

        return workItem;
    }
}

And in the controller, the method that ultimately places items on the queue:

public void AddRequestToQueue(MyEvent myEvent, string correlationId, string userName)
    {
        if (string.IsNullOrEmpty(correlationId)) correlationId = Guid.NewGuid().ToString();

        _apiLog.Log(new LogEntry($"Adding Update Event Request to Queue. OperationType={OperationType.ToString()}, EventId={myEvent.Id}, CorrelationId={ correlationId }"));

        BackGroundTaskQueue.QueueBackgroundWorkItem(async token =>
        {
            _apiLog.Log(new LogEntry($"Update Event Request Dequeued. OperationType={OperationType.ToString()}, EventId={myEvent.Id}, CorrelationId={ correlationId }"));

            await AddSomethingToDatabase(myEvent, correlationId, userName);

            var event = _converter.Convert<SomethingElse>(myEvent);

            await SendSomethingToRabbit(event, correlationId, OperationType);
        }, myEvent.Id.ToString(), correlationId);

    }

I am seeing up to 10 seconds between the log lines:

Adding Update Event Request to Queue

and

Update Event Request Dequeued