Symfony Messenger consumer pool

Question

I'm not sure this is something possible: to have a pool of messenger consumers for the same queue ?

I've tried to use the Redis consumer= options but that's not it.

Maybe a "pool" middleware could do some routing to specific transports?

Answer 1

Yes, you use multiple consumers for one queue. Symfony Messenger is designed to have multiple consumers. The example configuration from the Supervisor part of the documentation already has 2 instances:

Supervisor configuration files typically live in a /etc/supervisor/conf.d directory. For example, you can create a new messenger-worker.conf file there to make sure that 2 instances of messenger:consume are running at all times:

(..)

So you can just run bin/console messenger:consume async multiple times and in most cases, this will work without additional configuration. There is a warning about using the Redis Transport and multiple workers:

If you use the Redis Transport, note that each worker needs a unique consumer name to avoid the same message being handled by multiple workers. One way to achieve this is to set an environment variable in the Supervisor configuration file, which you can then refer to in messenger.yaml (see Redis section above):

environment=MESSENGER_CONSUMER_NAME=%(program_name)s_%(process_num)02d

Answer 2

I've come with a solution based on multiple sub queues/classes with a main spooler (sync transport).

<?php

declare(strict_types=1);

namespace App\MessageHandler;

use App\Message\Job;
use App\Message\JobInterface;
use App\Message\JobPool00;
use App\Message\JobPool01;
use App\Message\JobPool02;
// repeat...
use App\Message\JobPoolInterface;
use m2mQL\Job\JobManager;
use m2mQL\Service\Semaphore;
use Psr\Log\LoggerInterface;
use Symfony\Component\Messenger\Handler\MessageHandlerInterface;
use Symfony\Component\Messenger\MessageBusInterface;

class JobHandler implements MessageHandlerInterface
{
    protected $pool;
    protected $logger;

    public function __construct(LoggerInterface $logger, int $pool = 0)
    {
        $this->pool = $pool;
        $this->logger = $logger;
    }

    protected JobManager $jm;

    /**
     * @required
     */
    public function setJobManager(JobManager $jm)
    {
        $this->jm = $jm;
    }

    protected Semaphore $semaphore;

    /**
     * @required
     */
    public function setSemaphore(Semaphore $semaphore)
    {
        $this->semaphore = $semaphore;
    }

    protected MessageBusInterface $mb;

    /**
     * @required
     */
    public function setMessageBus(MessageBusInterface $mb)
    {
        $this->mb = $mb;
    }

    public const POOL_CLASS_MAP = [
        JobPool00::class,
        JobPool01::class,
        JobPool02::class,
        // repeat, each class uses a common interface+trait...
    ];

    public function __invoke(JobInterface $job)
    {
        if (!$this->pool || Job::class != \get_class($job)) {
            $this->jm->runJob($job->getJob());
            if ($this->pool > 0) {
                if ($job instanceof JobPoolInterface) {
                    // decr pool counter
                    $this->decrPool($job->getPool());
                }
            }

            return;
        }

        // pool of job consume
        // the main transport is 'sync'
        // each worker has a transport mapped with a classe (ugly but supported)

        // some trafic optimization
        $pool = $this->getLeastUsedPool();
        $this->logger->info("Using pool $pool");
        while ($this->incrPool($pool) < 1) {
            // nothing
        }
        $c = self::POOL_CLASS_MAP[$pool];
        $message = new $c($job->getJob(), $pool);

        $this->mb->dispatch($message);
    }

    public const COUNTERS_KEY = 'workers_pool_counters';

    protected function getLeastUsedPool(): int
    {
        $counters = $this->semaphore->getCounters(self::COUNTERS_KEY);
        if (empty($counters)) {
            return \rand(0, $this->pool - 1);
        }
        for ($i = 0; $i < $this->pool; ++$i) {
            $idx = sprintf('%02d', $i);
            if (empty($counters[$idx])) {
                $counters[$idx] = 0;
            }
        }
        \asort($counters, SORT_NUMERIC);
        $value = reset($counters);
        $counters = \array_filter($counters, function ($v, $k) use ($value) {
            if ($v !== $value) {
                return false;
            }
            $k = (int) \intval($k);
            if ($k >= $this->pool) {
                return false;
            }

            return true;
        }, ARRAY_FILTER_USE_BOTH);
        if (empty($counters)) {
            return \rand(0, $this->pool - 1);
        }

        return (int) \intval(\array_rand($counters));
    }

    protected function incrPool(int $pool): int
    {
        return $this->semaphore->incrCounter(self::COUNTERS_KEY, sprintf('%02d', $pool));
    }

    protected function decrPool(int $pool): int
    {
        return $this->semaphore->decrCounter(self::COUNTERS_KEY, sprintf('%02d', $pool));
    }

    public function resetPool(int $pool): void
    {
        $this->semaphore->setCounter(self::COUNTERS_KEY, sprintf('%02d', $pool), 0);
    }
}

the config:

parameters:
    app.messenger.common_pool_options: 'delete_after_ack=true'

framework:
    messenger:

        transports:
            # https://symfony.com/doc/current/messenger.html#transport-configuration
            jobs_pool00: '%env(MESSENGER_TRANSPORT_DSN_JOBS_POOLXX)%00?%app.messenger.common_pool_options%'
            jobs_pool01: '%env(MESSENGER_TRANSPORT_DSN_JOBS_POOLXX)%01?%app.messenger.common_pool_options%'
            jobs_pool02: '%env(MESSENGER_TRANSPORT_DSN_JOBS_POOLXX)%02?%app.messenger.common_pool_options%'
            // repeat...

        routing:
            # Route your messages to the transports
            'App\Message\Job': sync # pool dispatch
            'App\Message\JobPool00': jobs_pool00
            'App\Message\JobPool01': jobs_pool01
            'App\Message\JobPool02': jobs_pool02
            ...

supervisor launches X workers

[program:jobworker]
command=php bin/console m2mql:messenger:consume jobs_pool%(process_num)02d
numprocs=10