Ordering of responses from endpoints in RecipientList

Question

I am using camel 2.17 and Fuse 6.3. I have a scenario wherein I need to send my message to multiple endpoints which are doing some dB logging each.

I am using "recipientList" to send my message to these endpoints but here I face an issue that, my DB entries are not in order of the endpoint calls.

Suppose, I have 3 endpoints A,B,C and they log 1 message each in DB after some processing MessageA, MessageB and MessageC but when I run my route RL(a,b,c) I do not see the DB mesages in same order although these are direct endpoints.

Is there a way my route can wait for first endpoint to complete and then process the second endpoint?

Here is my route Sample

<route id="RouteStart">
        <from id="_from1" uri="file:.../>
        <bean id="_bean1" method="hasReadWriteAccess" ref="fileAccessValidator"/>
        <log id="_log1" message=${body}"/>
        <recipientList id="_recipientList1">
            <simple>FirstStepInsert, MessageDirQueue, Step1Queue</simple>
        </recipientList>
    </route>
    <!-- This route picks the file from queue, encodes the file in to UTF-8 BOM and validates file against XSD -->
    <route id="Step1Route">
        <from id="_from2" uri="Step1Queue"/>
        <log id="_log2" message=${body}"/>
         <setProperty id="_setProperty4" propertyName="stepName">
            <simple>File Validation</simple>
        </setProperty>
        <doTry id="_doTry1">
            <to id="_to1" uri="Step2Queue"/>
            <doCatch id="_doCatch1">
                <exception>org.apache.camel.ValidationException</exception>
                <to id="_to2" uri="file:..?autoCreate=true"/>
                <log id="_log7" message="Moved Invalid file ${file:name}"/>
            </doCatch>
            <doCatch id="_doCatch2">
                <exception>java.io.IOException</exception>
                <to id="_to3" uri="file:..?autoCreate=true"/>
                <log id="_log8" message="Moved XML file ${file:name} with Incorrect access"/>
            </doCatch>
            <doFinally id="_doFinally1">
                <to id="_to4" uri="direct:stepInsertLogging"/>
            </doFinally>
        </doTry>
    </route>
    <route id="Step2Route">
        <from id="_from3" uri="Step2Queue"/>
        <log id="_log9" message="${body}"/>
        <setProperty id="_setProperty12" propertyName="stepName">
            <simple>Data Transformation</simple>
        </setProperty>
        <recipientList id="_recipientList2">
            <simple>direct:stepInsertLogging, ReceiveDirQueue, Step3Queue</simple>
        </recipientList>
    </route>
    <route id="Step3Route">
        <from id="_from4" uri="Step3Queue"/>
        <setProperty id="_setProperty19" propertyName="stepName">
            <simple>File Delivered</simple>
        </setProperty>
        <to id="_to5" uri="file:..."/>
        <onException id="_onException1">
            <exception>java.io.IOException</exception>
            <redeliveryPolicy maximumRedeliveries="2" redeliveryDelay="0"/>
        <recipientList id="_recipientList3">
            <simple>direct:stepInsertLogging, direct:flowUpdateLogging</simple>
        </recipientList>
    </route>
    <!-- This route sends a copy of source file to Message Archive folder -->
    <route id="MessageDirRoute">
        <from id="_from9" uri="MessageDirQueue"/>
        <log id="_log23" message="${body}"/>
        <setProperty id="_setProperty63" propertyName="stepName">
            <simple>Data SourceFile Logging</simple>
        </setProperty>
        <to id="_to13" uri="file:.."/>
        <onException id="_onException4">
            <exception>java.io.IOException</exception>
            <handled>
                <constant>true</constant>
            </handled>
        <to id="_to14" uri="direct:stepInsertLogging"/>
    </route>

    <!-- This route sends a copy of destination file to Message Archive folder -->
    <route id="receiveDirectory-route">
        <from id="_from10" uri="ReceiveDirQueue"/>
        <setProperty id="_setProperty77" propertyName="stepName">
            <simple>Data DestinationFile Logging</simple>
        </setProperty>
        <to id="_to15" uri="file:.."/>
        <onException id="_onException5">
            <exception>java.io.IOException</exception>
            <redeliveryPolicy maximumRedeliveries="2" redeliveryDelay="0"/>
            <handled>
                <constant>true</constant>
            </handled>
        <to id="_to16" uri="direct:stepInsertLogging"/>
    </route>
    <!-- Event Logging Routes -->
    <route id="startMainLogRoute">
        <from id="_from11" uri="FirstStepInsert"/>
        <log id="_log25" message="Received File ${file:name} : ${body}"/>
        <setProperty id="_setProperty92" propertyName="stepName">
            <simple>File Received</simple>
        </setProperty>
        <log id="_log26" message="${property.flowId}"/>
        <recipientList id="_recipientList7" streaming="false">
            <simple>direct:flowInsertLogging, direct:stepInsertLogging</simple>
        </recipientList>
    </route>
    <route id="flowInsertLogRoute">
        <from id="_from12" uri="direct:flowInsertLogging"/>
        <log id="_log27" message="[flowInsertLogRoute]  Flow Id is ${property.flowId}"/>
        <process id="_process8" ref="flowProcessor"/>
        <transform id="_transform1">
            <method method="getFlowMap" ref="flowMapper"/>
        </transform>
        <log id="_log28" message="Executing the query {{sql.insertFlowDetail}}"/>
        <to id="_to17" uri="sql:{{sql.insertFlowDetail}}"/>
    </route>
    <route id="stepInsertLogRoute">
        <from id="_from13" uri="direct:stepInsertLogging"/>
        <log id="_log29" message="[stepInsertLogRoute] Flow Id is ${property.flowId}"/>
        <process id="_process9" ref="stepProcessor"/>
        <transform id="_transform2">
            <method method="getStepMap" ref="stepMapper"/>
        </transform>
        <log id="_log30" message="Executing the query {{sql.insertStepDetail}}"/>
        <to id="_to18" uri="sql:{{sql.insertStepDetail}}"/>
    </route>
    <route id="flowUpdateLogRoute">
        <from id="_from14" uri="direct:flowUpdateLogging"/>
        <log id="_log31" message="[flowUpdateLogRoute] Flow Id is ${property.flowId}"/>
        <transform id="_transform3">
            <method method="getFlowUpdateMap" ref="flowUpdateMapper"/>
        </transform>
        <log id="_log32" message="Executing the query {{sql.updateFlowDetail}}"/>
        <to id="_to19" uri="sql:{{sql.updateFlowDetail}}"/>
    </route>
</camelContext>

Here, I am updating the "stepName" property in my DB which is not following the execution order.

It should be:

File Received
Data SourceFile Logging
File Validation
Data Transformation
Data DestinationFile Logging
File Delivered

But I do not see this order in the DB entries.

Can you please show us your route(s)? The RecipientList should be processing each endpoint sequentially by default. — Erik Karlstrand, Dec 22 '17 at 08:17
It does process the requests sequentially, but are the responses handled in the same way? If endpoint A is taking 2 seconds to process the request, whereas the endpoint B processes it earlier, will it wait for endpoint A to complete the execution? — Pulkit, Dec 22 '17 at 08:44
Neither, my streaming is set to false (default) and I am not doing any parallel processing. — Pulkit, Dec 22 '17 at 09:50
@ThemisPirgiotis , this is the ideal case but it is not happening with my route. — Pulkit, Dec 22 '17 at 10:28
@Pulkit talking general your problem will not be solved, why don't provide a sample of your code to examine it more specific? — Themis Pyrgiotis, Dec 22 '17 at 10:29
@ThemisPyrgiotis, I have added my sample route. Please have a look. — Pulkit, Dec 22 '17 at 12:23

score 1 · Answer 1 · answered Dec 22 '17 at 21:37

Themis and NoMad17 have already said in comments. From the docs:

The recipients will receive a copy of the same Exchange, and Camel will execute them sequentially.

I see in your route that you're chaining other recipient lists. Maybe you are missing something since it's many routes. To validate the sequence from the recipient list, took this unit test as example:

public class RecipientListSequenceAggregateRouteTest extends CamelTestSupport {

    EmbeddedDatabase db;

    @Before
    public void setUp() throws Exception {
        db = new EmbeddedDatabaseBuilder()
            .setType(EmbeddedDatabaseType.DERBY).addScript("sql/test.sql").build();

        super.setUp();
    }

    @Override
    protected RoutesBuilder createRouteBuilder() throws Exception {
        return new RouteBuilder() {

            @Override
            public void configure() throws Exception {
                getContext().getComponent("sql", SqlComponent.class).setDataSource(db);

                from("direct:start")
                    .recipientList(constant("direct:a1, direct:a2, direct:a3, direct:select"));

                from("direct:a1")
                    .log("got message in a1: waiting 3s")
                    .process(new Processor() {
                        @Override
                        public void process(Exchange exchange) throws Exception {
                            Thread.sleep(3000);
                        }
                    })
                    .setBody(constant("a1"))
                    .recipientList(constant("direct:db, direct:flow"));

                from("direct:a2")
                    .log("got message in a2: waiting 5s")
                    .process(new Processor() {
                        @Override
                        public void process(Exchange exchange) throws Exception {
                            Thread.sleep(5000);
                        }                   
                    })
                    .setBody(constant("a2"))
                    .recipientList(constant("direct:db, direct:flow"));

                from("direct:a3")
                    .log("got message in a3: waiting 1s")
                    .process(new Processor() {
                        @Override
                        public void process(Exchange exchange) throws Exception {
                            Thread.sleep(1000);
                        }
                    });

                from("direct:db")
                    .log("got message in db from ${body}")
                    .setBody(simple("db_${in.body}"))
                    .to("sql:insert into log (body_in) values (:#${in.body})");
                    ;

                from("direct:flow")
                    .log("got message in flow from ${body}")
                    .setBody(simple("flow_${in.body}"))
                    .to("sql:insert into log (body_in) values (:#${in.body})");
                    ;

                from("direct:select")
                    .to("sql:select * from log order by time_in")
                    .log("results:\n ${body}")
                    .to("mock:result");
            }
        };
    }

    @Test
    public void test() throws InterruptedException {
        getMockEndpoint("mock:result").expectedMessageCount(1);
        Object results = template.requestBody("direct:start", "");
        assertNotNull(results);
        assertMockEndpointsSatisfied();
    }
}

The results are:

19:23:47.090 [main] INFO route2 - got message in a1: waiting 3s
19:23:50.093 [main] INFO route5 - got message in db from a1
19:23:50.315 [main] INFO route6 - got message in flow from a1
19:23:50.337 [main] INFO route3 - got message in a2: waiting 5s
19:23:55.343 [main] INFO route5 - got message in db from a2
19:23:55.351 [main] INFO route6 - got message in flow from a2
19:23:55.359 [main] INFO route4 - got message in a3: waiting 1s
19:23:56.428 [main] INFO route7 - results:
[{BODY_IN=db_a1, TIME_IN=2017-12-22 19:23:50.301}, {BODY_IN=flow_a1, TIME_IN=2017-12-22 19:23:50.334}, {BODY_IN=db_a2, TIME_IN=2017-12-22 19:23:55.348}, {BODY_IN=flow_a2, TIME_IN=2017-12-22 19:23:55.356}]

As you see, even chaining other recipient lists the messages were delivered sequentially. Could you take apart some samples from your route and see the logs? Try to analyze step by step. Seeing your DSL I couldn't see anything wrong with it.

If you need to manipulate the reply from the recipient list, you may need a custom Aggregator:

An AggregationStrategy that will assemble the replies from recipients into a single outgoing message from the Recipient List. By default Camel will use the last reply as the outgoing message. From Camel 2.12 onwards you can also use a POJO as the AggregationStrategy, see the Aggregator page for more details. If an exception is thrown from the aggregate method in the AggregationStrategy, then by default, that exception is not handled by the error handler. The error handler can be enabled to react if enabling the shareUnitOfWork option.

Ordering of responses from endpoints in RecipientList

1 Answers1