1

I have a beanstalkd queue and have enabled job dependencies via a sub queue that posts when the dependent job is complete, in this example I have distributed parallel processing, so that I can trigger an action once all jobs are complete.

The issue is that I am looping through the response queue picking up messages, but once complete it does not attempt to delete any of the messages until the end, causing it to attempt to close the same message multiple times. For this I am trying to enrich the original message with the responses from the parallel processor, is there any way to make message end after the direct is complete, or via another method of message aggregation?

@Bean
RouteBuilder exampleRoute() {
    return new RouteBuilder() {
        @Override
        public void configure() throws Exception {
            /**
             * Step 1: Read a message from the listener tube, this will take the form:
             * {
             *  parent_job_id : The parent job id
             *  split_count: Number of responses to expect
             * }
             **/
            from("beanstalk://localhost/dev_job_listener?onFailure=release&jobDelay=20&jobTimeToRun=10")
                .unmarshal().json(JsonLibrary.Jackson, Map.class)
                .setProperty("message", simple("${body}"))
                // Get required tube jobId to allow deletion of job at end of this route
                .process(new Processor() {
                    @Override
                    public void process(Exchange exchange) throws Exception {
                        final Deque<Long> stack = new ArrayDeque<>();
                        stack.push(Long.valueOf(exchange.getIn().getHeader("beanstalk.jobId").toString()));
                        exchange.setProperty("stack", stack);
                        Map<String, Object> message = (HashMap) exchange.getProperties().get("message");
                        exchange.setProperty("parent_job_id", Integer.valueOf(message.get("parent_job_id").toString()));
                        exchange.setProperty("message_count", Integer.valueOf(message.get("split_count").toString()));
                    }
                })
                /**
                 * Step 2: Listed to the response queue job_{parent_job_id} for the number of completion messages
                 * from the split_count.
                 */
                .to("direct:verifySubJobs").end()

                /**
                 * Step 3: Post to the complete queue that all jobs have completed
                 */
                .to("beanstalk://localhost/next_step?jobTimeToRun=10")
                .process(new Processor() {
                    @Override
                    public void process(Exchange exchange) throws Exception {
                        exchange.getIn().setHeader("beanstalk.jobId", exchange.getProperty("stack", Deque.class).pop());
                    }
                })
                .log("Ending Job : " + simple("${header.beanstalk.jobId}"));


            from("direct:verifySubJobs")
                .log("-> direct:verifySubJobs")
                .setProperty("url", simple("beanstalk://localhost/dev_job_" + simple("${property.parent_job_id}").getText()  + "?onFailure=release&jobTimeToRun=10"))
                .pollEnrich().simple("${property.url}")
                    .process(new Processor() {
                        @Override
                        public void process(Exchange exchange) throws Exception {
                            Deque<Long> stack = exchange.getProperty("stack", Deque.class);
                            stack.push(Long.valueOf(exchange.getIn().getHeader("beanstalk.jobId").toString()));
                            Integer counter = exchange.getProperty("message_count", Integer.class);
                            exchange.setProperty("message_count", counter);
                        }
                    }).end()
                        .choice().when(simple("${property.message_count} > 0"))
                            .to("direct:verifySubJobs")
                        .end()
                    .process(new Processor() {
                        @Override
                        public void process(Exchange exchange) throws Exception {
                            exchange.getIn().setHeader("beanstalk.jobId", exchange.getProperty("stack", Deque.class).pop());
                        }
                    }).end()
                .log("<- direct:verifySubJobs");
        }
    };
}
Chris
  • 105
  • 1
  • 10

0 Answers0