I am trying to create a spring batch application using annotation based approach with partitioner, which will be triggered by quartz scheduler, but getting following issues.
When the job is triggered each partition is executed sequentially instead of parallelly i.e if I have 10 partitions instead of all 10 getting triggered/processed together it process one by one.
When more than one instance of job(this is needed as per my requirement) gets triggered it's not getting synchronized properly i.e when 2nd instace is started it uses 1st instance data and 1st instance will stop processing but will be active.
Following are my configuration/class files.
BatchConfiguration.java -
@Configuration
@EnableBatchProcessing
public class BatchConfiguration
{
@Autowired
private JobBuilderFactory jobBuilders;
@Autowired
private StepBuilderFactory stepBuilders;
@Bean
@StepScope
public JdbcCursorItemReader reader(@Value("#{stepExecutionContext[someParam]}") String someParam) {
JdbcCursorItemReader jdbcCursorItemReader = new JdbcCursorItemReader();
jdbcCursorItemReader.setDataSource(getDataSource());
jdbcCursorItemReader.setSql("myQuery");
jdbcCursorItemReader.setRowMapper(new NotifRowMapper());
return jdbcCursorItemReader;}
@Bean
@StepScope
public MyProcessor processor() {
return new MyProcessor();}
@Bean
public MyPartitioner partitioner() {
MyPartitioner partitioner = new MyPartitioner();
partitioner.setDataSource(getDataSource());
partitioner.setSql("MyPartitionerQuery");
return partitioner;}
@Bean
@StepScope
public JdbcBatchItemWriter writer(DataSource dataSource) {
JdbcBatchItemWriter writer = new JdbcBatchItemWriter();
writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider());
writer.setSql("MyWriterQuery");
writer.setDataSource(dataSource);
return writer;}
@Bean
public Job springBatch() {
return jobBuilders.get("springBatch").start(masterStep()).build();}
@Bean
public Step masterStep() {
return stepBuilders.get("masterStep")
.partitioner(slave(reader(null), writer(getDataSource()),processor()))
.partitioner("slave", partitioner())
.taskExecutor(taskExecutor()).build();}
@Bean
public Step slave(JdbcCursorItemReader reader,JdbcBatchItemWriter writer, MyProcessor processor) {
return stepBuilders.get("slave")
.chunk(100).reader(reader).processor(processor).writer(writer).build();}
@Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(20);
taskExecutor.afterPropertiesSet();
return taskExecutor;}
@Bean
public JdbcTemplate jdbcTemplate(DataSource dataSource) {
return new JdbcTemplate(dataSource);}
@Bean
public DataSource getDataSource() {
return dataSource;
}
@Bean
public JobRepository getJobRepository() throws Exception {
MapJobRepositoryFactoryBean factory = new MapJobRepositoryFactoryBean();
factory.setTransactionManager(new ResourcelessTransactionManager());
factory.afterPropertiesSet();
factory.setIsolationLevelForCreate("ISOLATION_READ_COMMITTED");
return (JobRepository) factory.getObject();
}}
QuartzJob.java(Triggers spring batch job) -
public class QuartzJob implements org.quartz.Job
{
@Override
public void execute(org.quartz.JobExecutionContext jobExecutionContext) throws org.quartz.JobExecutionException
{
AnnotationConfigApplicationContext context;
try
{
context = new AnnotationConfigApplicationContext(BatchConfiguration.class);
JobLauncher jobLauncher = context.getBean(JobLauncher.class);
org.springframework.batch.core.Job newJob = context.getBean("springBatch", org.springframework.batch.core.Job.class);
JobParameters param = new JobParametersBuilder().addLong("time",System.currentTimeMillis()).toJobParameters();
jobLauncher.run(newJob, param);
} catch (Exception e){}}}
MyQuartzListener.java(Class triggers quartz job during server start up)-
public class MyQuartzListener implements ServletContextListener
{
private Scheduler scheduler;
@Override
public void contextDestroyed(ServletContextEvent arg0){ }
@Override
public void contextInitialized(ServletContextEvent ctx)
{
JobDetail job = JobBuilder.newJob(QuartzJob.class).withIdentity("SPRINGBATCH", "SPRINGBATCH").build();
Trigger trigger = TriggerBuilder.newTrigger().withIdentity("SPRINGBATCH", "SPRINGBATCH").startNow().withSchedule(SimpleScheduleBuilder.simpleSchedule().withIntervalInSeconds(60).repeatForever()).build();
try
{
scheduler = ((StdSchedulerFactory) ctx.getServletContext().getAttribute(QuartzInitializerListener.QUARTZ_FACTORY_KEY)).getScheduler();
job.getJobDataMap().put("quartztime", System.currentTimeMillis());
scheduler.scheduleJob(job, trigger);
} catch (SchedulerException e) {} }
}
MyPartitioner .java
public class MyPartitioner implements Partitioner
{
@Override
public Map<String, ExecutionContext> partition(int gridSize)
{
Map<String, ExecutionContext> partitionMap = new HashMap<String, ExecutionContext>();
List<String> partitionCodes = getPartitionCodes(sql);
int count = 1;
for (String partitionCode : partitionCodes)
{
ExecutionContext context = new ExecutionContext();
context.put("partitionCode", partitionCode);
context.put("name", "Thread" + count);
partitionMap.put(partitionCode, context);
count++;
}
return partitionMap;}}
Is there something wrong with this configuration? I am passing current time to each instance of job to identify each instance seperately, but still it's not working.