I need to add a Spring Batch Job in my Spring Boot application, i need to downloads a list of file from a FTP and process them. They are CSV files.
I have created a Job with only one Step (i will come back later on this point).
In my step i have a Reader (FlatFileItemReader), a processor (that transform my entity) and an itemWriter that write the data in my Database.
I want to delete the file that i have downloaded after it's processing.
To dot that, i have tryied to add a second step that juste delete the file after the processsing. With that step2, some times my file is not deleted. It's like my ItemReader is not closing the inputStream so i keep an handler on it.
I have tried an other solution, use a custom FlatItemReader, i have override the close() method, to delete the file after is closed. A weird thing happen with this solution, my close method is called twice and some times i can delete the file and some times i can't delete it...
See the logs below :
o.s.b.c.l.support.SimpleJobLauncher : Job: [SimpleJob: [name=job]] launched with the following parameters: [{time=1551704019790, PATH_TO_FILE=C:\tmp\tmpCorresp\ESPCORR_LITE2.CSV, organisationId=153}]
o.s.batch.core.job.SimpleStepHandler : Executing step: [step1]
c.m.b.a.config.BatchConfig : Do Close Started !!!!
c.m.b.a.config.BatchConfig : start Deletion
c.m.b.a.config.BatchConfig : File is not deleted
c.m.b.a.config.BatchConfig : Do Close Started !!!!
c.m.b.a.config.BatchConfig : start Deletion
c.m.b.a.config.BatchConfig : File is not deleted
o.s.b.c.l.support.SimpleJobLauncher : Job: [SimpleJob: [name=job]] completed with the following parameters: [{time=1551704019790, PATH_TO_FILE=C:\tmp\tmpCorresp\ESPCORR_LITE2.CSV, organisationId=153}] and the following status: [COMPLETED]
c.m.b.a.scheduled.TestTask : FTP FILES FOR organisation2 DOWNLOADED
c.m.b.a.scheduled.TestTask : Et BIM le flux !
the code of my close method :
@Override
public void close() throws ItemStreamException {
super.close();
deleteFileAfterClose();
}
private void deleteFileAfterClose(){
log.debug("start Deletion");
File f = null;
try {
f = resourceHandler.getFile();
} catch (IOException e) {
log.error("Error while retrieving file : ", e);
}
if(f != null && f.exists()){
boolean delete = f.delete();
if(delete){
log.debug("File is deleted");
}
else {
log.debug("File is not deleted");
}
}
}
I really thankfull if someone can help me.
Thanks !!!
Sincerely.
Nicolas Sagon.
edit :
My BatchConfig.java :
@Configuration
@PropertySource("classpath:config/default.properties")
@EnableBatchProcessing
public class BatchConfig {
private static final Logger log = LoggerFactory.getLogger(BatchConfig.class);
@Autowired
public JobBuilderFactory jobBuilderFactory;
@Autowired
public StepBuilderFactory stepBuilderFactory;
@Autowired
public CorrespondentDao correspondentDao;
@Autowired
public JobRepository jobRepository;
@Bean
public SimpleJobLauncher simpleJobLauncher() throws Exception {
SimpleJobLauncher simpleJobLauncher = new SimpleJobLauncher();
simpleJobLauncher.setJobRepository(jobRepository);
simpleJobLauncher.setTaskExecutor(new SyncTaskExecutor());
simpleJobLauncher.afterPropertiesSet();
return simpleJobLauncher;
}
@Bean
public Job job(Step step1, Step step2) {
return jobBuilderFactory.get("job")
.preventRestart()
.start(step1)
.next(step2)
.build();
}
@Bean
Step step1(FlatFileItemReader<CorrespondentEntity> reader, ItemWriter<CorrespondentEntity> writer) {
return stepBuilderFactory
.get("step1")
.<CorrespondentEntity, CorrespondentEntity>chunk(1)
.reader(reader)
.processor(new Processor())
.writer(writer)
.build();
}
//step for deleting the file
@Bean
Step step2(FileDeletingTasklet deletingTask) {
FileDeletingTasklet task = deletingTask;
return stepBuilderFactory.get("step2")
.allowStartIfComplete(true)
.tasklet(task)
.build();
}
@SuppressWarnings("Duplicates")
@Bean
@JobScope
public FlatFileItemReader<CorrespondentEntity> reader(@Value("#{jobParameters['PATH_TO_FILE']}") String pathToFile ) throws MalformedURLException {
FlatFileItemReader<CorrespondentEntity> reader = new CustomReader<>();
reader.setResource(new FileUrlResource(pathToFile));
reader.setLinesToSkip(1);
reader.setLineMapper(new csvLineMapper());
return reader;
}
@Bean
@JobScope
public ItemWriter<CorrespondentEntity> writer(CorrespondentDao correspondentDao, @Value("#{jobParameters['organisationId']}") Long organisationId){
return new Writer(correspondentDao, organisationId);
}
@Bean
@JobScope
public FileDeletingTasklet deletingTask(@Value("#{jobParameters['PATH_TO_FILE']}") String pathToFile){
return new FileDeletingTasklet(pathToFile);
}
private class CustomReader<T> extends FlatFileItemReader<T> implements ItemStream {
private Resource resourceHandler;
@Override
public void setResource(Resource resource) {
super.setResource(resource);
this.resourceHandler = resource;
}
@Override
public void close() throws ItemStreamException {
super.close();
}
//Not used for the moment
private void deleteFileAfterClose(){
log.debug("start Deletion");
File f = null;
try {
f = resourceHandler.getFile();
} catch (IOException e) {
log.error("Error while retrieving file : ", e);
}
if(f != null && f.exists()){
boolean delete = f.delete();
if(delete){
log.debug("File is deleted");
}
else {
log.debug("File is not deleted");
}
}
}
}
}
FileDeletingTasklet.java
import com.micropole.biomnis.authentification.scheduled.TestTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
import java.io.File;
public class FileDeletingTasklet implements Tasklet {
private static final Logger log = LoggerFactory.getLogger(TestTask.class);
private String filePath;
public FileDeletingTasklet(String filePath) {
this.filePath = filePath;
}
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
log.debug("try to delete this file : " + filePath);
File f = new File(filePath);
if(f.exists()){
boolean delete = f.delete();
if(delete){
log.debug("File is deleted !!!!");
return RepeatStatus.FINISHED;
}
else {
log.debug("File is not deleted !!!");
return RepeatStatus.FINISHED;
}
}
return RepeatStatus.FINISHED;
}
}