0

I am trying to set the spring boot application which will pool the csv . i do not see any activity happning in the spring boot application nor on filezilla SFTP server but if I change the same code to FTP then it works


  @Component
    @EnableIntegration
    public class IntegrationConfiguration {
        @Autowired
        FTPConfigProperties ftpConfigProperties;
        @Autowired
        private BeanFactory beanFactory;
        @Value("classpath:certificate.crt")
        Resource certficateFile;
        @Bean
        public SessionFactory<ChannelSftp.LsEntry> ftpSessionFactory() {
            DefaultSftpSessionFactory factory = new DefaultSftpSessionFactory();
            factory.setHost("127.0.0.1");
            factory.setPort(990);
            factory.setUser("abhinav");
            factory.setPassword("nssdw");

            factory.setPrivateKey(certficateFile);
            factory.setAllowUnknownKeys(true);
            return new CachingSessionFactory<ChannelSftp.LsEntry>(factory, 100000);
        }

        @Bean
        public SftpInboundFileSynchronizer ftpInboundFileSynchronizer() {
            SftpInboundFileSynchronizer fileSynchronizer = new SftpInboundFileSynchronizer(ftpSessionFactory());
            fileSynchronizer.setDeleteRemoteFiles(false);
            fileSynchronizer.setRemoteDirectory("/");
            fileSynchronizer.setFilter(filter());
            fileSynchronizer.setDeleteRemoteFiles(false);
            fileSynchronizer.setPreserveTimestamp(true);
            fileSynchronizer.setBeanFactory(beanFactory);
            return fileSynchronizer;
        }
    //here the poller is configured 
        @Bean
        @InboundChannelAdapter(channel = "fromSftpChannel", poller = @Poller(fixedDelay = "10000"))
        public MessageSource<File> ftpMessageSource() throws Exception {
            SftpInboundFileSynchronizingMessageSource source = new SftpInboundFileSynchronizingMessageSource(
                    ftpInboundFileSynchronizer());
            source.setLocalDirectory(new File("ftp-inbound"));
            source.setAutoCreateLocalDirectory(true);
            source.setMaxFetchSize(1);
            source.setBeanFactory(beanFactory);
            source.setUseWatchService(true);
            return source;
        }

        public CompositeFileListFilter<ChannelSftp.LsEntry> filter() {
            CompositeFileListFilter<ChannelSftp.LsEntry> filter = new CompositeFileListFilter<ChannelSftp.LsEntry>();
            filter.addFilter(new SftpSimplePatternFileListFilter("*.csv"));
            filter.addFilter(acceptOnceFilter());
            filter.addFilter(new LastModifiedLsEntryFileListFilter());
            return filter;
        }

        @Bean
        public SftpPersistentAcceptOnceFileListFilter acceptOnceFilter() {
            SftpPersistentAcceptOnceFileListFilter filter = new SftpPersistentAcceptOnceFileListFilter(metadataStore(),"ftpPersistentAcceptOnce");
            filter.setFlushOnUpdate(true);
            return filter;
        }


        @Bean
        public ConcurrentMetadataStore metadataStore() {
            PropertiesPersistingMetadataStore propertiesPersistingMetadataStore = new PropertiesPersistingMetadataStore();
            propertiesPersistingMetadataStore.setBaseDirectory("./metastore");
            propertiesPersistingMetadataStore.setFileName("ftpStream.properties");
            return propertiesPersistingMetadataStore;
        }

        @Bean
        @ServiceActivator(inputChannel = "jobChannel", outputChannel = "nullChannel")
        protected JobLaunchingMessageHandler launcher(JobLauncher jobLauncher) {
            return new JobLaunchingMessageHandler(jobLauncher);
        }
    }

here the next call where I trigger the spring batch then it goes to service actuator

@Component
public class FileToJobTransformer implements ApplicationContextAware {
    private ApplicationContext context;
   @Autowired
    private Job job;



    @Transformer(inputChannel = "fromSftpChannel", outputChannel = "jobChannel")
    public JobLaunchRequest transform(File aFile) throws Exception {
        String fileName = aFile.getName();
        JobParameters jobParameters = new JobParametersBuilder().addString(
                "input.file", aFile.getAbsolutePath()).toJobParameters();
        JobLaunchRequest request = new JobLaunchRequest(job, jobParameters);
        return request;
    }

    @Override
    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
        this.context = applicationContext;
    }
}

the custome code is as follow


public class LastModifiedLsEntryFileListFilter implements FileListFilter<ChannelSftp.LsEntry> {

    private static final long DEFAULT_AGE = 60;

    private volatile long age = DEFAULT_AGE;

    public long getAge() {
        return this.age;
    }

    public void setAge(long age) {
        setAge(age, TimeUnit.SECONDS);
    }

    public void setAge(long age, TimeUnit unit) {
        this.age = unit.toSeconds(age);
    }

    @Override
    public List<ChannelSftp.LsEntry> filterFiles(ChannelSftp.LsEntry[] files) {

        System.out.println("files = [" + files.length + "]");

        List<ChannelSftp.LsEntry> list = new ArrayList<ChannelSftp.LsEntry>();

        long now = System.currentTimeMillis() / 1000;

        for (ChannelSftp.LsEntry file : files) {
            if (file.getAttrs()
                    .isDir()) {
                continue;
            }
            int lastModifiedTime = file.getAttrs()
                    .getMTime();

            if (lastModifiedTime + this.age <= now) {
                list.add(file);
            }
        }

        Collections.reverse(list);
      ArrayList<ChannelSftp.LsEntry> oneElementList = new  ArrayList<ChannelSftp.LsEntry>(1) ;
      oneElementList.add(list.get(0));
        return oneElementList;
    }

}


  • I suggest you to turn on DEBUG logging for the `org.springframework.integration` category and also for the `com.jcraft.jsch`. This way you are going to see something in logs when the channel adapter attempts to connect to SFTP. Although it is not clear what is `LastModifiedLsEntryFileListFilter` why does it come already after `acceptOnceFilter`, but not before and why there is no any `age` option for that?.. – Artem Bilan Apr 11 '19 at 20:16
  • how can I turn log on for org.springframework.integration ? I can add more about ```LastModifiedLsEntryFileListFilter``` @ArtemBilan I have added more code . Now is it clear? – abhinav jain Apr 11 '19 at 20:19
  • Huh? I believe you have some logging config in your project. If not consider to add something like `Log4J` and its `log4j2.xml` into your project resources. Also you may consider logging feature in Spring Boot if you use one, of course: https://logging.apache.org/log4j/2.x/, https://docs.spring.io/spring-boot/docs/2.1.4.RELEASE/reference/htmlsingle/#boot-features-logging – Artem Bilan Apr 11 '19 at 20:21
  • ` oneElementList.add(list.get(0));` ? What is the purpose of that logic? The channel adapter fetches remote files and store them in the memory cache and doesn't go to server until the cache is over, but since you have already filtered them in the `acceptOnceFilter`, they are not going to appear any more. I'm confused... – Artem Bilan Apr 11 '19 at 20:26
  • Because I want to get the latest file from the list of filles on the ftp sever. Moreover if I comment all the filters then also its does not work . simply its ignoring the poller at all .@ArtemBilan – abhinav jain Apr 11 '19 at 20:30
  • You may have a connection issue, that's why I recommend to increase a logging level in your application to have more clues what is going on. – Artem Bilan Apr 11 '19 at 20:32
  • @ArtemBilan I just commented again all the filter again it does not work :-( – abhinav jain Apr 11 '19 at 20:35
  • @ArtemBilan I also suspect the connection issue let me check with the logging more – abhinav jain Apr 11 '19 at 20:36
  • `@Value("classpath:certificate.crt")` - that must be an RSA file for `PRIVATE KEY`. I can't figure out what and how is going to work with your filters, but the problem might be just because you can't connect to SFTP. Since `Poller` happens on the separate thread you might just simply lose an exception. But DEBUG logging is definitely going to help – Artem Bilan Apr 11 '19 at 20:37
  • @ArtemBilan I just got it working after so many changes in the requirement . You were correct it was connection issue but strange that it does not say until we enable the loggin. May be its a nice to have at least info message if connection fail? – abhinav jain Apr 23 '19 at 14:32
  • Good. May you share in your question what is the stack trace? – Artem Bilan Apr 23 '19 at 14:35
  • @ArtemBilan lot of stuff had been changed since then I had to restimulate the situation. I will do it later if it's not urgent? – abhinav jain Apr 23 '19 at 14:38
  • It's not: take your time. My point is to know a stack trace to determine the point where we indeed can add a WARN logging message about connection failure. – Artem Bilan Apr 23 '19 at 14:39

0 Answers0