0

I am scraping network traffic data using Selenium v4 and exporting it to an excel file using Apache POI. I am able to print scraped data in console using the println statement, however, I am not able to export all the data that the script scraped. Please help me with this. Here is my code:

Note: I have used Thread.sleep(), implicit wait, but that did not help.

public class CaptureNetworkTraffic 
{
    //CreateFile File;
    
    
    public static void main(String[] args) throws IOException, InterruptedException
    {
        
        //File=new CreateFile();
        XSSFWorkbook workbook= new XSSFWorkbook();
        XSSFSheet sheet= workbook.createSheet("Network Traffic");
        System.setProperty("webdriver.chrome.driver", "C:\\Users\\prash\\Downloads\\chromedriver_win32\\chromedriver.exe");
        ChromeDriver d1=new ChromeDriver();
        //d1.manage().timeouts().implicitlyWait(Duration.ofMinutes(1));

        DevTools devtool=d1.getDevTools();
        devtool.createSession();
        devtool.send(Network.enable(Optional.empty(), Optional.empty(), Optional.empty()));
        
        System.out.println("***************response Received**************");
        
        //devtool.addListener(null, null);
        devtool.addListener(Network.responseReceived(), response ->
        {
            
            Response res=response.getResponse();
            
            Object scrappedDataArray[][]= {{res.getUrl(),res.getResponseTime(),res.getHeaders()}};
            
            int rows=scrappedDataArray.length;
            int cols=scrappedDataArray[0].length;
            
            for(int r=0;r<rows;r++)
            {
                XSSFRow createRow=sheet.createRow(r);
                for(int c=0;c<cols;c++)
                {
                    XSSFCell createCell=createRow.createCell(c);
                    Object value=scrappedDataArray[r][c];
                    if(value instanceof String)
                        createCell.setCellValue((String)value);
                    if(value instanceof Integer)
                        createCell.setCellValue((Integer)value);
                    if(value instanceof Long)
                        createCell.setCellValue((Long)value);
                    
                    
                    
                }
            }
            
        });
        
        //DevTools devTools = d1.getDevTools();
        d1.get("https://www.google.com/");
        Thread.sleep(4000);
        String filepath=".\\scrapedDataFiles\\data.xlsx";   
        FileOutputStream fos= new FileOutputStream(filepath);
        //Thread.sleep(4000);

        workbook.write(fos);
        workbook.close();
        
        
    }
    
}
zolastro
  • 918
  • 12
  • 29
  • Are you sure, that your scrappedDataArray has actually String/Integer/Long values in it? it looks like (URL, Date, Map), which will be converted to string, when output to console. I assume you can't debug the code within selenium, but you can add debug output to a file, to see what's going on. – kiwiwings Dec 20 '21 at 00:31
  • @kiwiwings thanks for your comment. Yes, scrappedDataArray contact all types of data and I am totally not getting how to export this kind of run-time data to an excel file and yes I am unable to debug as well. can you please let me know the better way to scrape and export data – AutoMationKing Dec 20 '21 at 02:54

0 Answers0