I am scraping network traffic data using Selenium v4 and exporting it to an excel file using Apache POI. I am able to print scraped data in console using the println
statement, however, I am not able to export all the data that the script scraped. Please help me with this. Here is my code:
Note: I have used Thread.sleep()
, implicit wait, but that did not help.
public class CaptureNetworkTraffic
{
//CreateFile File;
public static void main(String[] args) throws IOException, InterruptedException
{
//File=new CreateFile();
XSSFWorkbook workbook= new XSSFWorkbook();
XSSFSheet sheet= workbook.createSheet("Network Traffic");
System.setProperty("webdriver.chrome.driver", "C:\\Users\\prash\\Downloads\\chromedriver_win32\\chromedriver.exe");
ChromeDriver d1=new ChromeDriver();
//d1.manage().timeouts().implicitlyWait(Duration.ofMinutes(1));
DevTools devtool=d1.getDevTools();
devtool.createSession();
devtool.send(Network.enable(Optional.empty(), Optional.empty(), Optional.empty()));
System.out.println("***************response Received**************");
//devtool.addListener(null, null);
devtool.addListener(Network.responseReceived(), response ->
{
Response res=response.getResponse();
Object scrappedDataArray[][]= {{res.getUrl(),res.getResponseTime(),res.getHeaders()}};
int rows=scrappedDataArray.length;
int cols=scrappedDataArray[0].length;
for(int r=0;r<rows;r++)
{
XSSFRow createRow=sheet.createRow(r);
for(int c=0;c<cols;c++)
{
XSSFCell createCell=createRow.createCell(c);
Object value=scrappedDataArray[r][c];
if(value instanceof String)
createCell.setCellValue((String)value);
if(value instanceof Integer)
createCell.setCellValue((Integer)value);
if(value instanceof Long)
createCell.setCellValue((Long)value);
}
}
});
//DevTools devTools = d1.getDevTools();
d1.get("https://www.google.com/");
Thread.sleep(4000);
String filepath=".\\scrapedDataFiles\\data.xlsx";
FileOutputStream fos= new FileOutputStream(filepath);
//Thread.sleep(4000);
workbook.write(fos);
workbook.close();
}
}