Tried reading Request Headers with Browsermob + Selenium in Java but only got empty har

Question

My goal is to read the request headers of the app portal.
Am using browsermob-core with selenium, but the entries in har always comes empty.
I tried removing the headless arg and also tried using a separate SSLProxy for BMP, but still the har is empty. Heres my code:

package com.example;

import io.github.bonigarcia.wdm.WebDriverManager;
import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.BrowserMobProxyServer;
import net.lightbody.bmp.client.ClientUtil;
import net.lightbody.bmp.core.har.Har;
import net.lightbody.bmp.core.har.HarEntry;
import net.lightbody.bmp.proxy.CaptureType;
import org.openqa.selenium.By;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class webDriver {
    private static final Logger logger
            = LoggerFactory.getLogger(webDriver.class);
            //= LoggerFactory.getLogger(ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME);

    public static void generateUIToken() throws Exception {
        
        WebDriver driver = null;
        BrowserMobProxy proxy = new BrowserMobProxyServer();
        proxy.setTrustAllServers(true);
        proxy.start(0);
        Har har;
        Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
        String hostIp = "localhost";
        seleniumProxy.setHttpProxy(hostIp + ":" + proxy.getPort());
        seleniumProxy.setSslProxy(hostIp + ":" + proxy.getPort());
        seleniumProxy.setSslProxy("trustAllSSLCertificates");

        WebDriverManager.chromedriver().setup();
        ChromeOptions options = new ChromeOptions();
        options.addArguments("--no-sandbox");
        options.addArguments("--headless");
        options.addArguments("--ignore-certificate-errors");
        options.addArguments("--disable-dev-shm-usage");

        DesiredCapabilities capabilities = new DesiredCapabilities();
        capabilities.setCapability(CapabilityType.PROXY, seleniumProxy);
        capabilities.setCapability(CapabilityType.ACCEPT_INSECURE_CERTS,true);
        capabilities.setCapability(CapabilityType.ACCEPT_SSL_CERTS,true);
        options.merge(capabilities);

        try {

            driver = new ChromeDriver(options);



            proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);

            proxy.setHarCaptureTypes(CaptureType.REQUEST_HEADERS, CaptureType.RESPONSE_HEADERS);
            proxy.enableHarCaptureTypes(CaptureType.REQUEST_HEADERS,CaptureType.RESPONSE_HEADERS);
            proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
            proxy.newHar("new.example.com");

            driver.get("https://new.example.com");

            String title = driver.getTitle();
            TimeUnit.SECONDS.sleep(10);
            WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(20));
            wait.until(ExpectedConditions.elementToBeClickable(By.id("userInput")));
            WebElement user_login = driver.findElement(By.id("userInput"));
            user_login.sendKeys("username");
            user_login.submit();
            logger.info(user_login + " - " + title);
            TimeUnit.SECONDS.sleep(10);

            wait.until(ExpectedConditions.elementToBeClickable(By.id("okta-signin-password")));
            WebElement pass_login = driver.findElement(By.id("okta-signin-password"));
            pass_login.sendKeys("password");
            pass_login.submit();
            logger.info(pass_login + " - " + title);

            TimeUnit.SECONDS.sleep(30);

            logger.info("Current" + driver.getCurrentUrl());
            har = proxy.getHar();
            List<HarEntry> entries = har.getLog().getEntries();
            for (HarEntry entry : entries) {
                logger.info("Request URL: " + entry.getRequest().getUrl());
                logger.info("Entry response status: " + entry.getResponse().getStatus());
                logger.info("Entry response text: " + entry.getResponse().getStatusText());

            }
            har.writeTo(new File("test.har"));
            TimeUnit.SECONDS.sleep(10);
            driver.quit();
            proxy.endHar();
            proxy.stop();

        }
        catch (Exception e){
            assert driver != null;
            driver.close();
            driver.quit();
        }
    }

    public static void main(String[] args) throws Exception {

        generateUIToken();


    }
}

And heres the output in har file:

{"log":{"version":"1.2","creator":{"name":"BrowserMob Proxy","version":"2.1.5","comment":""},"pages":[{"id":"new.example.com","startedDateTime":"2022-12-06T21:46:04.043Z","title":"new.example.com","pageTimings":{"comment":""},"comment":""}],"entries":[],"comment":""}}

Am using Java 17 and browsermob-core 2.1.5 with Selenium 4.7.0
Can anyone help me figure out why the har is always having empty entries?

Answer 1

Its exactly not answer your question but your problem might solve in other way. Selenium can intercept network and get request and response header. In selenium have 4.0.6 have that capablity.

    <dependency>
    
      <groupId>org.seleniumhq.selenium</groupId>
    
      <artifactId>selenium-java</artifactId>
    
      <version>4.6.0</version>

</dependency>

Here is the full code for how to intercept

public static void main(String[] args) throws InterruptedException, AWTException {

    ChromeOptions options = new ChromeOptions();

    ChromeDriver driver = new ChromeDriver(options);
    driver.get("https://www.google.com");
    DevTools devTools = driver.getDevTools();
    devTools.createSession();

    devTools.send(Network.enable(Optional.empty(),Optional.empty(),Optional.empty()));

    devTools.addListener(Network.requestWillBeSent(),
            request ->{
                System.out.println("Request URL:"+request.getRequest().getUrl());
                System.out.println("Request Method:"+request.getRequest().getMethod());
                System.out.println("Request Method:"+request.getRequest().getHeaders().toJson());
            });
    Thread.sleep(50000);
}

Now you can save all request inside a file a JSON or har extension