Get all HTTP requests with WebDriver/HtmlUnit

Question

I need to verify a request for testing via WebDriver. Unfortunately there is no easy way to do this as there is no native support. It seems like I should be able to use HtmlUnit to get requests but I have only been able to get responses. Is there a way to do this with HtmlUnit or do I need to setup something else like Browsermob Proxy? I am using Java to do this.

Thanks!

Answer 1

I've provided an example using HtmlUnit below:

final WebClient webClient = new WebClient(BrowserVersion.CHROME);
final HtmlPage loginPage = webClient.getPage("http://www.stackoverflow.com");
WebResponse response = loginPage.getWebResponse(); // the response loaded to create this page
WebRequest request = response.getWebRequest();    // the request used to load this page

Answer 2

If I understand your question correctly you want to see every request and response that is made by HTMLUnit.

If you are using windows download Fiddler http://www.telerik.com/fiddler

Set the http proxy setting for HTMLUnit to use Fiddler as a proxy.

        BrowserVersion bv = BrowserVersion.CHROME;
        bv.setUserAgent("Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36");
        webClient = new WebClient(bv, "127.0.0.1", 8888);

The above by itself will work for any site that does not use HTTPS

If you want to capture HTTPS traffic create the class below in your project

import java.security.AccessController;
import java.security.InvalidAlgorithmParameterException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.PrivilegedAction;
import java.security.Security;
import java.security.cert.X509Certificate;

import javax.net.ssl.ManagerFactoryParameters;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactorySpi;
import javax.net.ssl.X509TrustManager;

public final class XTrustProvider extends java.security.Provider
{
    /**
     * 
     */
    private static final long   serialVersionUID    = 1L;
    private final static String NAME        = "XTrustJSSE";
    private final static String INFO        = "XTrust JSSE Provider (implements trust factory with truststore validation disabled)";
    private final static double VERSION = 1.0D;

    @SuppressWarnings({ "unchecked", "rawtypes" })
    public XTrustProvider()
    {
        super(NAME, VERSION, INFO);

        AccessController.doPrivileged(new PrivilegedAction()
        {
            public Object run()
            {
                put("TrustManagerFactory." + TrustManagerFactoryImpl.getAlgorithm(), TrustManagerFactoryImpl.class.getName());
                return null;
            }
        });
    }

    public static void install()
    {
        if (Security.getProvider(NAME) == null)
        {
            Security.insertProviderAt(new XTrustProvider(), 2);
            Security.setProperty("ssl.TrustManagerFactory.algorithm", TrustManagerFactoryImpl.getAlgorithm());
        }
    }

    public final static class TrustManagerFactoryImpl extends TrustManagerFactorySpi
    {
        public TrustManagerFactoryImpl()
        {
        }

        public static String getAlgorithm()
        {
            return "XTrust509";
        }

        protected void engineInit(KeyStore keystore) throws KeyStoreException
        {
        }

        protected void engineInit(ManagerFactoryParameters mgrparams) throws InvalidAlgorithmParameterException
        {
            throw new InvalidAlgorithmParameterException(XTrustProvider.NAME + " does not use ManagerFactoryParameters");
        }

        protected TrustManager[] engineGetTrustManagers()
        {
            return new TrustManager[]
            { new X509TrustManager()
            {
                public X509Certificate[] getAcceptedIssuers()
                {
                    return null;
                }

                public void checkClientTrusted(X509Certificate[] certs, String authType)
                {
                }

                public void checkServerTrusted(X509Certificate[] certs, String authType)
                {
                }
            } };
        }
    }
}

Call the install method of the class

XTrustProvider.install();

Be sure to call the above method before HTMLUnit makes any http requests.

Now you will capture all the requests that are made by HTMLUnit including https requests.

If you run into any issues comment and I will help.

Answer 3

How about just something like this:

HtmlUnitDriver driver = new HtmlUnitDriver() {
  public Set<WebRequest> requests = new HashSet<>();
  @Override
  protected WebClient modifyWebClient(WebClient originalClient) {
    return new WebClient() {
      @Override
      public WebResponse getPage(WebWindow window, WebRequest request) {
        requests.add(request);
        return super.getPage(window, request)
      }
      @Override
      public WebResponse loadWebResponse(WebRequest request) {
        requests.add(request);
        return super.loadWebResponse(request);
      }
      // If it's really necessary for your use case, you can also override the "download" method in a similar way, but note that this is an internal API
    }
  }
};
driver.open("http://www.example.com/");
Set<WebRequest> requests = (Set<WebRequest>) driver.getClass().getField("requests").get(driver);
for (WebRequest request : requests) {
  System.out.println(request.getUrl().toString()); // or whatever you want
}

Of course, if the order is important, you could use a List instead of a Set, but then you'd have to check if the request was already there to avoid duplicates.