SpookyJs show all HTTP headers

Question

I have a site that is behaving differently when I interact with it in a traditional browser versus spooky/casper/phantom. I would like to debug by printing out the entire http header requests and responses to the console or a file to see what is different between my browser and the phantom browser (similar to how I would using developer tools on the browser). How can I get all http request / response including headers in a spooky event handler?

Answer 1

I'm controlling CasperJS from a node module via SpookyJS - so using the advice of Artjom B. I was able to determine that I just needed to add an event listener to the CasperJS options JSON passed into SpookyJS when building the object. For anyone using SpookyJS, that code would look roughly like the following:

var Spooky = require( 'spooky' );

var spooky = new Spooky(
      {
        child: {
            'transport'         : 'http'
        },
        casper: {
            logLevel: 'debug'
          , verbose: true
          , onResourceRequested : function( C, requestData, request ){ this.emit('console', JSON.stringify( requestData ) ) }
          , onResourceReceived  : function( C, response ){ this.emit('console', JSON.stringify( response ) ) }
        }
      }, function ( err ) {
        if ( err ) {
            var e = new Error( 'Failed to initialize SpookyJS' );
            e.details = err;
            throw e;
        }

        spooky.start( "www.something.com" );

        spooky.run();
     }
);

spooky.on('console', function (line) {
   console.log(line);
});

Answer 2

In CasperJS you can listen to a couple of events to give you additional information.

casper.on('resource.requested', function(requestData, request) {
    this.echo('Request (#' + requestData.id + '): Headers' + JSON.stringify(requestData.headers, undefined, 4));
});

See page.onResourceRequested for more information.

---

Additionally, you should capture as many screenshots with casper.capture() as it makes sense in order to understand what is going on.

There are also some events that could help you get to see more errors:

// http://docs.casperjs.org/en/latest/events-filters.html#remote-message
casper.on("remote.message", function(msg) {
    this.echo("Console: " + msg);
});

// http://docs.casperjs.org/en/latest/events-filters.html#page-error
casper.on("page.error", function(msg, trace) {
    this.echo("Error: " + msg);
    // maybe make it a little fancier with the code from the PhantomJS equivalent
});

// http://docs.casperjs.org/en/latest/events-filters.html#resource-error
casper.on("resource.error", function(resourceError) {
    this.echo("ResourceError: " + JSON.stringify(resourceError, undefined, 4));
});

// http://docs.casperjs.org/en/latest/events-filters.html#page-initialized
casper.on("page.initialized", function(page) {
    // CasperJS doesn't provide `onResourceTimeout`, so it must be set through 
    // the PhantomJS means. This is only possible when the page is initialized
    page.onResourceTimeout = function(request) {
        console.log('Response Timeout (#' + request.id + '): ' + JSON.stringify(request));
    };
});