postman schema validation into reporter-htmlextra

Question

I'm currently running some tests with postman where I get a schema and try to validate my results against it. I know the schema is not consistent with the response I'm getting but I wanted to know how is it possible to expand the results to give a bit more information.

so for example if I have a request like this:

GET /OBJ/{ID}

it just fails with the feedback:

Schema is valid:
expected false to be true

I was hoping to manage to get a bit more feedback in my newman report

this is an example of my test:

pm.test("Status code is 200", function () {
  pm.response.to.have.status(200);
});

// only preform tests if response is successful 
if (pm.response.code === 200) {
  var jsonData = pm.response.json();

  pm.test("Data element contains an id", function () {
    var jsonData = pm.response.json();
    pm.expect(jsonData.id).eql(pm.environment.get("obj_id"));
  });

  pm.test('Schema is valid', function() {
    pm.expect(tv4.validate(jsonData, pm.globals.get("objSchema"))).to.be.true;
  });
}

and this is how I run my tests:

const newman = require('newman');

newman.run({
    insecure: true,
    collection: require('../resources/API.postman_collection.json'),
    environment: require('../resources/API.postman_environment.json'),
    reporters: 'htmlextra',
    reporter: {
        htmlextra: {
            export: './build/newman_report.html',
            logs: true,
            showOnlyFails: false,
            darkTheme: false
        }
    }
}, function (err) {
    if (err) { 
      throw err; 
    }
    console.log('collection run complete!');
});

is there a way I can get more information about the validation failure? I tried a few quick google search but have not come up to nothing that seemed meaningful

score 0 · Answer 1 · answered Nov 27 '19 at 07:35

it's not exactly what I wanted but I managed to fix it with something like this:

// pre-check

var schemaUrl = pm.environment.get("ocSpecHost") + "type.schema";

pm.sendRequest(schemaUrl, function (err, response) {
  pm.globals.set("rspSchema", response.json());
});

// test

var basicCheck = () => {
  pm.test("Status code is 200", function () {
    pm.response.to.have.status(200);
  });

  pm.test("Response time is less than 200ms", function () {
    pm.expect(pm.response.responseTime).to.be.below(200);
  });
};

// create an error to get the output from the item validation
var outputItemError = (err) => {
  pm.test(`${err.schemaPath} ${err.dataPath}: ${err.message}`, function () {
    pm.expect(true).to.be.false; // just output the error
  });
};

var itemCheck = (item, allErrors) => {
  pm.test("Element contains an id", function () {
    pm.expect(item.id).not.eql(undefined);
  });
  var Ajv = require('ajv');
  ajv = new Ajv({
    allErrors: allErrors,
    logger: console
  });
  var valid = ajv.validate(pm.globals.get("rspSchema"), item);
  if (valid) {
    pm.test("Item is valid against schema", function () {
      pm.expect(valid).to.be.true; // just to output that schema was validated
    });
  } else {
    ajv.errors.forEach(err => outputItemError(err));
  }
};

// check for individual response
var individualCheck = (allErrors) => {
  // need to use eval to run this section
  basicCheck();
  // only preform tests if response is successful 
  if (pm.response.code === 200) {
    var jsonData = pm.response.json();

    pm.test("ID is expected ID", function () {
      var jsonData = pm.response.json();
      pm.expect(jsonData.id).eql(pm.environment.get("nextItemId"));
    });
    itemCheck(jsonData, allErrors);
  }
}
individualCheck(true);

just create a function to do an item test where I do a stupid assert.false to output each individual error in the schema path

postman schema validation into reporter-htmlextra

1 Answers1