Using AWS4 Signature via Postman for CRUD Elastic operations

Question

I'm trying to post data to Elasticsearch managed by AWS using AWS4 signing method. I would like to achieve this via postman pre-script. I tried using below script which worked perfectly for GET operation of Elastic search but its not working for POST or PUT or DELETE operation & keep giving me error message that the signature does not match for POST operation. Can someone help me in fixing below pre-script in postman?

var date = new Date().toISOString();
var amzdate = date.replace(/[:\-]|\.\d{3}/g, "");
var dateStamp = amzdate.slice(0, -8);

pm.environment.set('authorization', getAuthHeader(request.method, request.url, request.data));
pm.environment.set('xAmzDate', amzdate);

function getPath(url) {
    var pathRegex = /.+?\:\/\/.+?(\/.+?)(?:#|\?|$)/;
    var result = url.match(pathRegex);
    return result && result.length > 1 ? result[1] : '';
}

function getQueryString(url) {
    var arrSplit = url.split('?');
    return arrSplit.length > 1 ? url.substring(url.indexOf('?') + 1) : '';
}

function getSignatureKey(secretKey, dateStamp, regionName, serviceName) {
    var kDate = sign("AWS4" + secretKey, dateStamp);
    var kRegion = sign(kDate, regionName);
    var kService = sign(kRegion, serviceName);
    var kSigning = sign(kService, "aws4_request");
    return kSigning;
}

function sign(key, message) {
    return CryptoJS.HmacSHA256(message, key);
}

function getAuthHeader(httpMethod, requestUrl, requestBody) {
    var ACCESS_KEY = pm.globals.get("access_key");
    var SECRET_KEY = pm.globals.get("secret_key");
    var REGION = 'us-east-1';
    var SERVICE = 'es';
    var ALGORITHM = 'AWS4-HMAC-SHA256';

    var canonicalUri = getPath(requestUrl);
    var canonicalQueryString = getQueryString(requestUrl);

    if (httpMethod == 'GET' || !requestBody) {
        requestBody = '';
    } else {
        requestBody = JSON.stringify(requestBody);
    }

    var hashedPayload = CryptoJS.enc.Hex.stringify(CryptoJS.SHA256(requestBody));

    var canonicalHeaders = 'host:' + pm.environment.get("ESHost") + '\n' + 'x-amz-date:' + amzdate + '\n';
    var signedHeaders = 'host;x-amz-date';
    var canonicalRequestData = [httpMethod, canonicalUri, canonicalQueryString, canonicalHeaders, signedHeaders, hashedPayload].join("\n");
    var hashedRequestData = CryptoJS.enc.Hex.stringify(CryptoJS.SHA256(canonicalRequestData));

    var credentialScope = dateStamp + '/' + REGION + '/' + SERVICE + '/' + 'aws4_request';
    var stringToSign = ALGORITHM + '\n' + amzdate + '\n' + credentialScope + '\n' + hashedRequestData;

    var signingKey = getSignatureKey(SECRET_KEY, dateStamp, REGION, SERVICE);
    var signature = CryptoJS.HmacSHA256(stringToSign, signingKey).toString(CryptoJS.enc.Hex);

    var authHeader = ALGORITHM + ' ' + 'Credential=' + ACCESS_KEY + '/' + credentialScope + ', ' + 'SignedHeaders=' + signedHeaders + ', ' + 'Signature=' + signature;
    return authHeader;
}

Update: The issue is only for POST and PUT operations. DELETE works fine with above pre-script. When searching elastic search using POST does not match signatures. — Freephone Panwal, Feb 04 '19 at 06:18

score 3 · Accepted Answer · answered Feb 20 '19 at 16:24

The code from the OP is almost accurate just has a few bugs

1) getPath should return "/" when path=''
2) check if request.data is empty object if so requestBody = ''
3) no need to do JSON.stringify(request.data) since request.data returns a json string

The fixed snippet is below:

var date = new Date().toISOString();
var amzdate = date.replace(/[:\-]|\.\d{3}/g, "");
var dateStamp = amzdate.slice(0, -8);


pm.environment.set('authorization', getAuthHeader(request.method, request.url, request.data));
pm.environment.set('xAmzDate', amzdate);

function getPath(url) {
    var pathRegex = /.+?\:\/\/.+?(\/.+?)(?:#|\?|$)/;
    var result = url.match(pathRegex);
    return result && result.length > 1 ? result[1] : '/';
}

function getQueryString(url) {
    var arrSplit = url.split('?');
    return arrSplit.length > 1 ? url.substring(url.indexOf('?') + 1) : '';
}

function getSignatureKey(secretKey, dateStamp, regionName, serviceName) {
    var kDate = sign("AWS4" + secretKey, dateStamp);
    var kRegion = sign(kDate, regionName);
    var kService = sign(kRegion, serviceName);
    var kSigning = sign(kService, "aws4_request");
    return kSigning;
}

function sign(key, message) {
    return CryptoJS.HmacSHA256(message, key);
}

function getAuthHeader(httpMethod, requestUrl, requestBody) {
    var ACCESS_KEY = pm.globals.get("access_key");
    var SECRET_KEY = pm.globals.get("secret_key");
    var REGION = 'us-east-1';
    var SERVICE = 'es';
    var ALGORITHM = 'AWS4-HMAC-SHA256';

    var canonicalUri = getPath(requestUrl);
    var canonicalQueryString = getQueryString(requestUrl);


    if (httpMethod == 'GET' || !requestBody || Object.keys(requestBody).length === 0) {
        requestBody = '';
    } 

    var hashedPayload = CryptoJS.enc.Hex.stringify(CryptoJS.SHA256(requestBody));

    var canonicalHeaders = 'host:' + pm.environment.get("ESHost") + '\n' + 'x-amz-date:' + amzdate + '\n';
    var signedHeaders = 'host;x-amz-date';
    var canonicalRequestData = [httpMethod, canonicalUri, canonicalQueryString, canonicalHeaders, signedHeaders, hashedPayload].join("\n");
    var hashedRequestData = CryptoJS.enc.Hex.stringify(CryptoJS.SHA256(canonicalRequestData));

    var credentialScope = dateStamp + '/' + REGION + '/' + SERVICE + '/' + 'aws4_request';
    var stringToSign = ALGORITHM + '\n' + amzdate + '\n' + credentialScope + '\n' + hashedRequestData;

    var signingKey = getSignatureKey(SECRET_KEY, dateStamp, REGION, SERVICE);
    var signature = CryptoJS.HmacSHA256(stringToSign, signingKey).toString(CryptoJS.enc.Hex);

    var authHeader = ALGORITHM + ' ' + 'Credential=' + ACCESS_KEY + '/' + credentialScope + ', ' + 'SignedHeaders=' + signedHeaders + ', ' + 'Signature=' + signature;
    return authHeader;
}

score 0 · Answer 2 · answered Feb 03 '19 at 21:29

When setting up a CloudWatch Logs to Amazon Elasticsearch stream, AWS creates a Node.js Lambda function which does proper AWS SigV4 URL signing. Here's the relevant part from that script that you could reuse to properly generate your postman request:

function buildRequest(endpoint, body) {
  var endpointParts = endpoint.match(/^([^\.]+)\.?([^\.]*)\.?([^\.]*)\.amazonaws\.com$/);
  var region = endpointParts[2];
  var service = endpointParts[3];
  var datetime = (new Date()).toISOString().replace(/[:\-]|\.\d{3}/g, '');
  var date = datetime.substr(0, 8);
  var kDate = hmac('AWS4' + process.env.AWS_SECRET_ACCESS_KEY, date);
  var kRegion = hmac(kDate, region);
  var kService = hmac(kRegion, service);
  var kSigning = hmac(kService, 'aws4_request');

  var request = {
    host: endpoint,
    method: 'POST',
    path: '/_bulk',
    body: body,
    headers: { 
      'Content-Type': 'application/json',
      'Host': endpoint,
      'Content-Length': Buffer.byteLength(body),
      'X-Amz-Security-Token': process.env.AWS_SESSION_TOKEN,
      'X-Amz-Date': datetime
    }
  };

  var canonicalHeaders = Object.keys(request.headers)
    .sort(function(a, b) { return a.toLowerCase() < b.toLowerCase() ? -1 : 1; })
    .map(function(k) { return k.toLowerCase() + ':' + request.headers[k]; })
    .join('\n');

  var signedHeaders = Object.keys(request.headers)
    .map(function(k) { return k.toLowerCase(); })
    .sort()
    .join(';');

  var canonicalString = [
    request.method,
    request.path, '',
    canonicalHeaders, '',
    signedHeaders,
    hash(request.body, 'hex'),
  ].join('\n');

  var credentialString = [ date, region, service, 'aws4_request' ].join('/');

  var stringToSign = [
    'AWS4-HMAC-SHA256',
    datetime,
    credentialString,
    hash(canonicalString, 'hex')
  ] .join('\n');

  request.headers.Authorization = [
    'AWS4-HMAC-SHA256 Credential=' + process.env.AWS_ACCESS_KEY_ID + '/' + credentialString,
    'SignedHeaders=' + signedHeaders,
    'Signature=' + hmac(kSigning, stringToSign, 'hex')
  ].join(', ');

  return request;
}

function hmac(key, str, encoding) {
  return crypto.createHmac('sha256', key).update(str, 'utf8').digest(encoding);
}

function hash(str, encoding) {
  return crypto.createHash('sha256').update(str, 'utf8').digest(encoding);
}

I compared both above and mine but somehow I don't see any difference. Can you point me explicitly where the issue could be? I've been struggling to make it work from last 2 days. — Freephone Panwal, Feb 04 '19 at 05:06
It's probably something small that's hard to spot. If you can, you may want to try my function. It definitely works with POST. — Daniel Vassallo, Feb 04 '19 at 19:38
I don't think I can copy your function exactly as it is as in Postman as postman uses different crypto function unless I'm missing something. Also, I noticed that your canonicalString contains 7 params which seems to be different than what AWS has mentioned and has only 6 params. That one definitely gives me error with 7 params (2 of which are empty). However, if you can copy my function in postman, you'll be able to reproduce the issue quickly. Let me know if there is any other way I can use your function. — Freephone Panwal, Feb 04 '19 at 19:59
I tried your function (modified to use postman function) but no luck and same error: I want to post the modified function but can't post it in the comment, not sure how to post it here. — Freephone Panwal, Feb 04 '19 at 20:35
here is the modified version of your code I added: https://pastebin.com/EwFSMPvZ but its not working in postman and resulting in same signaure mismatch error :( Appreciate if you can take a look. Thanks in advance! — Freephone Panwal, Feb 04 '19 at 21:18

Using AWS4 Signature via Postman for CRUD Elastic operations

2 Answers2