DAX Client JS with Lambda triggers multiple errors such as ECONRESET, Client does not have permission to invoke DefineKeySchema

Question

Hi I have a Dax cluster on top of a DynamoDB. Everything is set in the same VPC and subnets are created correctly. Also the permissions are setup correcty:

Lambda has full access to dynamodb:* and dax:*
Dax cluster policy has full access for DynamoDB - dynamodb:* and also a trust policy sts:assumeRole for dax service.

This is how I connect to Dax client inside my Lambda:

const AWS = require("aws-sdk");
const AmazonDaxClient = require("amazon-dax-client"); //VERSION: 1.2.9
let docClient = new AWS.DynamoDB.DocumentClient({
    region: "eu-west-1",
    convertEmptyValues: true,
  });
let daxService = null;
let daxClient = null;

async function connectToDAX(useDAX = false, force = false) {
  try {
    if (!useDAX) {
      return docClient;
    } else {
      if (force) {
        daxService = null;
        daxClient = null;
      }
      if (daxService == null) {
        const endpoint = await amazon.getParam(
          `/${lambdaService.getCurrentStage()}/project/aws/daxClusterEndpoint`
        );
        const region = await amazon.getParam(
          `/${lambdaService.getCurrentStage()}/prjoect/region`
        );
        const daxConfig = {
          endpoints: [endpoint],
          region: region,
        };
        daxService = new AmazonDaxClient(daxConfig);
        daxClient = new AWS.DynamoDB.DocumentClient({
          service: daxService,
          region: "eu-west-1",
          convertEmptyValues: true,
        });
        console.log("created new DAX connection");
      }
      return daxClient;
    }
  } catch (e) {
    console.log("error on connectToDax", e);
    return null;
  }
}

async function queryTable(params, dax = false) {
  try {
    let client = dax ? daxClient : docClient;
    if (dax && !daxClient) {
      client = await connectToDAX(dax);
    }

    return await new Promise(function (resolve, reject) {
      client.query(params, function (err, data) {
        if (err) {
          console.log("queryTable", params);
          console.log("query error", err);
          reject(err);
        } else {
          resolve(data);
        }
      });
    });
  } catch (e) {
    console.log("error dynamodb queryTable", e);
    daxClient = null;
    daxService = null;
    console.log("retrying queryTable with new dax connection");
    return await queryTable(params, dax);
  }
}

So, I am sending like 100 requests at a time towards API where my Lambda is getting invoked and it calls queryTable().

I am getting a lot of errors, 30% of all requests are Errors, which I would like to fix. Errors like:

Unknown application error occurred

{"errorType":"Error","errorMessage":"read ECONNRESET","code":"ECONNRESET","errno":-104,"syscall":"read","stack":["Error: read ECONNRESET"," at TCP.onStreamRead (internal/stream_base_commons.js:209:20)"]}

 Client does not have permission to invoke DefineKeySchema

So I have spent a lot of time troubleshooting this. Went trough all permissions and setup, but if it was something wrong there, NONE of the request would be successful.

I am thinking that the problem is with the DAX Client connection when Lambda concurrent invocations are happening. I have tried to force each concurrent execution to create new Dax Client connection, but that resulted even with more errors.

I am retrying as you can see in the queryTable() function.

I don't know what to do anymore. Please give some input.

DAX Client JS with Lambda triggers multiple errors such as ECONRESET, Client does not have permission to invoke DefineKeySchema

0 Answers0