aws upload object to S3 bucket and pass details of data to lambda

Question

Working my way through tutorials for AWS...So ive created an S3 bucket which when a file is dropped into it calls my lambda 'testHelloWorld' which sends an email...this all works fine (see below)

 'use strict';

console.log('Loading function');

var aws = require('aws-sdk');
var ses = new aws.SES({
   region: 'us-west-2'
});

exports.handler = function(event, context) {
    console.log("Incoming: ", event);
   // var output = querystring.parse(event);

    var eParams = {
        Destination: {
            ToAddresses: ["johnb@hotmail.com"]
        },
        Message: {
            Body: {
                Text: {
                    Data: "Hey! What is up?"
                }
            },
            Subject: {
                Data: "Email Subject!!!"
            }
        },
        Source: "johnb@hotmail.com"
    };

    console.log('===SENDING EMAIL===');
    var email = ses.sendEmail(eParams, function(err, data){
        if(err) console.log(err);
        else {
            console.log("===EMAIL SENT===");
            console.log(data);


            console.log("EMAIL CODE END");
            console.log('EMAIL: ', email);
            context.succeed(event);

        }
    });

};

but I want to extend the email to include data on the file that was uploaded to the bucket. I have found How to trigger my Lambda Function once the file is uploaded to s3 bucket which gives a node.js code snippet which should capture the data. I have tried to import this into my existing lambda

 'use strict';

console.log('Loading function');

var aws = require('aws-sdk');
var ses = new aws.SES({
   region: 'us-west-2'
});
var s3 = new aws.S3({ apiVersion: '2006-03-01', accessKeyId: process.env.ACCESS_KEY, secretAccessKey: process.env.SECRET_KEY, region: process.env.LAMBDA_REGION });

exports.handler = function(event, context, exit){
    console.log("Incoming: ", event);
   // var output = querystring.parse(event);


 // Get the object from the event and show its content type
   // const bucket = event.Records[0].s3.bucket.name;
   // const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
    const params = {
       Bucket: 'bucketName',
       Key: 'keyName',
       Source : 'SourceName',
       Destination : 'DestinationName',
       Message : 'MessageName'
    };


     s3.getObject(function(err, data){
         if (err) {
           console.log('ERROR ' + err);
          // exit(err);
         } else {
           // the data has the content of the uploaded file
           var eParams = {
        Destination: {
            ToAddresses: ["johnboy@hotmail.com"]
        },
        Message: {
            Body: {
                Text: {
                    Data: data
                }
            },
            Subject: {
                Data: "Email Subject!!!"
            }
        },
        Source: "johnboy@hotmail.com"
    };
         }
     });     


    console.log('===SENDING EMAIL===');
    var email = ses.sendEmail(eParams, function(err, data){
        if(err) console.log(err);
        else {
            console.log("===EMAIL SENT===");
            console.log(data);


            console.log("EMAIL CODE END");
            console.log('EMAIL: ', email);
            context.succeed(event);

        }
    });

};

but this is failing on the params

message: 'There were 3 validation errors:

* MissingRequiredParameter: Missing required key \'Source\' in params * MissingRequiredParameter: Missing required key \'Destination\' in params * MissingRequiredParameter: Missing required key \'Message\' in params', code: 'MultipleValidationErrors', errors:

These source, destination and message are listed in the params, are they not correctly formatted and it isnt picking them up?

I cant find much online....any help appreciated

UPDATE Ok iv got it working without failing...if i use the test function in the lambda with the following code...

   'use strict';

console.log('Loading function');

var aws = require('aws-sdk');
var ses = new aws.SES({
   region: 'us-west-2'
});

var s3 = new aws.S3({ apiVersion: '2006-03-01', accessKeyId: process.env.ACCESS_KEY, secretAccessKey: process.env.SECRET_KEY, region: process.env.LAMBDA_REGION });


exports.handler = function(event, context) {
    console.log("Incoming: ", event);
   // var output = querystring.parse(event);

   var testData = null;

   // Get the object from the event and show its content type
   // const bucket = event.Records[0].s3.bucket.name;
   // const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
    const params = {
       Bucket: 'bucket',
       Key: 'key',
    };

     s3.getObject(params, function(err, data){
         if (err) {
           console.log('ERROR ' + err);
           exit(err);
         } else {
           testData = data;
         }
     }); 

    var eParams = {
        Destination: {
            ToAddresses: ["jim@him.com"]
        },
        Message: {
            Body: {
                Text: { Data: 'testData2' + testData}
            },
            Subject: {
                Data: "Email Subject!!!"
            }
        },
        Source: "jim@him.com"
    };

    console.log('===SENDING EMAIL===');
    var email = ses.sendEmail(eParams, function(err, data){
        if(err) console.log(err);
        else {
            console.log("===EMAIL SENT===");
            console.log(data);


            console.log("EMAIL CODE END");
            console.log('EMAIL: ', email);
            context.succeed(event);

        }
    });

};

I get the email with the body- testData2null

So I tried uploading an image through the s3 bucket and I still get the email with the body testData2null

is there anyway to debug this further or does anyone kno who it is saying null. I never actually tested the code from the other post which passes the data over to the email I just assumed it would work. Does anyone else know who to obtain the data from the upload please? thanks

maxpaj · Accepted Answer · 2017-12-06T10:04:37.433

You are declaring the var eParams within the callback of s3.getObject, but then you run the ses.sendMail outside of the callback. I think that's why!

You also need to move the ses.sendEmail to inside the callback of s3.getObject if you want to send the data from your object inside the email.

Try this:

s3.getObject(function(err, objectData) {
    if (err) {
        console.log('Could not fetch object data: ', err);
    } else {
        console.log('Data was successfully fetched from object');
        var eParams = {
            Destination: {
                ToAddresses: ["johnboy@hotmail.com"]
            },
            Message: {
                Body: {
                    Text: {
                        Data: objectData
                    }
                },
                Subject: {
                    Data: "Email Subject!!!"
                }
            },
            Source: "johnboy@hotmail.com"
        };

        console.log('===SENDING EMAIL===');

        var email = ses.sendEmail(eParams, function(err, emailResult) {
            if (err) console.log('Error while sending email', err);
            else {
                console.log("===EMAIL SENT===");
                console.log(objectData);
                console.log("EMAIL CODE END");
                console.log('EMAIL: ', emailResult);
                context.succeed(event);
            }
        });
    }
});

@John No, I moved the `sendEmail` call to the right position in the code. If you check the opening and closing curly braces in the code, you'll see that the callback function of `getObject` is closed before you run the `sendEmail`. — maxpaj, Dec 06 '17 at 09:59
@John If you try my code, you'll probably solve the other problem you're having with getting null from data. You are using the `data` variable from the `getObject` call outside of the callback. — maxpaj, Dec 06 '17 at 10:03
thanks for the help. That solved that issue but now I am getting access denied from accessing the data. Again thank you, maybe if you have time you might know something about my current issue please https://stackoverflow.com/questions/47686447/access-denied-in-lambda-when-trying-to-access-file-uploaded-to-s3-bucket — John, Dec 07 '17 at 02:09

Aniket Thakur · Answer 2 · 2017-12-06T15:40:31.903

2

You need to read on how Nodejs works. It is event based and depends on callbacks and promises. You should do -

 s3.getObject(params, function(err, data){
      //This is your callback for s3 API call. DO stuff here
     if (err) {
       console.log('ERROR ' + err);
       exit(err);
     } else {
       testData = data;
        // Got your data. Send the mail here
     }
 });

I have added my comments in code above. Since Nodejs is single threaded it will make S3 api call and go ahead. When it is sending mail s3 api call is not complete so data is null. It is better to use promises here.

Anyway read up on callback and promises in nodejs and how it works. But hope this answers your logical error.

edited Dec 06 '17 at 15:40

answered Dec 06 '17 at 14:41

Aniket Thakur

66,731
38
279
289

Thank you for your help greatly appreciated I now understand node,js However I have marked the other answer as correct simply because they give a full example....However I now have another issue regarding access...if you have the time maybe you could take a look at it thank you again https://stackoverflow.com/questions/47686447/access-denied-in-lambda-when-trying-to-access-file-uploaded-to-s3-bucket – John Dec 07 '17 at 02:08

aws upload object to S3 bucket and pass details of data to lambda

2 Answers2

Linked