I'm trying to use S3, specifically multer-s3, for image upload for a traditional web app that currently has multer/file system file upload (GitHub repo with previous code before the failed S3 upload attempt can be found here). The app is deployed to Heroku, which has ephemeral file storage, so the old setup is a no-go.
I tried to use multer-s3 to do it based on this tutorial https://www.youtube.com/watch?v=ASuU4km3VHE&t=1364s, but got up to about 20 mins in, trying to send the POST request to the new image-upload route but am getting a 500 error, whereas in the tutorial an AWS image path is provided in the response.
Here's what I tried so far:
I created a bucket and get my access code and keys. In my S3 bucket settings, under Permissions -> Block public access, I set everything to off. I also added CORS config code as suggested by Heroku here (but I still get a 500 error without it).
In the util folder, I added a file named file-upload.js with this code (I'm using a Nodemon.json file for the config keys):
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const { uuid } = require('uuidv4');
aws.config.update({
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
region: 'us-west-2',
});
const s3 = new aws.S3();
const upload = multer({
storage: multerS3({
s3,
bucket: 'nodejs-shop',
acl: 'public-read',
// Called when saving image to AWS
metadata(req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
// Called before saving image to AWS
key(req, file, cb) {
cb(null, uuid());
},
}),
});
module.exports = upload;
In the routes folder, I added a file named file-upload.js, with this code:
const express = require('express');
const router = express.Router();
const upload = require('../util/file-upload');
// Will send image under this key 'image' in request to server
const singleUpload = upload.single('image');
router.post('/image-upload', (req, res, next) => {
// Callback function called after image is uploaded or will get error from server
singleUpload(req, res, (err) => {
return res.json({ imageUrl: req.file.location });
});
});
module.exports = router;
In app.js, I imported the routes file const fileRoutes = require('./routes/file-upload'); and added the middleware after the authRoutes middleware app.use(fileRoutes);. I also commented out all the previously used multer code in app.js.
Current app.js code:
const path = require('path');
const fs = require('fs');
// const https = require('https');
const express = require('express');
const bodyParser = require('body-parser');
const mongoose = require('mongoose');
const session = require('express-session');
const MongoDBStore = require('connect-mongodb-session')(session);
const csrf = require('csurf');
const flash = require('connect-flash');
// const multer = require('multer');
// const { uuid } = require('uuidv4');
const helmet = require('helmet');
const compression = require('compression');
const morgan = require('morgan');
const errorController = require('./controllers/error');
const User = require('./models/user');
const MONGODB_URI =
// process object is globally available in Node app; part of Node core runtime. The env property contains all environment variables known by process object. Using nodemon.json to store environment variables, but could alternatively use dotenv package for this (see https://www.youtube.com/watch?v=17UVejOw3zA)
`mongodb+srv://${process.env.MONGO_USER}:${process.env.MONGO_PASSWORD}@cluster0-4yuid.mongodb.net/${process.env.MONGO_DEFAULT_DATABASE}`;
const app = express();
const store = new MongoDBStore({
uri: MONGODB_URI,
collection: 'sessions',
});
// Secret used for signing/hashing token is stored in session by default
const csrfProtection = csrf();
// Don't want to start server until file is read in, thus using synchronous version
// const privateKey = fs.readFileSync('server.key');
// const certificate = fs.readFileSync('server.cert');
// Commenting out original file upload method since changed to use AWS S3 for image upload/hosting
// const fileStorage = multer.diskStorage({
// destination: (req, file, cb) => {
// // First arg is for error message to throw to inform multer something is wrong with incoming file and it should not store it; with null, telling multer okay to store it
// cb(null, 'images');
// },
// filename: (req, file, cb) => {
// cb(null, uuid());
// },
// });
// const fileFilter = (req, file, cb) => {
// file.mimetype === 'image/png' ||
// file.mimetype === 'image/jpg' ||
// file.mimetype === 'image/jpeg'
// ? cb(null, true)
// : cb(null, false);
// };
app.set('view engine', 'ejs');
// Setting this explicity even though the views folder in main directory is where the view engine looks for views by default
app.set('views', 'views');
const adminRoutes = require('./routes/admin');
const shopRoutes = require('./routes/shop');
const authRoutes = require('./routes/auth');
const fileRoutes = require('./routes/file-upload');
// Create write stream (for passing to morgan, used to log request data), for logging request data in file instead of console
// flags: 'a': a is for append; new data will be appended to that file (additional log statements are added to end of existing file rather than overwriting it)
const accessLogStream = fs.createWriteStream(
path.join(__dirname, 'access.log'),
{ flags: 'a' }
);
// Set secure response header(s) with Helmet
// In my app, in developer tools (in the network tab) I can see it added one additional response header for localhost, Strict-Transport-Security. This HTTP header tells browsers to stick with HTTPS and never visit the insecure HTTP version. Once a browser sees this header, it will only visit the site over HTTPS for the next 60 days
app.use(helmet());
// Compress assets. Note: Compression is normally done by hosting providers, but deploying to Heroku which does offer it
app.use(compression());
// Log request data using writable file stream created above. Which data is logged and how to format it is passed into funtion
// Also normally handled by hosting providers
// app.use(morgan('combined', { stream: accessLogStream }));
app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json());
// Commented out since changed to use AWS S3 for image upload/hosting
// app.use(multer({ storage: fileStorage, fileFilter }).single('image'));
app.use(express.static(path.join(__dirname, 'public')));
app.use('/images', express.static(path.join(__dirname, 'images')));
app.use(
session({
secret: 'my secret',
resave: false,
saveUninitialized: false,
store,
})
);
app.use(csrfProtection);
app.use(flash());
app.use((req, res, next) => {
// Locals field: Express feature for setting local variables that are passed into views. For every request that is executed, these fields are set for view that is rendered
res.locals.isAuthenticated = req.session.isLoggedIn;
res.locals.csrfToken = req.csrfToken();
next();
});
app.use((req, res, next) => {
// When you throw an error in synchronous places (outside of callbacks and promises), Express will detect this and execute next error handling middleware. But if error is thrown within async code (in then or catch block), Express error handling middleware won't be executed; app will simply crash; have to use next()
// throw new Error('sync dummy');
if (!req.session.user) {
return next();
}
User.findById(req.session.user._id)
.then((user) => {
if (!user) {
return next();
}
req.user = user;
next();
})
// catch block will be executed in the case of technical issue (e.g., database down, or insufficient permissions to execute findById())
.catch((err) => {
// Within async code snippets, need to use next wrapping error, outside you can throw error
next(new Error(err));
});
});
app.use('/admin', adminRoutes);
app.use(shopRoutes);
app.use(authRoutes);
app.use(fileRoutes);
app.get('/500', errorController.get500);
app.use(errorController.get404);
// Error-handling middleware. Express executes this middleware when you call next() with an error passed to it
app.use((error, req, res, next) => {
// res.status(error.httpStatusCode).render(...);
// res.redirect('/500');
res.status(500).render('500', {
pageTitle: 'Server Error',
path: '/500',
isAuthenticated: req.session.isLoggedIn,
});
});
mongoose
.connect(MONGODB_URI, { useUnifiedTopology: true, useNewUrlParser: true })
.then((result) => {
// First arg for createServer() configures server, second is request handler, in this case, Express application
// Commenting out because just as with request logging and asset compression, it's handled by hosting provider, and browsers don't accept custom/self-signed certificate; will be displayed as insecure with a message that connection is not private
// https
// .createServer({ key: privateKey, cert: certificate }, app)
// .listen(process.env.PORT || 3000);
app.listen(process.env.PORT || 3000);
})
.catch((err) => {
console.log(err);
});
This is my Postman request, similar to the one in the tutorial video, and as you can see I just get a 500 error.
