How can I convert my CSV to JSON and save it to MongoDB when uploading multiple files in one form with different mimetypes?

Question

I'm trying to save a form to my MongoDB that includes both an image upload and a CSV upload. I'm using the Node modules Multer and CSVtoJSON to accomplish this. I've been able to accomplish each task on their own, but am struggling with getting them both to save to the database in the same form. After an innumerable amount of attempts, the code is currently not throwing any errors, I just get "Cannot POST /" in the browser. The files are properly saving to my uploads folder. I believe the failure is my logic to convert the file to JSON before creating the database entry.

Here's my app.js file:


    const express = require('express') ;
const app = express();

const dotenv = require('dotenv');
const bodyParser = require('body-parser'); 
const mongoose = require('mongoose');  
const fs = require('fs'); 
const path = require('path'); 
const multer = require('multer');
const csv = require('csvtojson');

const imageSchema = require('./models/Profile'); 

dotenv.config({ path: './config/config.env' });

mongoose.connect(process.env.MONGO_URI, { 
        useNewUrlParser: true, useUnifiedTopology: true 
    }, err => { 
        console.log('Database is connected') 
    }
); 

app.use(bodyParser.urlencoded({ extended: false })) 
app.use(bodyParser.json()) 
  
app.set("view engine", "ejs"); 

const storage = multer.diskStorage({
    destination: (req, file, cb) => { 
        if (file.fieldname === "image") { // if uploading resume
            cb(null, 'uploads/images');
        } else if (file.fieldname === "csv") { // else uploading image
            cb(null, 'uploads/csvs');
        } else {
            console.log("A CSV file is required to create a user profile.");
        }

        // if (file.mimetype=='image/jpeg' || file.mimetype=='image/png' || file.mimetype=='jpg') {
        //  cb(null, './uploads/images/');      
        // } else if (file.mimetype=='text/csv' || file.mimetype=='csv') {
        //  cb(null, './uploads/csvs/');
        // } else (file.mimetype != 'image/jpeg' || file.mimetype != 'image/png' || file.mimetype != 'jpg' || file.mimetype != 'text/csv' || file.mimetype != 'csv') {
        //  // cb(null, false)
        //  console.log("File type is not supported. Please upload correct file types.");
        // }
    }, 
    filename: (req, file, cb) => { 
        cb(null, Date.now() + '_' + file.originalname) 
    } 
}); 

const upload = multer({ storage }).fields([
    {name: 'image', maxCount: 1}, 
    {name: 'csv', maxCount: 1}
]); 

// Retriving the image 
app.get('/', (req, res) => { 
    imageSchema.find({}, (err, items) => { 
        if (err) { 
            console.log(err); 
        } 
        else { 
            res.render('app', { items }); 
        } 
    }); 
}); 

// Uploading the image 
app.post('/', upload), (req, res, next) => { 

    // const conversion = csv().fromFile(req.file.path).then((jsonObj) => {
    //  console.log(jsonObj);
    //  // csvSchema.insertMany(jsonObj,(err,data)=>{
    //     //     if(err){
    //     //         console.log(err);
    //     //     }
    //     // });
    // })

    // Convert CSV to json
    function csvToJson(filePath){
        csv()
            .fromFile(filePath)
            .then((jsonObj)=>{
                console.log(jsonObj);               
            })
    }

    const obj = { 
        name: req.body.name,  
        img: { 
            data: fs.readFileSync(path.join(__dirname + '/uploads/images' + req.file.filename)), 
            contentType: 'image/png image/jpeg'
        },
        csv: { 
            data: csvToJson(fs.readFileSync(path.join(__dirname + '/uploads/csvs' + req.file.filename))),
            contentType: 'text/csv'
        }
    }

    imageSchema.create(obj, (err, item) => { 
        if (err) {
            console.log(err); 
        } 
        else { 
            item.save(); 
            res.redirect('/'); 
        } 
    }); 

    fs.unlinkSync(filePath);
}; 


const PORT = process.env.PORT || 5000;

app.listen(PORT, () =>
    console.log(`Server running in ${process.env.NODE_ENV} mode on port ${PORT}`)
);

Here's a simplified version of my schema:


    var mongoose  =  require('mongoose');

var csvSchema = new mongoose.Schema({
    A: { type: Number },
    B: { type: Number },
    C: { type: Number },
    D: { type: Number },
    E: { type: Number }
});

var imageSchema = new mongoose.Schema({ 
    name: String,  
    img: 
    { 
        data: Buffer, 
        contentType: String 
    },
    csv: [csvSchema],
    createdAt: {type: Date, default: Date.now}
}); 

module.exports = mongoose.model('Image', imageSchema);

Here's my app.ejs file:


    <!DOCTYPE html> 
<html lang="en"> 

<head> 
    <meta charset="UTF-8"> 
    <meta name="viewport" content="width=device-width, initial-scale=1.0"> 
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.0/css/bulma.min.css">
    <link
        rel="stylesheet"
        href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/all.min.css"
        integrity="sha256-+N4/V/SbAFiW1MPBCXnfnP9QSN3+Keu+NlB+0ev/YKQ="
        crossorigin="anonymous"
    />
    <title>Create Profile</title> 
</head> 

<script>
    // 1. Display filename when select file
// 2. Clear filename when form reset

document.addEventListener('DOMContentLoaded', () => {
  // 1. Display file name when select file
  let fileInputs = document.querySelectorAll('.file.has-name')
  for (let fileInput of fileInputs) {
    let input = fileInput.querySelector('.file-input')
    let name = fileInput.querySelector('.file-name')
    input.addEventListener('change', () => {
      let files = input.files
      if (files.length === 0) {
        name.innerText = 'No file selected'
      } else {
        name.innerText = files[0].name
      }
    })
  }

  // 2. Remove file name when form reset
  let forms = document.getElementsByTagName('form')
  for (let form of forms) {
    form.addEventListener('reset', () => {
      console.log('a')
      let names = form.querySelectorAll('.file-name')
      for (let name of names) {
        name.innerText = 'No file selected'
      }
    })
  }
})
</script>

<body>  
    <section class="hero is-dark">
        <div class="hero-body">
            <div class="container">
                <div class="columns is-vcentered">
                    <span style="font-size: 80px;">
                        <i class="fas fa-user-alt size:7x"></i>
                    </span>
                    <div class="column">
                        <h1 class="title">
                            Create User Profile
                        </h1>
                        <h2 class="subtitle">
                            All fields are required
                        </h2>
                    </div>
                </div>
            </div>
        </div>
    </section>
    <section class="section">
        <div class="container">     
            <div class="columns"> 
                <div class="column is-two-fifths"> 
                    <box class="box">
                        <h1 class="title">Create Profile</h1> 
                        <hr>
                        <form class="form" action="/" method="POST" enctype="multipart/form-data"> 
                            <div class="field"> 
                                <label class="label" for="name">Name</label> 
                                <input class="input" type="text" id="name" placeholder="Enter your name"
                                    value="" name="name" required> 
                            </div> 
                            <div class="field"> 
                                <label class="label" for="image">Profile Picture</label>
                                <div class="file has-name is-link is-fullwidth"> 
                                    <input class="file-input" type="file" id="image"
                                        name="image" value="" required> 
                                        <label class="file-label" for="image">
                                            <span class="file-cta">
                                                <span class="file-icon">
                                                    <i class="fas fa-upload"></i>
                                                </span>
                                                <span class="file-label">
                                                    Choose an image...
                                                </span>
                                            </span>
                                            <span class="file-name"'>
                                                No file selected
                                            </span>
                                        </label>
                                    </input>   
                                </div>
                            </div> 
                            <div class="field">
                                <label class="label" for="csv">Data</label>
                                <div class="file has-name is-link is-fullwidth"> 
                                    <input class="file-input" type="file" id="csv"
                                        name="csv" value="" required> 
                                        <label class="file-label" for="csv">
                                            <span class="file-cta">
                                                <span class="file-icon">
                                                    <i class="fas fa-upload"></i>
                                                </span>
                                                <span class="file-label">
                                                    Choose a CSV file...
                                                </span>
                                            </span>
                                            <span class="file-name">
                                                No file selected
                                            </span>
                                        </label>
                                    </input>   
                                </div> 
                            </div><hr>
                            <div class="control has-text-centered"> 
                                <button class="button is-success" type="submit"><b>Submit</b></button> 
                            </div> 
                        </form>
                    </box>
                </div>   

                <hr> 

                <div class="column">                
                    <h1 class="title">Manage Profiles</h1>
                    <hr>                  
                    <div> 
                        <% items.forEach(function(image) { %> 
                            <box class="box">
                                <div> 
                                    <div> 
                                        <h5 class="title"><%= image.name %></h5> 
                                    </div><hr>
                                    <div class='card-content is-flex is-horizontal-center'>
                                        <img src="data:image/<%=image.img.contentType%>;base64, 
                                            <%=image.img.data.toString('base64')%>">
                                    </div>
                                    <div>
                                        <h6 class="subtitle"><%= image.csv %></h6> 
                                    </div>
                                </div>
                            </box> 
                        <% }) %> 
                    </div>                     
                </div>
            </div>
        </div>
    </section>
</body>

<style>
    section.section {
        padding: 30px 0px;
    }
    #hero-image {
        width: 99px;
    }
    .is-horizontal-center {
        justify-content: center;
    }
    button.button {
        width: 33%;
    }
</style>

</html>

Here's my package.json file:


    {
  "name": "combiningimgcsvuploads",
  "version": "1.0.0",
  "description": "",
  "main": "app.js",
  "scripts": {
    "start": "node app.js",
    "dev": "nodemon app.js"
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "body-parser": "^1.19.0",
    "csvtojson": "^2.0.10",
    "dotenv": "^8.2.0",
    "ejs": "^3.1.3",
    "express": "^4.17.1",
    "mongoose": "^5.9.21",
    "multer": "^1.4.2"
  },
  "devDependencies": {
    "nodemon": "^2.0.4"
  }
}

Any assistance that could be provided would be greatly appreciated. I'm open to any and all suggestions--even if that means a completely different approach, so long as it's more efficient.

kenput3r · Answer 1 · 2020-07-04T16:15:17.413

0

You likely need to wait for your csvToJson function to finish before moving on to the next code block. This can be done with async await. Also, I think you are missing a return statement. Try this:

    // Convert CSV to json
    async function csvToJson(filePath){
        csv()
            .fromFile(filePath)
            .then((jsonObj)=>{
                console.log(jsonObj);
                return jsonObj;              
            })
    }
   
    const json_data = await csvToJson(fs.readFileSync(path.join(__dirname + '/uploads/csvs' + req.file.filename))); 
    const json_data_string = JSON.stringify(json_data);

    const obj = { 
        name: req.body.name,  
        img: { 
            data: fs.readFileSync(path.join(__dirname + '/uploads/images' + req.file.filename)), 
            contentType: 'image/png image/jpeg'
        },
        csv: { 
            data: json_data_string,
            contentType: 'application/json'
        }
    }

edited Jul 04 '20 at 16:15

answered Jul 04 '20 at 14:46

kenput3r

229
2
9

Thank you for taking the time to respond to my question! I tried out your suggestion and also tried placing the json_data object in the async function; however, both yielded the same results: "Cannot POST /". Though your suggestion has given me further perspective and will provide food-for-thought--thank you! – jrfii Jul 04 '20 at 15:12
I'm assuming that csvToJson is return a valid JSON. Try stringifying your JSON before sending it off. Updated answer. – kenput3r Jul 04 '20 at 16:01
Also, I noticed you have your content type set to text/csv, but aren't you trying to send JSON? I also changed contentType: 'application/json'. I'm not 100% sure that it will be necessary to stringify the JSON at that point. – kenput3r Jul 04 '20 at 16:16
Thanks again! The result is still the same: "Cannot POST /". My app.js file currently reflects your suggestions, the only difference being that the json_data object is sitting within the async function, as required. I'm going to have to further toy around with things, but I think I'll need to step away from things for a little bit--I'm feeling the fatigue! :) – jrfii Jul 04 '20 at 16:21
Tried defining the contentType as 'application/json', with and without stringifying, same results as before. – jrfii Jul 04 '20 at 16:28
It's probably worth noting that it's breaking before getting to console.log(jsonObj). – jrfii Jul 04 '20 at 16:51

How can I convert my CSV to JSON and save it to MongoDB when uploading multiple files in one form with different mimetypes?

1 Answers1