How to upload multiple files to GoogleDrive API using for/loop (forEach)

Question

I use the Google Drive API to upload multiple files. I faced with a problem running out of RAM while uploading multiples files. I try to use forEach (for loop) for my code to avoid uploading multiple files at the same time, but It doesn't work the way I expected. It always loop through the entire of list files and upload the same time.

I try to use async/await syntax to block the loop but It didn't work the way I expected. Here is my code:

const fs = require("fs");
const readline = require("readline");
const { google } = require("googleapis");

let files = ["file1.mp4", "file2.mp4"];

const SCOPES = ["https://www.googleapis.com/auth/drive.metadata.readonly"];

const TOKEN_PATH = "token.json";
fs.readFile("credentials.json", (err, content) => {
  if (err) return console.log("Error loading client secret file:", err);
  // Authorize a client with credentials, then call the Google Drive API.
  authorize(JSON.parse(content), uploadFiles);
});

function authorize(credentials, callback) {
  const { client_secret, client_id, redirect_uris } = credentials.installed;
  const oAuth2Client = new google.auth.OAuth2(
    client_id,
    client_secret,
    redirect_uris[0]
  );

  // Check if we have previously stored a token.
  fs.readFile(TOKEN_PATH, (err, token) => {
    if (err) return getAccessToken(oAuth2Client, callback);
    oAuth2Client.setCredentials(JSON.parse(token));
    callback(oAuth2Client);
  });
}

function getAccessToken(oAuth2Client, callback) {
  const authUrl = oAuth2Client.generateAuthUrl({
    access_type: "offline",
    scope: SCOPES
  });
  console.log("Authorize this app by visiting this url:", authUrl);
  const rl = readline.createInterface({
    input: process.stdin,
    output: process.stdout
  });
  rl.question("Enter the code from that page here: ", code => {
    rl.close();
    oAuth2Client.getToken(code, (err, token) => {
      if (err) return console.error("Error retrieving access token", err);
      oAuth2Client.setCredentials(token);
      // Store the token to disk for later program executions
      fs.writeFile(TOKEN_PATH, JSON.stringify(token), err => {
        if (err) console.error(err);
        console.log("Token stored to", TOKEN_PATH);
      });
      callback(oAuth2Client);
    });
  });
}

async function uploadFiles(auth) {
  for (file of files) {
    var fileMetadata = {
      name: file
    };
    var media = {
      body: fs.createReadStream("test/" + file)
    };
    google.drive({ version: "v3", auth });
    const result = await drive.files.create(
      {
        resource: fileMetadata,
        media: media,
        fields: "id"
      },
      function(err, fileid) {
        if (err) {
          // Handle error
          console.error(err);
        } else {
          console.log("File Id: ", fileid.data.id);
          console.log("Uploaded..:" + file);
        }
      }
    );
    console.log("Uploading file..:" + file);
  }
}

I just want to ask why the loop does not upload single files each?

Answer 1

I try to use forEach (for loop) for my code to avoid uploading multiple files at the same time

You can't, the process is entirely asynchronous. You passed a callback as an argument to the function drive.files.create.

By the way, if you want to use async/await, you should wrap your function into a promisified one.

function myCreateFunc (fileInfos) {
    return new Promise((resolve, reject) => {
            google.drive.create(filesInfos, function callback(err, fileId) {
                    if(err)
                        reject(err)

                    resolve(fileId)
            })
    });
}

Answer 2

Even after a long time, I will post my answer because I had the same problem.

Requirements:

1. Enable APIS and Services for your project.
2. Create a service account and download the key.
3. If you need to upload your files into a folder, you need the folder id.
4. Install googleapis using npm install googleapis.

Follow these steps to get your folder id.

1. To find folder id, you need to provide permission to the service account user.
2. Share it with edit access and get the link to the folder.
3. You will see something like this: https://drive.google.com/drive/u/1/folders/xxxxxXXXXXXxxxxxxxXXXXXXX
4. In this case folder id is xxxxxXXXXXXxxxxxxxXXXXXXX

    const { google } = require("googleapis");
    var fs = require("fs");

    const KEYFILEPATH = "path to your keyfile.json";
    const SCOPES = ["https://www.googleapis.com/auth/drive"];

    const auth = new google.auth.GoogleAuth({
      keyFile: KEYFILEPATH,
      scopes: SCOPES,
    });

    const uploadFiles = async (auth) => {
      const driveService = google.drive({ version: "v3", auth });
      let count = 0;

      for (const file of fs.readdirSync(
        "full file path to your images/files folder"
      )) {
        // Log the file name.
        console.log(JSON.stringify(file));

        let fileMetadata = {
          name: file,
          parents: ["folder id"], //Optional
        };

        let media = {
          mimeType: "image/jpeg",
          body: fs.createReadStream(
            `full file path to your images/files folder/${file}`
          ),
        };

        const task = driveService.files.create({
          resource: fileMetadata,
          media: media,
          fields: "id",
        });

        try {
          await task;
          count = count + 1;
        } catch (e) {
          console.log(e);
          return null;
        }
      }
      // log the total count of uploaded files.
      console.log("Count :", count);
      return;
    };

    uploadFiles(auth).catch(console.error);