Any solution for file upload problem on Arduino ESP32 via http

Question

First the problem.

The user can upload file from the web with ajax. If the file is relatively big, the uploading takes a while. If the user's connection is lost or something happens during the uploading process, the file is going to be damaged or empty.

How should I secure the upload process so the file remains the same if it fails for some reason?

I'm using the following libraries on the Arduino ESP32:

I have a basic file upload handler on my esp32 which looks like this:

server.on("/uploading", HTTP_POST, [](AsyncWebServerRequest * request) {
  }, handleFileUpload);

void handleFileUpload(AsyncWebServerRequest * request, String filename,size_t index, uint8_t *data, size_t len, bool final) {
  if (!index) {
    if (!filename.startsWith("/"))
      filename = "/" + filename;
    if (LITTLEFS.exists(filename)) {
      LITTLEFS.remove(filename);
    }
    uploadFile = LITTLEFS.open(filename, "w");
  }
  for (size_t i = 0; i < len; i++) {
    uploadFile.write(data[i]);
  }
  if (final) {
    uploadFile.close();
    if(filename == "/myHomeProgram.json"){initProgram = true;}
    AsyncWebServerResponse *response = request->beginResponse(200, "text/plain", "File Uploaded;"+filename);
    response->addHeader("Access-Control-Allow-Origin","*");
    request->send(response);
  }

}

This is working pretty well, the files are uploaded correctly 99% of the cases, but if it fails I lost the file data, or if some other part of the program wants to open the same file it fails too. Should I write to a temporary file and after if it succeeded write the content to the intended file somehow? Here is an example from client ( JS ) side:

// Example call: 
saveFile(JSON.stringify(places),"/myHomeProgram.json","application/json");

function saveFile(data, filename, type) {
    var file = new Blob([data], {type: type});
    form = new FormData();
    form.append("blob", file, filename);
    $.ajax({
        url: '/uploading', 
        type: 'POST',
        data: form,
        processData: false,
        contentType: false
      }).done(function(resp){
        var response = resp.split(";");
        
        $(".saveIconGraph").removeClass("fas fa-spinner fa-spin");
        $(".saveIconGraph").addClass("far fa-save");

        if(response[1] == "/myHomeProgram.json"){
            toast("success","saveOk","progInfo",3500);
            showSaved();
            setTimeout(() => {
                $("#saveMe").fadeOut( "slow", function() { 
                    showSave();
                });
            }, 1000);
            initPlaces();
        }
      }).fail(function(resp){
        var response = resp.split(";");

        $(".saveIconGraph").removeClass("fas fa-spinner fa-spin");
        $(".saveIconGraph").addClass("far fa-save");
        
        if(response[1] == "/myHomeProgram.json"){
            toast("error","saveNotOk","progInfo",3500);
            showSaveError();
            $("#saveMeBtn").addClass("shakeEffect");
            setTimeout(() => {
                $("#saveMeBtn").removeClass("shakeEffect");
                showSave();
            }, 4500);
        }
      });
}

I could save the file in a temporary char variable before write, and on the final I could match the size of the file and the temporary variable size and if it is not the same, roll back to the previous. Is this manageable?

Something like this:

String uploadTemp = "";
inline boolean saveFileToTemp(String fileName){
  uploadTemp = "";
  File f = LITTLEFS.open(fileName, "r");
  if (!f) {
    f.close();
    return false;
  }else{
    for (int i = 0; i < f.size(); i++){
      uploadTemp += (char)f.read();
    }
  }
  f.close();
  return true;
}

inline boolean revertBackFile(String fileName){
  File g = LITTLEFS.open(fileName, "w");
  if (!g) {
    g.close();
    return false;
  }else{
    g.print(uploadTemp);
  }
  g.close();
  return true;
}

inline boolean matchFileSizes(String fileName,boolean isFileExists){
  boolean isCorrect = false;
  if(isFileExists){
    File writedFile = LITTLEFS.open(fileName, "w");
    if( writedFile.size() == uploadTemp.length()){
      isCorrect = true;
    }else{
      isCorrect = false;
    }
    writedFile.close();
    return isCorrect;
  }else{
    return true;
  }
}

void handleFileUpload(AsyncWebServerRequest * request, String filename,size_t index, uint8_t *data, size_t len, bool final) {
  String webResponse;
  boolean error = false,isFileExists = false;
  if (!index) {
    if (!filename.startsWith("/"))
      filename = "/" + filename;
    if (LITTLEFS.exists(filename)) {
      isFileExists = true;
      // Save the file to a temporary String if it success we continue.
      if( saveFileToTemp(filename) ){
        LITTLEFS.remove(filename);
      }else{
        // If the file save was fail we abort everything.
        webResponse = "File NOT Uploaded " + filename;
        final = true;
        error = true;
      }
    }
    if( !error ){
      uploadFile = LITTLEFS.open(filename, "w");
    }
  }
  if( !error ){
    // Copy content to the actual file
    for (size_t i = 0; i < len; i++) {
      uploadFile.write(data[i]);
    }
  }
  if (final) {
    uploadFile.close();
    if( !error ){
      if( matchFileSizes(filename,isFileExists) ){
        if(filename == "/myHomeProgram.json"){initProgram = true;}
        webResponse = "File Uploaded " + filename;
      }else{
        error = true;
        webResponse = "File length mismatch";
      }
    }
    if( error ){
      revertBackFile(filename);
    }
    Serial.println(webResponse);
    AsyncWebServerResponse *response = request->beginResponse(200, "text/plain", webResponse);
    response->addHeader("Access-Control-Allow-Origin","*");
    request->send(response);
  }

}

With this test if i try to upload a file relatively big, like 50kb the watchdog got triggered, but working with small files. — Dr.Random, May 19 '21 at 08:10

Dr.Random · Accepted Answer · 2021-05-20T06:45:20.580

It seems to me that the problem solved.

I have managed to replace the String buffer with a char one in external memory. It seems stable but requires more testing. I'll post the solution but if anyone has a better approach feel free to comment here.

Thanks.

char * uploadTemp;
inline boolean saveFileToTemp(String fileName){
  File f = LITTLEFS.open(fileName, "r");
  if (!f) {
    f.close();
    return false;
  }else{
    size_t fileSize = f.size();
    uploadTemp = (char*)ps_malloc(fileSize + 1);
    for (int i = 0; i < fileSize; i++){
      uploadTemp[i] = (char)f.read();
    }
    uploadTemp[fileSize] = '\0';
  }
  f.close();
  return true;
}

inline boolean revertBackFile(String fileName){
  File g = LITTLEFS.open(fileName, "w");
  if (!g) {
    g.close();
    return false;
  }else{
    g.print(uploadTemp);
  }
  g.close();
  return true;
}

inline boolean matchFileSizes(String fileName,boolean isFileExists){
  boolean isCorrect = false;
  if(isFileExists){
    File writedFile = LITTLEFS.open(fileName, "w");
    if( writedFile.size() == sizeof(uploadTemp)){
      isCorrect = true;
    }else{
      isCorrect = false;
    }
    writedFile.close();
    return isCorrect;
  }else{
    return true;
  }
}

void handleFileUpload(AsyncWebServerRequest * request, String filename,size_t index, uint8_t *data, size_t len, bool final) {
  boolean isFileExists = false,error = false;
  String webResponse = "";
  int httpStatus = 200;

  // Start of the file upload
  if (!index) {
    // Make sure that there is a / char at the start of the string
    if (!filename.startsWith("/")){ filename = "/" + filename; }
    // Check if the file exists
    if (LITTLEFS.exists(filename)) {
      isFileExists = true;
      // Get the file contents for safety reasons
      // If it succeded we can create a new file in the palce
      if( saveFileToTemp(filename) ){
        uploadFile = LITTLEFS.open(filename, "w");
      }else{
        // If we can not save it abort the upload process.
        webResponse = "File NOT Uploaded " + filename;
        final = true;error = true;
      }
    }
  }
  // If we have no error at this point, we can start to copy the content to the file.
  if( !error ){
    for (size_t i = 0; i < len; i++) {
      uploadFile.write(data[i]);
    }
  }
  // If no more data we can start responding back to the client
  if (final) {
    uploadFile.close();
    // Check if we got any error before.

    if( !error && matchFileSizes(filename,isFileExists) ){
      // Copyed file is the same, upload success.
      if(filename == "/myHomeProgram.json"){initProgram = true;}
      webResponse = "File Uploaded " + filename;
    }else{
      webResponse = "File length mismatch";
      revertBackFile(filename);
      httpStatus = 500;
    }

    free(uploadTemp);
    AsyncWebServerResponse *response = request->beginResponse(httpStatus, "text/plain", webResponse);
    response->addHeader("Access-Control-Allow-Origin","*");
    request->send(response);

  }
}

EDIT:

Yeah, so it was completely wrong.

I have to do the following things:

Save the file we want to upload if it exist into a temporary char array.
Get the uploaded file into a temporary file on upload.
If everything was a success, copy the contents of the temporary file to the intended file.
If something fails, revert back the saved file to the original and report an error.

Something like this ( still in test ):

char * prevFileTemp;

inline boolean saveFileToTemp(String fileName){
  File f = LITTLEFS.open(fileName, "r");
  if (!f) {
    f.close();
    return false;
  }else{
    size_t fileSize = f.size();
    prevFileTemp = (char*)ps_malloc(fileSize + 1);
    for (int i = 0; i < fileSize; i++){
      prevFileTemp[i] = (char)f.read();
    }
  }
  f.close();
  return true;
}

inline boolean revertBackFile(String fileName){
  if (LITTLEFS.exists(fileName)) {
    Serial.println("Reverting back the file");
    File g = LITTLEFS.open(fileName, "w");
    if (!g) {
      g.close();
      return false;
    }else{
      g.print(prevFileTemp);
    }
    g.close();
  }
  return true;
}


static const inline boolean copyContent(String fileName){
  File arrivedFile  = LITTLEFS.open(uploadTemp, "r");
  File newFile      = LITTLEFS.open(fileName, "w");
  // Check if we can open the files as intended.
  if( !arrivedFile || !newFile){
    revertBackFile(fileName);
    return false;
  }
  // Copy one file content to another.
  for (size_t i = 0; i < arrivedFile.size(); i++) { newFile.write( (char)arrivedFile.read() ); }
  // Check the sizes, if no match, abort mission.
  if( newFile.size() != arrivedFile.size()){ return false; }
  
  arrivedFile.close();newFile.close();
  return true;
}

boolean isFileExists = false,uploadError = false,newFileArrived = false;
String webResponse = "",newArrivalFileName = "";
int httpStatus = 200;

inline void resetVariables(){
  isFileExists  = false;
  uploadError   = false;
  webResponse   = "";
  httpStatus    = 200;
}


void handleFileUpload(AsyncWebServerRequest * request, String filename,size_t index, uint8_t *data, size_t len, bool final) {
  // Start file upload process
  if (!index) {
    // Reset all the variables
    resetVariables();
    // Make sure that there is a '/' char at the start of the string
    if (!filename.startsWith("/")){ filename = "/" + filename; }
    // Open the temporary file for content copy if it is exist
    if (LITTLEFS.exists(filename)) {
      if( saveFileToTemp(filename) ){
        uploadFile = LITTLEFS.open(uploadTemp, "w");
      }else{
        // If we can not save it abort the upload process.
        webResponse = "File NOT Uploaded " + filename;
        final = true;uploadError = true;
      }
    }
  }
  // If we have no error at this point, we can start to copy the content to the temporary file.
  if( !uploadError ){
    for (size_t i = 0; i < len; i++) {
      uploadFile.write(data[i]);
    }
  }
  // If no more data we can start responding back to the client
  if (final) {
    if (!filename.startsWith("/")){ filename = "/" + filename; }
    uploadFile.close();
    if( !uploadError && copyContent(filename) ){
      webResponse = "File Uploaded " + filename;
    }else{
      webResponse = "File length mismatch";
      revertBackFile(filename);
      httpStatus = 500;
    }
    free(prevFileTemp);
    AsyncWebServerResponse *response = request->beginResponse(httpStatus, "text/plain", webResponse);
    response->addHeader("Access-Control-Allow-Origin","*");
    request->send(response);
  }

}

Any solution for file upload problem on Arduino ESP32 via http

1 Answers1