Why does HTTP POST on Arduino fail sending large files

Question

I am trying to send a large file to a server using HTTP POST. I can send up files of 4kB or less but when I try to send a file of 1.5MB there appears to be a time out. The server disconnects without sending the data

Does anyone know how to fix this?

The code is: void readdata() {

  //Variables for WiFi Shield
  char ssid[] = "Example";           //your network SSID (name)
  char pass[] = "Example1";           
  char server[] = "http://example.com";

  while (status != WL_CONNECTED) {
    Serial.print(F("Attempting to connect to SSID: "));
    Serial.println(ssid);

    status = WiFi.begin(ssid, pass);

    // wait 1 second for connection:
    delay(1000);
  }
  Serial.println(F("Connected to wifi"));
 // #endif

  // check for the presence of the shield:
  if (WiFi.status() == WL_NO_SHIELD) {
    Serial.println(F("WiFi shield not present"));
    // don't continue:
   while (true);
  }
logfile=SD.open("LOG0038.CSV");
filesize=logfile.size();
  logfile.close();

  int fileLength = 0;
  fileLength = 137 + filesize + 40;
  Serial.println(filesize);

  Serial.println(F("\nStarting connection to server..."));
  Serial.println(server);
  // if you get a connection, report back via serial:
  if (client.connect(server, 80)) {
    Serial.println(F("connected to server"));

    // Make a HTTP request:

  client.setTimeout(Time);
  client.print(F("POST /uploadf HTTP/1.1\r\n"));
  client.print(F("Host: example.com\r\n"));
  client.print(F("User-Agent: Frank/1.0\r\n"));
  client.print(F("Accept-Encoding: gzip, deflate\r\n"));
  client.print(F("Accept: */*\r\n"));
  client.print(F("Connection: keep-alive\r\n"));
  client.print(F("Content-Length: "));
  client.print(fileLength);
  client.print(F("\r\n"));
  client.print(F("Content-Type: multipart/form-data; boundary=710ff0c6cf2d4c73b12db64cab12e58c\r\n"));
  client.print(F("\r\n"));
  client.print(F("--710ff0c6cf2d4c73b12db64cab12e58c\r\nContent-Disposition: form-data; name=\"file\"; filename=\""));
  client.print("1000LOG0032.CSV");
  client.print(F("\""));
  client.print(F("\r\n"));
  client.print(F("Content-Type: text/plain\r\n\r\n"));
  logfile=SD.open("LOG0038.CSV");
  while(logfile.available()){
    while(logfile.position() < logfile.size()){
      String dataBuff = logfile.readStringUntil('\n');
      client.print(dataBuff);
     // Serial.println(dataBuff);
      client.print("\n");
    }
  }
  client.print(F("\r\n--710ff0c6cf2d4c73b12db64cab12e58c--\r\n"));
  //client.flush();
  }

  while(client.connected()){
    while (client.available()&& status==WL_CONNECTED) {
      char c = client.read();
     Serial.write(c);
   }
  }

  // if the server's disconnected, stop the client:
  if (!client.connected()) {
    Serial.print(F("\r\n"));
    Serial.println(F("disconnecting from server."));
 //   client.flush();
    client.stop();

  } 
 logfile.close();
 while (true);
  }

Is the server closing the connection? Did you check if there's a limit in HTTP post size in your server — FrancescoAzzola, Nov 08 '16 at 10:46
I checked the server and it has no limit to the HTTP POST size - it looks like the arduino will only send 20k of data at a time — Dave, Nov 09 '16 at 11:09

score 0 · Answer 1 · answered Oct 19 '16 at 12:53

This might not be a real answer but unfortunately i don't have the needed reputation to write this as a comment.

I don't know anything about arduino but whenever you are working with POST there should be an option to limit max_post_size. In a basic LAMP setup it's usually found in .httaccess.

I hope this helps you troubleshooting.

score 0 · Answer 2 · edited May 20 '23 at 22:29

One can send a POST request with data fields appended with binary data as shown in the code below. The presented code was made to make a POST request to a Dataverse repository (dataverse.org), however, can be easily adapted to any other POST request. One final note, the code below is optimized for Tensilica's Xtensa LX6/7 microprocessors, vis-a-vis ESP32.

// *********************************************************
//            Upload Dataset to Harvard's Dataverse
// *********************************************************
void UploadToDataverse() {
  //Check WiFi connection status
  if(WiFi.status() != WL_CONNECTED){
    mserial.printStrln("WiFi Disconnected");
    if (connect2WIFInetowrk()){
      UploadToDataverse();
    }
  }
  // Start sending dataset file
  File datasetFile = FFat.open("/"+EXPERIMENTAL_DATA_FILENAME, FILE_READ);
  if (!datasetFile){
    mserial.printStrln("Dataset file not found");
    return;
  }
    
  String boundary = "7MA4YWxkTrZu0gW";
  String contentType = "text/csv";
  // DATASET_REPOSITORY_URL = "/api/files/:persistentId/replace?persistentId=" +PERSISTENT_ID;
  DATASET_REPOSITORY_URL =  "/api/datasets/:persistentId/add?persistentId="+PERSISTENT_ID;
  
  String datasetFileName = datasetFile.name();
  String datasetFileSize = String(datasetFile.size());
  mserial.printStrln("Dataset File Details:");
  mserial.printStrln("Filename:" + datasetFileName);
  mserial.printStrln("size (bytes): "+ datasetFileSize);
  mserial.printStrln("");
    
  WiFiClientSecure client;

  int str_len = SERVER_URL.length() + 1; // Length (with one extra character for the null terminator)
  char SERVER_URL_char [str_len];    // Prepare the character array (the buffer) 
  SERVER_URL.toCharArray(SERVER_URL_char, str_len);    // Copy it over 
    
  client.stop();
  client.setCACert(HARVARD_ROOT_CA_RSA_SHA1);
  if (!client.connect(SERVER_URL_char, SERVER_PORT)) {
      mserial.printStrln("Cloud server URL connection FAILED!");
      mserial.printStrln(SERVER_URL_char);
      int server_status = client.connected();
      mserial.printStrln("Server status code: " + String(server_status));
      return;
  }
  mserial.printStrln("Connected to the dataverse of Harvard University"); 
  mserial.printStrln("");
  // We now create a URI for the request
  mserial.printStr("Requesting URL: ");
    mserial.printStrln(DATASET_REPOSITORY_URL);

  // Make a HTTP request and add HTTP headers    
  // post header
  String postHeader = "POST " + DATASET_REPOSITORY_URL + " HTTP/1.1\r\n";
  postHeader += "Host: " + SERVER_URL + ":" + String(SERVER_PORT) + "\r\n";
  postHeader += "X-Dataverse-key: " + API_TOKEN + "\r\n";
  postHeader += "Content-Type: multipart/form-data; boundary=" + boundary + "\r\n";
  postHeader += "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n";
  postHeader += "Accept-Encoding: gzip,deflate\r\n";
  postHeader += "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n";
  postHeader += "User-Agent: AeonLabs LDAD Smart DAQ device\r\n";
  postHeader += "Keep-Alive: 300\r\n";
  postHeader += "Connection: keep-alive\r\n";
  postHeader += "Accept-Language: en-us\r\n";

  // jsonData header
  String jsonData = "{\"description\":\"LIVE Experimental data upload from LDAD Smart 12bit DAQ \",\"categories\":[\"Data\"], \"restrict\":\"false\", \"tabIngest\":\"false\"}";     
  String jsonDataHeader = "--" + boundary + "\r\n";
  jsonDataHeader += "Content-Disposition: form-data; name=\"jsonData\"\r\n\r\n";
  jsonDataHeader += jsonData+"\r\n";

  // dataset header
  String datasetHead = "--" + boundary + "\r\n";
  datasetHead += "Content-Disposition: form-data; name=\"file\"; filename=\"" + datasetFileName + "\"\r\n";
  datasetHead += "Content-Type: " + contentType + "\r\n\r\n";

  // request tail
  String tail = "\r\n--" + boundary + "--\r\n\r\n";

  // content length
  int contentLength = jsonDataHeader.length() + datasetHead.length() + datasetFile.size() + tail.length();
  postHeader += "Content-Length: " + String(contentLength, DEC) + "\n\n";
  
  // send post header
  int postHeader_len=postHeader.length() + 1; 
  char charBuf0[postHeader_len];
  postHeader.toCharArray(charBuf0, postHeader_len);
  client.print(charBuf0);
  mserial.printStr(charBuf0);

  // send key header
  char charBufKey[jsonDataHeader.length() + 1];
  jsonDataHeader.toCharArray(charBufKey, jsonDataHeader.length() + 1);
  client.print(charBufKey);
  mserial.printStr(charBufKey);

  // send request buffer
  char charBuf1[datasetHead.length() + 1];
  datasetHead.toCharArray(charBuf1, datasetHead.length() + 1);
  client.print(charBuf1);
  mserial.printStr(charBuf1);

  // create buffer
  const int bufSize = 2048;
  byte clientBuf[bufSize];
  int clientCount = 0;

  while (datasetFile.available()) {
    clientBuf[clientCount] = datasetFile.read();
    clientCount++;
    if (clientCount > (bufSize - 1)) {
        client.write((const uint8_t *)clientBuf, bufSize);
        clientCount = 0;
    }
  }

  datasetFile.close();
    
  if (clientCount > 0) {
      client.write((const uint8_t *)clientBuf, clientCount);
      mserial.printStrln("[binary data]");
  }

  // send tail
  char charBuf3[tail.length() + 1];
    tail.toCharArray(charBuf3, tail.length() + 1);
    client.print(charBuf3);
  mserial.printStr(charBuf3);



  // Read all the lines of the reply from server and print them to mserial
    mserial.printStrln("");
    mserial.printStrln("Response Headers:");
  String responseHeaders = "";

  while (client.connected()) {
    // mserial.printStrln("while client connected");
    responseHeaders = client.readStringUntil('\n');
    mserial.printStrln(responseHeaders);
    if (responseHeaders == "\r") {
      mserial.printStrln("======   end of headers ======");
      break;
    }
  }

  String responseContent = client.readStringUntil('\n');
  mserial.printStrln("Harvard University's Dataverse reply was:");
  mserial.printStrln("==========");
  mserial.printStrln(responseContent);
  mserial.printStrln("==========");
  mserial.printStrln("closing connection");
  client.stop();
  }

Why does HTTP POST on Arduino fail sending large files

2 Answers2