Is there a way to increase SD Card Write speed in LUA on Nodemcu ESP8266?

Question

The maximum write speed I can achieve is 2.4 KB/s. Is there a way to increase this?

Using LUA on a NodeMCU ESP8266 and the SPI module in User_Modules.h. #define BUILD_FATFS is also enabled in user_config.h.

I have a datalogger that is sampling 920SPS or ~1.1ms/Sample for 10 hours at a time. 1.1 ms should be lots of time to write two Bytes to a SD card or a buffer of xxx Bytes in between samples, however the max write speed I see is 498 ms to write 1200 Bytes or 7ms to write 3 Bytes. This is a long way from SD class 0 standard of 12.5MB/s. The logger ends up missing ~450 Samples when I dump 1200 B to the card.

local adc1 = nil
local t_tbl={}
local n=1

function adcReady(_,_,c)
    
    _,_, adctbl[n], _ = adc1:read()
    n=n+1
    if n>400 then
    
        t_tbl[1]=tmr.now()
        
        file.open("/SD0/sddata.txt","a")
        for k,v in ipairs(adctbl) do 
            file.write(v..",")
            adctbl[k]=nil
        end
        file.close()
        
        t_tbl[2]=tmr.now()
        
        print(t_tbl[2] - t_tbl[1])
        n=1
        
    end
end

do
    local adc = {
        ADC1_ID             =   0,
        ADC1_ADDRESS        =   ads1115.ADDR_GND, 
        GAIN                =   ads1115.GAIN_4_096V, 
        SAMPLES             =   ads1115.DR_920SPS, 
        CHANNEL             =   ads1115.SINGLE_0,       
        MODE                =   ads1115.CONTINUOUS, 
        CONV_READY          =   ads1115.CONV_RDY_1, 
    }
    i2c.setup(i2c0.id, i2c0.sda, i2c0.scl, i2c0.speed)
    ads1115.reset()
    adc1 = ads1115.ads1015(adc.ADC1_ID, adc.ADC1_ADDRESS)   
    adc1:setting(adc.GAIN, adc.SAMPLES, adc.CHANNEL, adc.MODE, adc.CONV_READY)  
    
    spi.setup(1, spi.MASTER, spi.CPOL_LOW, spi.CPHA_LOW, 8, 2, spi.HALFDUPLEX)
    vol = file.mount("/SD0", 8)   -- 2nd parameter is optional for non-standard SS/CS pin
    file.open("/SD0/sddata.txt","w+")
    file.close()
    
    tmr.create():alarm(1000,tmr.ALARM_SINGLE,function()
        gpio.mode(i2c0.conv_rdy,gpio.INT) 
        gpio.trig(i2c0.conv_rdy,'up', adcReady) --enable interrupt, active low rising edge==conv ready  
    end)
end

Answer 1

You can speedup file write by preparing 2Kbyte-aligned chunks of text.
Replace your adcReady with:

local log_text = ""
local chunk_size = 2*1024

function adcReady(_,_,c)
   _, _, adctbl[n], _ = adc1:read()
   n = n + 1
   if n > 400 then
   
      t_tbl[1] = tmr.now()
      
      log_text = log_text..table.concat(adctbl, ",", 1, n-1)..","
      local size = #log_text - #log_text % chunk_size
      local log_text_to_save = log_text:sub(1, size)
      log_text = log_text:sub(size + 1)
      
      t_tbl[2] = tmr.now()
      
      if size ~= 0 then 
         file.open("/SD0/sddata.txt","a")
         file.write(log_text_to_save)
         file.close()
      end
      
      t_tbl[3] = tmr.now()
      
      print(t_tbl[2] - t_tbl[1], t_tbl[3] - t_tbl[2])  -- for strings and GC, for File operations
      n = 1
      
   end
end

Is it faster than 498 ms?

---

UPDATE:

New version with cached tostring()

local num2str = {}

function adcReady(_,_,c)
   _, _, adctbl[n], _ = adc1:read()
   n = n + 1
   if n > 400 then

      t_tbl[1] = tmr.now()

      for i = 1, n - 1 do
         local v = adctbl[i]
         local s = num2str[v]
         if not s then
            s = v..","
            num2str[v] = s
         end
         adctbl[i] = s
      end
      local log_text_to_save = table.concat(adctbl, "", 1, n-1)

      t_tbl[2] = tmr.now()

      file.open("/SD0/sddata.txt","a")
      file.write(log_text_to_save)
      file.close()

      t_tbl[3] = tmr.now()

      print(t_tbl[2] - t_tbl[1], t_tbl[3] - t_tbl[2])  -- for strings and GC, for File operations
      n = 1

   end
end

Is it faster than previous version?

---

UPDATE2:

local chr = string.char

function adcReady(_,_,c)
   _, _, adctbl[n], _ = adc1:read()
   n = n + 1
   if n > 400 then

      t_tbl[1] = tmr.now()

      for i = 1, n - 1 do
         local v = adctbl[i]
         -- 0<=v<=4095
         local s
         if v < 10 then
            s = chr(v + 48, 44)
         else
            local m10 = v % 10
            if v < 100 then
               s = chr((v - m10)/10 + 48, m10 + 48, 44)
            else
               local m100 = v % 100
               if v < 1000 then
                  s = chr((v - m100)/10 + 48, (m100 - m10)/10 + 48, m10 + 48, 44)
               else
                  local m1000 = v % 1000
                  s = chr((v - m1000)/1000 + 48, (m1000 - m100)/100 + 48, (m100 - m10)/10 + 48, m10 + 48, 44)
               end
            end
         end
         adctbl[i] = s
      end
      local log_text_to_save = table.concat(adctbl, "", 1, n-1)

      t_tbl[2] = tmr.now()

      file.open("/SD0/sddata.txt","a")
      file.write(log_text_to_save)
      file.close()

      t_tbl[3] = tmr.now()

      print(t_tbl[2] - t_tbl[1], t_tbl[3] - t_tbl[2])  -- for strings and GC, for File operations
      n = 1

   end
end

---

UPDATE3:

For Lua 5.3 and hex digits in log:

-- log output is in hex
local high = {} -- [1] = "", [2] = "1", ..., [256] = "FF"
local low = {}  -- [1] = "0,", [2] = "1,", ..., [16] = "F,"
for x = 0, 255 do  -- replace 255 with 127 (to save memory) if ADC generates only positive values 0x0000-0x7FF0
   high[x+1] = string.format("%X", x*16):sub(1, -2)
   if x < 16 then
      low[x+1] = string.format("%X,", x)
   end
end

-- in case of out-of-memory error reduce measures count (400) to 256
local measures = 400   -- recommended values are powers of 2
local measures_2 = measures*2

-- adctbl[] is not used anymore, text_buffer[] is used instead
local text_buffer = {}  -- array of (2*measures) elements
for x = 1, measures_2 do
   text_buffer[x] = ""
end

function adcReady(_,_,c)
   local _, _, v = adc1:read()
   -- 0x0000<=v<=0xFFF0
   text_buffer[n] = high[(v>>8)+1]
   text_buffer[n+1] = low[((v>>4)&15)+1]
   n = n + 2
   if n > measures_2 then

      t_tbl[1] = tmr.now()

      local log_text_to_save = table.concat(text_buffer, "", 1, n-1)

      t_tbl[2] = tmr.now()

      file.open("/SD0/sddata.txt","a")
      file.write(log_text_to_save)
      file.close()

      t_tbl[3] = tmr.now()

      print(t_tbl[2] - t_tbl[1], t_tbl[3] - t_tbl[2])  -- for strings and GC, for File operations
      n = 1

   end
end