Writing 9-bit values to byte array (or EEPROM) without wasting remaining bit in the following byte

Question

I could not found an answer with google so i went for it and programmed quite a few hours.

I want to save 9-bit values to eeprom without wasting the other 7 bits. I save values that would be up to 500 and i have not much EEPROM left.

The same principle can be applied to arrays, which i did just to not waer down the EEPROM.

So I made this little program:

/*
 * Write only a certain number of bits to EEPROM.
 *
 * keeps the other bit in the byte of the eeprom as they are.
 *
 *  Working version with 9 bits:
 *          2019-10-03 15:57
 *          2019-10-03 22:09 tested with chars too
 *          2019-10-04 08:25 works with 7 bit chars also!
 *          2019-10-04 12:27 fixed the combining of oldByte and new values in writeBitsToEEPROM(), because chars like 'ö' altered previous bit (left side) that should not have been altered.
 *
 */


#include "arduino.h"
#include "EEPROM.h"
#include "math.h"


#define BIT_BLOCKS_COUNT 15
#define BLOCK_BYTE_COUNT 17



#define ARRAY_SIZE BLOCK_BYTE_COUNT+2

//TODO: change back to original value
#define EEPROM_SIZE ARRAY_SIZE

byte fakeEEPROM[ARRAY_SIZE] = {0};


String byteToString(byte value){
  char byteChar[9];
  byteChar[8] = '\0'; //we need a terminator

  for(int i=7; i>=0; i--){
    byteChar[7-i] = (value & (1 << i)) ? '1' : '0';
  }

  return String(byteChar);
}


String byteToString(unsigned long value, byte bytesToRead){
    String str1 = byteToString(value >> 8);
    String str2 = byteToString(value & 0xFF);

    return str1 + " " + str2;
}


int globBlockStartAdress = 0;
byte globNumberOfBits = 0;
int globBlockSizeBytes = 0;

bool initBitBlock(int blockStartAdress, int blockCount, byte numberOfBits) {

    globBlockStartAdress = blockStartAdress;
    globNumberOfBits = numberOfBits;

    // calc needed number of bytes and roud up
    int tempBlockSize = blockCount * numberOfBits / 8;
    if(blockCount * numberOfBits % 8)
        tempBlockSize++;

    // make number of bytes even
    if(tempBlockSize % 2)
        tempBlockSize++;

    globBlockSizeBytes = tempBlockSize;

    if(blockStartAdress + globBlockSizeBytes > EEPROM_SIZE)
        return false;

    return true;
}

/*
 * Writes 1 to 9 bits to "internalAdress" within a designated block in eeprom
 */
void writeBitsToEEPROM(unsigned int bitsToBeWritten, int internalAdress){

    //TODO: check if value is not higher than what can be stored
//  if(bitsToBeWritten){
//
//  }

    int trueEEPROMAdress = globBlockStartAdress + internalAdress * globNumberOfBits / 8;

    if(trueEEPROMAdress + 1 >= ARRAY_SIZE || internalAdress * globNumberOfBits / 8 >= globBlockSizeBytes){
        Serial.print("globBlockSizeBytes: ");
        Serial.println(globBlockSizeBytes);

        Serial.println("FEHLER writeBitsToEEPROMWTF: ");
        Serial.println(trueEEPROMAdress + 1);
        Serial.println(internalAdress * globNumberOfBits / 8 );

    }

    byte startBitOfEEPROMByte = (internalAdress * globNumberOfBits) % 8;

    unsigned int oldIntFromEEPROM = (fakeEEPROM[trueEEPROMAdress] << 8) | fakeEEPROM[trueEEPROMAdress + 1];

    //Todo: change to eeprom
    //filter out only the bits that need to be kept.
    //EEPROM.get(trueEEPROMAdress, oldEEPROMByteBits);

    // there might be bits in the byte that we dont want to change. left side and right side
    unsigned int mask1KeepFromEEPROM = (0xFFFF <<  (16 - startBitOfEEPROMByte));
    unsigned int mask2KeepFromEEPROM = (0xFFFF >>  (startBitOfEEPROMByte + globNumberOfBits));

    //if(16 - startBitOfEEPROMByte - numberOfBits > 0)
    //mask2KeepFromEEPROM= (0xFFFF >>  (startBitOfEEPROMByte + numberOfBits));

    // masks combined
    unsigned int maskIntToKeepFromEEPROM = mask1KeepFromEEPROM | mask2KeepFromEEPROM;


    int newEEPROMInt = (oldIntFromEEPROM & maskIntToKeepFromEEPROM) | ((bitsToBeWritten << (16 - globNumberOfBits - startBitOfEEPROMByte) & ~maskIntToKeepFromEEPROM));


    //Todo: change to eeprom
    //write
    //EEPROM.update(trueEEPROMAdress, newEEPROMByteBitsA);
    fakeEEPROM[trueEEPROMAdress] =      (newEEPROMInt >> 8);
    fakeEEPROM[trueEEPROMAdress + 1] =  (byte) newEEPROMInt;

    if(trueEEPROMAdress + 1 > BLOCK_BYTE_COUNT){
        Serial.println("FEHLER writeBitsToEEPROM");
        Serial.println(trueEEPROMAdress + 1);

        Serial.println("blockStartAdress");
        Serial.println(globBlockStartAdress);

        Serial.println("internalAdress");
        Serial.println(internalAdress);

        Serial.println("numberOfBits");
        Serial.println(globNumberOfBits);
    }


//  Serial.print("trueEEPROMAdress: ");
//  Serial.println(trueEEPROMAdress);
//
//  Serial.print("internalAdress: ");
//  Serial.println(internalAdress);
//
//  Serial.print("globNumberOfBits: ");
//  Serial.println(globNumberOfBits);
//
//  Serial.print("bitsToBeWritten:         ");
//  Serial.println(byteToString(bitsToBeWritten,2));
//
//  Serial.print(" mask1KeepFromEEPROM:    ");
//  Serial.println(byteToString(mask1KeepFromEEPROM,2));
//
//  Serial.print("mask2KeepFromEEPROM:     ");
//  Serial.println(byteToString(mask2KeepFromEEPROM,2));
//
//  Serial.print("maskIntToKeepFromEEPROM: ");
//  Serial.println(byteToString(maskIntToKeepFromEEPROM,2));
//
//  Serial.print("oldIntFromEEPROM:        ");
//  Serial.println(byteToString(oldIntFromEEPROM,2));
//
//  Serial.print("newEEPROMInt:            ");
//  Serial.println(byteToString(newEEPROMInt,2));
//
//  Serial.print("512:                     ");
//  Serial.println(byteToString(512, 2));
//
//  Serial.print("65535:                   ");
//  Serial.println(byteToString(65535, 2));
}


unsigned int ReadBitsFromEEPROM(int internalAdress){

    int trueEEPROMAdress = globBlockStartAdress + internalAdress * globNumberOfBits / 8;
    byte startBitOfEEPROMByte = (internalAdress * globNumberOfBits) % 8;

    if(trueEEPROMAdress + 1 > BLOCK_BYTE_COUNT)
        Serial.println("FEHLER readBits");
    unsigned int oldIntFromEEPROM = (fakeEEPROM[trueEEPROMAdress] << 8) | fakeEEPROM[trueEEPROMAdress + 1];

    //Todo: change to eeprom
    //filter out only the bits that need to be kept.
    //EEPROM.get(trueEEPROMAdress, oldEEPROMByteBits);

    unsigned int mask1KeepFromEEPROM = (0xFFFF <<  (16 - startBitOfEEPROMByte));
    unsigned int mask2KeepFromEEPROM = (0xFFFF >>  (startBitOfEEPROMByte + globNumberOfBits));

    unsigned int maskIntToKeepFromEEPROM = mask1KeepFromEEPROM | mask2KeepFromEEPROM;

    unsigned int valueFromEEPROM = ~maskIntToKeepFromEEPROM & oldIntFromEEPROM;


//  Serial.print("trueEEPROMAdress: ");
//  Serial.println(trueEEPROMAdress);
//
//  Serial.print("internalAdress: ");
//  Serial.println(internalAdress);
//
//  Serial.print("numberOfBits: ");
//  Serial.println(numberOfBits);
//
//  Serial.print(" mask1KeepFromEEPROM:    ");
//  Serial.println(byteToString(mask1KeepFromEEPROM,2));
//
//  Serial.print("mask2KeepFromEEPROM:     ");
//  Serial.println(byteToString(mask2KeepFromEEPROM,2));
////
//  Serial.print("maskIntToKeepFromEEPROM: ");
//  Serial.println(byteToString(maskIntToKeepFromEEPROM,2));
////
//  Serial.print("oldIntFromEEPROM:        ");
//  Serial.println(byteToString(oldIntFromEEPROM,2));

    return (valueFromEEPROM >> (16 - globNumberOfBits - startBitOfEEPROMByte));
}



void setup() {
    Serial.begin(57600);
    Serial.print(F("\n# Programversion: "));

    Serial.print(__TIME__);
    Serial.print(" ");
    Serial.println(__DATE__);



    Serial.println("Setup finished");
    delay(1000);
}


void printEEPROM(){

    for(int i = 0; i < ARRAY_SIZE; i++){
        byte b;

        //Todo: change to eeprom
        //EEPROM.get(i, b);
        b = fakeEEPROM[i];

        Serial.print(byteToString(b));
        Serial.print(" ");
    }
    Serial.println();
}


void testNumbers() {

    Serial.println("bits?");
    while( ! Serial.available());
    String input = Serial.readString();

    unsigned int value = input.toInt();


    initBitBlock(1, 15, 9);

    //  Serial.print("value:              ");
    //  Serial.println(byteToString(value));



    for(int i = 0; i < BIT_BLOCKS_COUNT;i++){

        for(int j = 0; j < BLOCK_BYTE_COUNT; j++){
            fakeEEPROM[j] = 0xFF;
            if(j > BLOCK_BYTE_COUNT)
                Serial.println("FEHLER testNumbers");
        }

//      Serial.print("EEPROM before: ");
//      printEEPROM();

        writeBitsToEEPROM(value, i);
        Serial.print("Returned: ");
        Serial.println(ReadBitsFromEEPROM(i));

//      Serial.print("EEPROM after:  ");
//      printEEPROM();
//      Serial.println();
    }
    delay(1000);
}


#define CHAR_COUNT 16


void testChars() {

//  Serial.println("bits?");
//  while( ! Serial.available());
//  String input = Serial.readString();
//
//  unsigned int value = input.toInt();

    initBitBlock(1, CHAR_COUNT, 7);

    Serial.println("string?");
    while( ! Serial.available());
    String input = Serial.readString();

    Serial.println(input);

    char testString[CHAR_COUNT] = {'\0'};

    input.toCharArray(testString, CHAR_COUNT, 0);

    for(int j = 0; j < ARRAY_SIZE; j++){
            fakeEEPROM[j] = 0;//xFF;
        }


    for(int i = 0; i < CHAR_COUNT; i++){



        Serial.print("EEPROM before: ");
        printEEPROM();

        writeBitsToEEPROM(testString[i], i);


        Serial.print("EEPROM after:  ");
        printEEPROM();
        Serial.println();
    }


    Serial.println("Returned: ");
    for(int i = 0; i < CHAR_COUNT; i++){


        Serial.print((char) ReadBitsFromEEPROM(i));

    }

    Serial.println();


    delay(1000);

}

void loop(){
    testChars();
    testNumbers();

}

which of course it not complete. Its just for saving those 9-bit values.

My question is: Has anyone else programmed a function like this - or knows where to find this - that is not limited to 9 bits (10 bits will span over 3 bytes)?

I have a hard time understanding what you are asking, but I doubt your EEPROM is bit-addressable, so 1 or 8 bits written - it will "waste" a byte at very least. — Eugene Sh., Oct 03 '19 at 20:38
Most EEPROMs I'm aware of are programmed in 32-bit chunks, some even in 64-bit chunks. You can certainly slice those up any way you like by masking off and shifting the bits you need, but you still have to program them as they are aligned (in particular, to change a value, you must first read the whole word, erase the whole word, then re-program it, even if only one bit is changed). — Lee Daniel Crocker, Oct 03 '19 at 20:39
Buffer your output in RAM then write out at all the info at once to your hardware EEPROM N-bytes at a time as your hardware allows. On the last write, you will have to add a few extra zero bits to pad to the write limit. Don't write a bunch of small chunks all over the place. This is inefficient and also adds wear and tear to the lifetime of the EEPROM. We had to do this sort of thing all the time on the black and white Gameboy for saving game data. — Michael Dorgan, Oct 03 '19 at 20:55
Bummer about your pasting problem. I can't chase your link, so I'm afraid we're outta luck. — Steve Summit, Oct 03 '19 at 20:55
You can also use bitfields within structures to simplify the and/or math for lining up your bits. Then you can just output the whole structure at once. — Michael Dorgan, Oct 03 '19 at 20:58
An optimal solution may depend on how you need to update the EEPROM, the size of the EEPROM, and how much RAM you have to spare. Do you for example need to write "random access" or are all values written sequentially at the same time? — Clifford, Oct 03 '19 at 22:14

Graeme · Answer 1 · 2019-10-03T22:23:52.797

This function should take the number of bits given by bitsPerVal from each value in the input array pVals and pack them into the byte array pointed to by pOutBytes:

#include <stdint.h>

void pack_bits(uint32_t *pVals, size_t numVals, int bitsPerVal, uint8_t *pOutBytes)
{
    uint32_t mask = ~(UINT32_MAX << bitsPerVal);
    int outBitsLeft = 8;
    int inBitsLeft = bitsPerVal;

    while(numVals > 0)
    {
        if(inBitsLeft > outBitsLeft)
        {
            inBitsLeft -= outBitsLeft;
            *pOutBytes |= (*pVals & mask) >> inBitsLeft; 
            mask >>= outBitsLeft;
            outBitsLeft = 0;
        }
        else
        {
            outBitsLeft -= inBitsLeft;
            *pOutBytes |= (*pVals & mask) << outBitsLeft;
            mask = ~(UINT32_MAX << bitsPerVal);
            inBitsLeft = bitsPerVal;
            --numVals;
            ++pVals;
        }

        if(0 == outBitsLeft)
        {
            outBitsLeft = 8;
            ++pOutBytes;
        }
    }
}

The array pointed to by pOutBytes must suitably sized (ie ((numVals*bitsPerVal) + 7) / 8) and initialised to zero before calling. You can write it to your EEPROM after.

Hopefully this works well, I have done much testing on it though.

score 0 · Answer 2 · answered Oct 03 '19 at 21:02

Here is an example of how 10 bits (actually 16-bits when written...) from 2 different fields could write to 16-bits of output.

struct EEPROM_Output 
{
 uint16_t a   : 9;  // 0 - 511 can be stored here
 uint16_t b   : 1;  // 0 or 1 here.
 uint16_t pad : 6;  // Future use - we place this here to make it obvious that there are bits remaining.
};

void foo()
{
    struct EEPROM_Output save;
    save.a = 100;
    save.b = 1;

    WriteToEEPROM(&save, sizeof(save));
}

But how can i chain more 9 bit-values in a row into the EEPROM or a array with this? — Tobey66, Oct 05 '19 at 07:30

Writing 9-bit values to byte array (or EEPROM) without wasting remaining bit in the following byte

2 Answers2