nodejs write 64bit unsigned integer to buffer

Question

I want to store a 64bit (8 byte) big integer to a nodejs buffer object in big endian format.

The problem about this task is that nodejs buffer only supports writing 32bit integers as maximum (with buf.write32UInt32BE(value, offset)). So I thought, why can't we just split the 64bit integer?

var buf = new Buffer(8);

buf.fill(0) // clear all bytes of the buffer
console.log(buf); // outputs <Buffer 00 00 00 00 00 00 00 00>

var int = 0xffff; // as dezimal: 65535

buf.write32UInt32BE(0xff, 4); // right the first part of the int
console.log(buf); // outputs <Buffer 00 00 00 00 00 00 00 ff>

buf.write32UInt32BE(0xff, 0); // right the second part of the int
console.log(buf); // outputs <Buffer 00 00 00 ff 00 00 00 ff>

var bufInt = buf.read32UInt32BE(0) * buf.read32UInt32BE(4);
console.log(bufInt); // outputs 65025

As you see this nearly works. The problem is just splitting the 64bit integer and finding the missing 510 at reading it. Would somebody mind showing the solutions for these two issues?

Answer 1

I think what you are looking for is:

var bufInt = (buf.readUInt32BE(0) << 8) + buf.readUInt32BE(4);

Shift the first number by 8 bits and add (instead of multiplying), wich returns 65535

---

EDIT

Another way to write would be:

var buf = new Buffer(8);
buf.fill(0);

var i = 0xCDEF; // 52719 in decimal

buf.writeUInt32BE(i >> 8, 0); //write the high order bits (shifted over)
buf.writeUInt32BE(i & 0x00ff, 4); //write the low order bits

console.log(buf); //displays: <Buffer 00 00 00 cd 00 00 00 ef>

var bufInt = (buf.readUInt32BE(0) << 8) + buf.readUInt32BE(4);
console.log(bufInt); //displays: 52719

Answer 2

I'm confused because your example value of 0xFFFF is only 16-bit, not 64-bit.

Keep in mind that the JS number type is specified as an IEEE754 floating-point value, so it is not guaranteed to be able to hold a 64-bit unsigned value. If you want real 64-bit integer support, you need to use a module to provide it, like bignum. The readme for that has examples of reading and writing values to buffers.

Floating-point values can only represent values up to 2^53 - 1 without losing precision. You can see that in this example using standard JS numbers:

var b = new Buffer([0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF])
var firstHalf = b.readUInt32BE(0); // 4294967295
var secondHalf = b.readUInt32BE(4); // 4294967295

var val = firstHalf * 0x100000000 + secondHalf; // 18446744073709552000

The result of this is 18446744073709552000 when the proper value is 18446744073709551615.

var bignum = require('bignum');

var b = new Buffer([0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF])
var val = bignum.fromBuffer(b);

This results in a BigNum object with the value 18446744073709551615.

Also, to elaborate on your example code, the value you are using is only 16-bit, and you are trying to work with it using 32-bit functions. You can just do this:

var buf = new Buffer(2);

buf.fill(0) // clear all bytes of the buffer
console.log(buf); // outputs <Buffer 00 00>

var int = 0xffff; // as decimal: 65535

// Write it with a standard 16-bit function calls.
buf.writeUInt16BE(int);

// OR write it with 2 8-bit function calls.
buf.writeUInt8(int & 0xff, 0); // right the first part of the int
buf.writeUInt8((int >> 8) & 0xFF, 1); // right the second part of the int
console.log(buf); // outputs <Buffer ff ff>

// Read it as a 16-bit value.
var bufInt = buf.readUInt16BE(0);
console.log(bufInt);

// OR read it as two 8-bit values.
var bufInt = (buf.readUInt8(1) << 8) + buf.readUInt8(0);

Answer 3

Reading and writings UINT numbers up to Number.MAX_SAFE_INTEGER.

This works in node.js only, is not portable on browser side.

function uintToBase62(n) {
  if (n < 0) throw 'unsupported negative integer';

  let uintBuffer;
  if (n < 0x7FFFFFFF) {
    uintBuffer = new Buffer(4);
    uintBuffer.writeUInt32BE(n, 0);
  } else {
    // `~~` double bitwise operator
    // The most practical way of utilizing the power of this operator is to use it as a replacement
    // for Math.floor() function as double bitwise NOT performs the same operation a lot quicker.
    // You can use it, to convert any floating point number to a integer without performance overkill
    // that comes with Math.floor(). Additionally, when you care about minification of your code,
    // you end up using 2 characters (2 tildes) instead of 12.
    // http://rocha.la/JavaScript-bitwise-operators-in-practice
    const big = ~~(n / 0x0100000000);
    const low = (n % 0x0100000000);
    uintBuffer = new Buffer(8);
    uintBuffer.writeUInt32BE(big, 0);
    uintBuffer.writeUInt32BE(low, 4);
  }

  return uintBuffer.toString('hex');
}

to convert it

function uintFromBase62(uintBuffer) {
  const n = parseInt(uintBuffer.toString('hex'), 16);
  return n;
}

Answer 4

Reading / Writing 64bit values:

const int64 = Date.now()   // 1456909977176 (00 00 01 53 36 9a 06 58)
const b = new Buffer(8)
const MAX_UINT32 = 0xFFFFFFFF

// write
const big = ~~(int64 / MAX_UINT32)
const low = (int64 % MAX_UINT32) - big

b.writeUInt32BE(big, 0)  // 00 00 01 53 00 00 00 00
b.writeUInt32BE(low, 4)  // 00 00 01 53 36 9a 06 58

// read
var time = parseInt(b.toString('hex'), 16)
time == int64 // true

I use this code without any special modules.

UPDATE

Works only for numbers <= Number.MAX_SAFE_INTEGER

Answer 5

// sending time
var sending_time = new Date().getTime();
buffer.writeInt32LE(parseInt(sending_time & 0xffffffff, 10), 16);
buffer.writeInt32LE(parseInt(sending_time / 0xffffffff, 10), 20);

Answer 6

Bitwise operations in JavaScript/EcmaScript will force the number (or any value really) into a 32-bit integer.

You really need to use bignum for larger values if you need certain precision, or dealing with buffers with larger than 32-bit values.

var bignum = require('bignum');

//max safe integer to big number
var num = bignum(Number.MAX_SAFE_INTEGER.toString());
var buf = num.toBuffer({endian:'big',size:8 /*8-byte / 64-bit*/});

console.log(buf); // 

The example above uses Number.MAX_SAFE_INTEGER which is Math.pow(2,53)-1. If you need to use larger numbers, they should be accessed as bignum strings... if you can stay within the range of Integers in JS, you can keep them and convert as above.

Answer 7

This is not an exact answer to the question.
Expand the question, here I show an answer assuming that it is a way to convert a 64-bit value to a buffer value.

• Writing to buffer

buf = Buffer.alloc(8)
view = new DataView(buf.buffer)
view.setBigInt64(0, BigInt(0x1826d1705e0), false) // true when little endian
console.log(buf)
// Output: <Buffer 00 00 01 82 6d 17 05 e0>

• Reading from buffer

view = new DataView(buf.buffer)
n = Number(view.getBigUint64(0, false))
console.log(n.toString(16))
// Output: 1826d1705e0