Compress and decompress utf8 string with brotli

Question

I'm trying to write compress and decompress a JSON string, but the decompression always fails with:

Error: Decompression failed
    at BrotliDecoder.zlibOnError [as onerror] (zlib.js:170:17) {
  errno: -6,
  code: 'ERR_CL_SPACE'
}

I can't send the buffer returned by the compressor, because I need to send the compressed string through an HTTP response to get it back.

Moreover, if I use base64 as format, all work but the string output is bigger than the simple JSON.stringify, so I would avoid it.

My code:

const zlib = require('zlib')

function compress (json) {
  zlib.brotliCompress(JSON.stringify(json), {
    params: {
      [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT
    }
  }, (err, data) => {
    console.log(err)
    console.log(data.toString('utf8'))
    console.log('-------------------------------')
    decompress(Buffer.from(data.toString('utf8'), 'utf8'))
  })
}

function decompress (str) {
  zlib.brotliDecompress(str, (err, data) => {
    console.log(err)
    console.log(data)
  })
}

const obj = {
  wt: '5de52e98aa54253147060a01',
  ex: ['b9ac4a6b-2e72-4bf3-abd5-debf2ece4ba4']
}
compress(obj)

Are there any params to get a valid utf8 string output from the compression?

Maybe this abstraction will help you. Hope it helps. https://www.npmjs.com/package/compression — Naor Levi, Dec 03 '19 at 08:23

Terry Lennox · Accepted Answer · 2019-12-03T10:21:43.583

I think you'll have to use base64 for sending data over the wire in any case, this will result is a size penalty of ~37%. This might actually be more efficient than sending UTF8 since many of the bytes will require a multi-byte encoding (when sending binary data).

In any case, compression and decompression work when we pass our compressed data as base64.

I believe the reason it was not working with UTF8 is that converting a buffer to utf8 and back again is not always reversible, that is you don't necessarily get the same result back (try encoding random data to utf8 and back again!)

const zlib = require('zlib')

// Encode the buffer in base 64. UCS2 will work, but UTF8 will not.
const encoding = "base64";

function compress (json) {
  zlib.brotliCompress(JSON.stringify(json), {
    params: {
      [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT
    }
  }, (err, data) => {
    if (err) {
      console.error("brotliCompress: An error occurred:", err);
    } else {
      console.log("Compressed data:", data.toString("base64"));
      console.log('-------------------------------')
      decompress(data.toString(encoding), encoding);
    }
  })
}

function decompress (encodedData, encoding) {
  const compressedData = Buffer.from(encodedData, encoding);
  zlib.brotliDecompress(compressedData, (err, data) => {
    if (err) {
      console.error("brotliDecompress: An error occurred:", err);
    } else {
      console.log("Decompressed data:", data.toString("utf8"));
    }
  });
}

const obj = {
  wt: '5de52e98aa54253147060a01',
  ex: ['b9ac4a6b-2e72-4bf3-abd5-debf2ece4ba4']
}

compress(obj);

And to show Buffer.from(data.toString("utf8"), "utf8") is not always reversible:

const crypto = require("crypto");
const data = crypto.randomBytes(100);

console.log("Bytes (base64):", data .toString("base64"));

const encoding = "utf8";
const dataUtf8 = data.toString(encoding);
const decoded = Buffer.from(dataUtf8, encoding);
console.log("Bytes (base64 decoded):", decoded.toString("base64"));

You'll see the round trip results in a different buffer.

Compress and decompress utf8 string with brotli

1 Answers1