deserialise google flat buffers

Question

I have a nodejs application that uses google flat buffers.

the fbs file for the drinks schema:

namespace MyAlcoholist;

table Drink {
drink_type_name: string;
drink_company_name: string;
drink_brand_name: string;
drink_flavor_type_name : string;
liquid_color_type_name : string;
liquid_color_is_transparent : bool;
alcohol_vol : float;
calories_for_100g : uint;
global_image_id: uint;
drink_flavor_id: uint;
}

table Drinks { drinks:[Drink]; }

root_type Drinks;

compiled it with flatc -s drinks.fbs, it generated the js file drinks_generated.js. so far so good.

I use the following code in my nodejs server to prepare and create the flat buffer out of an array:

flatBuffersUtil.js

var flatbuffers = require('../js/flatbuffers').flatbuffers;
var builder = new flatbuffers.Builder();

var drinks = require('../fbs/drinks_generated').MyAlcoholist; // Generated by `flatc`.

function drinkArrayToBuffer(drinkArray) {
var drinksVectArray = [];
drinkArray.forEach(function (element, index, array) {
    var drinkObj = element;
    var drinkBrandName = builder.createString(drinkObj.drink_brand_name);
    var drinkCompanyName = builder.createString(drinkObj.drink_company_name);
    var drinkflavorTypeName = builder.createString(drinkObj.drink_flavor_type_name);
    var drinkTypeName = builder.createString(drinkObj.drink_type_name);
    var liquidColorTypeName = builder.createString(drinkObj.liquid_color_type_name);
    drinks.Drink.startDrink(builder);
    drinks.Drink.addAlcoholVol(builder, drinkObj.alcohol_vol);
    drinks.Drink.addCaloriesFor100g(builder, drinkObj.calories_for_100g);
    drinks.Drink.addDrinkBrandName(builder, drinkBrandName);
    drinks.Drink.addDrinkCompanyName(builder, drinkCompanyName);
    drinks.Drink.addDrinkFlavorId(builder, drinkObj.drink_flavor_id);
    drinks.Drink.addDrinkFlavorTypeName(builder, drinkflavorTypeName);
    drinks.Drink.addDrinkTypeName(builder, drinkTypeName);
    drinks.Drink.addGlobalImageId(builder, drinkObj.global_image_id);
    drinks.Drink.addLiquidColorIsTransparent(builder, drinkObj.is_transparent);
    drinks.Drink.addLiquidColorTypeName(builder, liquidColorTypeName);
    var drink = drinks.Drink.endDrink(builder);
    drinksVectArray.push(drink);
})
var drinksVect = drinks.Drinks.createDrinksVector(builder, drinksVectArray);
builder.finish(drinksVect);
var buf = builder.dataBuffer();
var drinksArray = drinks.Drinks.getRootAsDrinks(buf);
return buf;
}

module.exports.drinkArrayToBuffer = drinkArrayToBuffer;

usage of flatBuffersUtil.js:

...
var data = flatBuffersUtil.drinkArrayToBuffer(result);
res.send(data);

the client is done with angularjs 1.5.0 when I receive the buffer I try to create an object out of it using the following code:

in main index file:

<script type="text/javascript" src="js/flatbuffers.js"></script>
<script type="text/javascript" src="js/fbs/drinks_generated.js"></script>

then in angular controller:

           $http({
                data: data,
                method: 'POST',
                url: 'https://myalcoholist.com:8888/drink/get_list/all_drinks',
            }).then(function successCallback(response) {
                var buffer = response.data;
                var drinks = MyAlcoholist.Drinks.getRootAsDrinks(buffer);
                deferred.resolve(drinks);
                // this callback will be called asynchronously
                // when the response is available
            }, function errorCallback(response) {
                // called asynchronously if an error occurs
                // or server returns response with an error status.
            });

the problem is that the function MyAlcoholist.Drinks.getRootAsDrinks(buffer); fails with the following error:

TypeError: bb.position is not a function
at Function.MyAlcoholist.Drinks.getRootAsDrinks (drinks_generated.js:256)
at successCallback (admin-drinks-controller.js:22)
at angular.js:15552
at m.$eval (angular.js:16820)
at m.$digest (angular.js:16636)
at m.$apply (angular.js:16928)
at g (angular.js:11266)
at t (angular.js:11464)
at XMLHttpRequest.u.onload (angular.js:11405)

now.. the function's code in the generated drinks_generated.js file is the following:

MyAlcoholist.Drinks.getRootAsDrinks = function(bb, obj) {
return (obj || new MyAlcoholist.Drinks).__init(bb.readInt32(bb.position()) + bb.position(), bb);
};

so bb is supposed to a bytebuffer type with the function position(), but what I receive is an Object of bytes. I guess I need to typecast it first or somehow... but this is just a guess.

any ideas?

update

ok first i revised my drinksArrayToBuffer function to the following code:

function drinkArrayToBuffer(drinkArray) {
var drinksVectArray = [];
drinkArray.forEach(function (element, index, array) {
    var drinkObj = element;
    var drinkBrandName = builder.createString(drinkObj.drink_brand_name);
    var drinkCompanyName = builder.createString(drinkObj.drink_company_name);
    var drinkflavorTypeName = builder.createString(drinkObj.drink_flavor_type_name);
    var drinkTypeName = builder.createString(drinkObj.drink_type_name);
    var liquidColorTypeName = builder.createString(drinkObj.liquid_color_type_name);
    drinks.Drink.startDrink(builder);
    drinks.Drink.addAlcoholVol(builder, drinkObj.alcohol_vol);
    drinks.Drink.addCaloriesFor100g(builder, drinkObj.calories_for_100g);
    drinks.Drink.addDrinkBrandName(builder, drinkBrandName);
    drinks.Drink.addDrinkCompanyName(builder, drinkCompanyName);
    drinks.Drink.addDrinkFlavorId(builder, drinkObj.drink_flavor_id);
    drinks.Drink.addDrinkFlavorTypeName(builder, drinkflavorTypeName);
    drinks.Drink.addDrinkTypeName(builder, drinkTypeName);
    drinks.Drink.addGlobalImageId(builder, drinkObj.global_image_id);
    drinks.Drink.addLiquidColorIsTransparent(builder, drinkObj.is_transparent);
    drinks.Drink.addLiquidColorTypeName(builder, liquidColorTypeName);
    var drink = drinks.Drink.endDrink(builder);
    drinksVectArray.push(drink);
})
var drinksVect = drinks.Drinks.createDrinksVector(builder, drinksVectArray);
drinks.Drinks.startDrinks(builder);
drinks.Drinks.addDrinks(builder, drinksVect);
var endDrinksOffset = drinks.Drinks.endDrinks(builder);
drinks.Drinks.finishDrinksBuffer(builder, endDrinksOffset);
var buf = builder.dataBuffer();
return buf;
}

now in the server side i am able to deserialise the buffer and use the data properly. the problem is with deserialising on the client side.

after i'm sending the buffer, the client side receives the following object:

Object
 bytes_: Object  
 position_: 7148
 __proto__: Object

now.. I created the following function to convert the buffer to json object on the client side:

function drinksByteArrayToArray(buffer) {
var res = [];
var byteBuffer =new flatbuffers.ByteBuffer(buffer);
var drinks = MyAlcoholist.Drinks.getRootAsDrinks(byteBuffer);
var drinksLength = drinks.drinksLength();
for (var i=0;i<drinksLength;i++) {
    var drink = drinks.drinks(i);
    var drinkObj = {
        drink_flavor_id: drink.drinkFlavorId(),
        drink_type_name: drink.drinkTypeName(),
        drink_company_name: drink.drinkCompanyName(),
        drink_brand_name: drink.drinkBrandName(),
        drink_flavor_type_name: drink.drinkFlavorTypeName(),
        liquid_color_type_name: drink.liquidColorTypeName(),
        is_transparent: drink.liquidColorIsTransparent(),
        alcohol_vol: drink.alcoholVol(),
        calories_for_100g: drink.caloriesFor100g(),
        global_image_id: drink.globalImageId
    }
    res.push(drinkObj);
}
return res;
}

when I excute this function, the length of drinks is zero. so the buffer seems to be empty.

now I think that the problem is with the following code:

var byteBuffer =new flatbuffers.ByteBuffer(buffer);

I think i need convert the data to bytebuffer in some other way.

I understood that I need to give flatbuffers.ByteBuffer an array of bytes, so I tried doing the following thing:

var byteBuffer =new flatbuffers.ByteBuffer(buffer.bytes_);

but the results are the same.

any ideas?

yes, looks like you need to create a `flatbuffers.ByteBuffer` using an array of bytes — Aardappel, Feb 19 '16 at 16:27
I agree with Aardappel. You probably need to wrap `var buffer = response.data` in a `flatbuffers.ByteBuffer`. (Something like `var buffer = flatbuffers.ByteBuffer(response.data)`.) P.S. You do not seem to use `drinksArray` at all in the `flatBuffersUtil.js` snippet. `var drinksArray = drinks.Drinks.getRootAsDrinks(buf); // unused` — MrHappyAsthma, Feb 20 '16 at 00:03
You're sending the whole ByteBuffer across. Try `res.send(data.bytes())` — Aardappel, Feb 20 '16 at 16:50
@Aardappel, thanks for your response. using buffer.bytes_ is the same as sending the .bytes(), anyways i tried that and the results are the same. — ufk, Feb 20 '16 at 17:07

score 6 · Accepted Answer · edited Jun 20 '20 at 09:12

thanks for your support. i was able to finally come with a solution. what I was missing is to use setPosition on the byteBuffer after setting it's bytes. Why? I don't know! all I know is that i tried that and now it works.

so i'm sending the whole buffer that contains bytes_ for the data and position_ for... i guess... the length or something. so after I'm sending the buffer on the client side i perform these two commands:

var byteBuffer =new flatbuffers.ByteBuffer(buffer.bytes_);
byteBuffer.setPosition(buffer.position_);

if anyone can assist here in providing more information on why i need to do that, that would be swell :) anyhow... it works yay!

update

I opened an issue on https://github.com/google/flatbuffers/issues/3781 and I got my answer :)

There are examples in https://github.com/google/flatbuffers/blob/master/tests/JavaScriptTest.js. To serialize a flatbuffer:

var fbb = new flatbuffers.Builder();
// ... build the flatbuffer ...
var uint8Array = fbb.asUint8Array();

To deserialize a flatbuffer:

var bb = new flatbuffers.ByteBuffer(uint8Array);
var drinks = MyAlcoholist.Drinks.getRootAsDrinks(bb);

thanks!

deserialise google flat buffers

update

1 Answers1

update