0

Does anyone have an example of Cereal based armadillo matrix serialization to JSON? Binary serialization below seems to be working.

Inside mat_extra_meat.hpp

template<class Archive, class eT>
typename std::enable_if<cereal::traits::is_output_serializable<cereal::BinaryData<eT>, Archive>::value, void>::type
save( Archive & ar, const Mat<eT>& m ) {
    uword n_rows = m.n_rows;
    uword n_cols = m.n_cols;
    ar( n_rows );
    ar( n_cols );
    ar( cereal::binary_data(
        reinterpret_cast< void * const >( const_cast< eT* >( m.memptr() ) ),
        static_cast< std::size_t >( n_rows * n_cols * sizeof( eT ) ) ) );
}

template<class Archive, class eT>
typename std::enable_if<cereal::traits::is_input_serializable<cereal::BinaryData<eT>, Archive>::value, void>::type
load( Archive & ar, Mat<eT>& m ) {
    uword n_rows;
    uword n_cols;
    ar( n_rows );
    ar( n_cols );

    m.resize( n_rows, n_cols );

    ar( cereal::binary_data(
        reinterpret_cast< void * const >( const_cast< eT* >( m.memptr() ) ),
        static_cast< std::size_t >( n_rows * n_cols * sizeof( eT ) ) ) );
}

Test with this:

int main( int argc, char** argv ) {

    arma::mat xx1 = arma::randn( 10, 20 );
    std::ofstream ofs( "test", std::ios::binary );
    cereal::BinaryOutputArchive o( ofs );
    o( xx1 );
    ofs.close();
    // Now load it.
    arma::mat xx2;
    std::ifstream ifs( "test", std::ios::binary );
    cereal::BinaryInputArchive i( ifs );
    i( xx2 );

}
SparcU
  • 742
  • 1
  • 7
  • 27

1 Answers1

1

You have two options for JSON serialization - you can take a quick and dirty approach that won't really be human readable, or you can make it human readable at the cost of increased serialization size and time.


For the quick version, you can modify your existing code to use saveBinaryValue and loadBinaryValue, which exist within the text archives of cereal (JSON and XML).

e.g.:

ar.saveBinaryValue( reinterpret_cast<void * const>( const_cast< eT* >( m.memptr() ) ), 
                    static_cast<std::size_t>( n_rows * n_cols * sizeof( eT ) ) );

and similarly for the load.

This will base64 encode your data and write it as a string. You would of course need to specialize the function to only apply to text archives (or just JSON) within cereal.


The alternative is to individually serialize each element. You have two choices again here, the first is to serialize as a JSON array (e.g. myarray: [1, 2, 3, 4, 5, ...]) or as a bunch of individual name-value-pairs: "array1" : "1", "array2": "2", ...

The convention in cereal has been to use JSON arrays for dynamically re-sizable containers (e.g. vector), but because we're purely emphasizing readability with this example, I'll use arrays even though your armadillo matrix would not be something you would like users to be able to add or remove elements from using JSON:

namespace arma
{
  // Wraps a particular column in a class with its own serialization function.
  // This is necessary because cereal expects actual data to follow a size_tag, and can't
  // serialize two size_tags back to back without creating a new node (entering a new serialization function).
  //
  // This wrapper serves the purpose of creating a new node in the JSON serializer and allows us to
  // then serialize the size_tag, followed by the actual data
  template <class T>
  struct ColWrapper
  {
    ColWrapper(T && m, int c, int nc) : mat(std::forward<T>(m)), col(c), n_cols(nc) {}
    T & mat;
    int col;
    int n_cols;

    template <class Archive>
    void save( Archive & ar ) const
    {
      ar( cereal::make_size_tag( mat.n_rows ) );
      for( auto iter = mat.begin_col(col), end = mat.end_col(col); iter != end; ++iter )
        ar( *iter );
    }

    template <class Archive>
    void load( Archive & ar )
    {
      cereal::size_type n_rows;

      // Test to see if we need to resize the data
      ar( cereal::make_size_tag( n_rows ) );
      if( mat.n_rows != n_rows )
        mat.resize( n_rows, n_cols );

      for( auto iter = mat.begin_col(col), end = mat.end_col(col); iter != end; ++iter )
        ar( *iter );
    }
  };

  // Convenience function to make a ColWrapper
  template<class T> inline
  ColWrapper<T> make_col_wrapper(T && t, int c, int nc)
  {
    return {std::forward<T>(t), c, nc};
  }

  template<class Archive, class eT, cereal::traits::EnableIf<cereal::traits::is_text_archive<Archive>::value> = cereal::traits::sfinae>
  inline void save( Archive & ar, const Mat<eT>& m )
  {
    // armadillo stored in column major order
    uword n_rows = m.n_rows;
    uword n_cols = m.n_cols;

    // First serialize a size_tag for the number of columns. This will make expect a dynamic
    // sized container, which it will output as a JSON array. In reality our container is not dynamic,
    // but we're going for readability here.
    ar( cereal::make_size_tag( n_cols ) );
    for( auto i = 0; i < n_cols; ++i )
      // a size_tag must be followed up with actual serializations that create nodes within the JSON serializer
      // so we cannot immediately make a size_tag for the number of rows. See ColWrapper for more details
      ar( make_col_wrapper(m, i, n_cols) );
  }

  template<class Archive, class eT, cereal::traits::EnableIf<cereal::traits::is_text_archive<Archive>::value> = cereal::traits::sfinae>
  inline void load( Archive & ar, Mat<eT>& m )
  {
    // We're doing essentially the same thing here, but loading the sizes and performing the resize for the matrix
    // within ColWrapper
    cereal::size_type n_rows;
    cereal::size_type n_cols;

    ar( cereal::make_size_tag( n_cols ) );
    for( auto i = 0; i < n_cols; ++i )
      ar( make_col_wrapper(m, i, n_cols) );
  }
} // end namespace arma

Example program to run the above:

int main(int argc, char* argv[])
{
  std::stringstream ss;
  std::stringstream ss2;

  {
    arma::mat A = arma::randu<arma::mat>(4, 5);
    cereal::JSONOutputArchive ar(ss);
    ar( A );
  }

  std::cout << ss.str() << std::endl;

  {
    arma::mat A;
    cereal::JSONInputArchive ar(ss);
    ar( A );
    {
      cereal::JSONOutputArchive ar2(ss2);
      ar2( A );
    }
  }

  std::cout << ss2.str() << std::endl;

  return 0;
}

and its output:

{
    "value0": [
        [
            0.786820954867802,
            0.2504803406880287,
            0.7106712289786555,
            0.9466678009609704
        ],
        [
            0.019271058195813773,
            0.40490214481616768,
            0.25131781792803756,
            0.02271243862792676
        ],
        [
            0.5206431525734917,
            0.34467030607918777,
            0.27419560360286257,
            0.561032100176393
        ],
        [
            0.14003945653337478,
            0.5438560675050177,
            0.5219157100717673,
            0.8570772835528213
        ],
        [
            0.49977436000503835,
            0.4193700240544483,
            0.7442805199715539,
            0.24916812957858262
        ]
    ]
}
{
    "value0": [
        [
            0.786820954867802,
            0.2504803406880287,
            0.7106712289786555,
            0.9466678009609704
        ],
        [
            0.019271058195813773,
            0.40490214481616768,
            0.25131781792803756,
            0.02271243862792676
        ],
        [
            0.5206431525734917,
            0.34467030607918777,
            0.27419560360286257,
            0.561032100176393
        ],
        [
            0.14003945653337478,
            0.5438560675050177,
            0.5219157100717673,
            0.8570772835528213
        ],
        [
            0.49977436000503835,
            0.4193700240544483,
            0.7442805199715539,
            0.24916812957858262
        ]
    ]
}
Azoth
  • 1,652
  • 16
  • 24
  • Thanks Azoth. It seems that still there are still some issues with json serailization. Json looks like below (No closing bracket for columns) { "value0": [ 1.0198155217002611, -1.7958489999259313, [ 0.41323348787432163, -0.41711609563173047, [ 1.0883809274165732, -1.0135035381568509, [] – SparcU Nov 16 '16 at 17:45
  • Do you know why ar( cereal::make_size_tag( n_rows ) ); does not cause json to add closing bracket to json array? – SparcU Nov 21 '16 at 13:34
  • I'll try and install armadillo and see what is going wrong. – Azoth Nov 21 '16 at 23:03
  • I've updated my answer with an actual working solution. My explanation as to what is going on is a bit lacking, but if you dig into the JSON serializer and read my comments it should make sense. – Azoth Nov 21 '16 at 23:54
  • Thank you Azoth! It worked. Now I am trying to expand your solution to arma::cube and unfortunately without much success – SparcU Nov 26 '16 at 17:15