Searching bit-field data in Boost multi index container

Question

I'm looking for the optimal solution for retrieving bit-field based objects from multi-index container. For simplicity, the data is:

enum Bit
{
    b0 = 1,
    b1 = 2,
    b2 = 4
};

struct Item
{
    int field; // contains Bit values
    int info;
};

As far as I know there could be 2 possibilities:

Use a custom predicate in equal_range call (This example is not working because I don't know how to define the predicate):

struct MyTag;

using boost::multi_index_container;
using namespace boost::multi_index;

typedef composite_key<Item,   
    member<Item, int, &Item::field>,
    member<Item, int, &Item::info> > CompositeKey;
typedef hashed_non_unique< tag<MyTag>, CompositeKey> Index1;
typedef boost::multi_index_container<Item, indexed_by<Index1> > Container;

struct Predicate: public Index1::pred_type
{
    /// what to put here?
};

void f()
{
    Item item1 = { int( b0 | b1), 123};
    Item item2 = { int( b1 | b2), 123};
    Item item3 = { int( b2), 123};

    Container c;
    c.insert( item1);
    c.insert( item2);
    c.insert( item3);

    auto result = c.get<MyTag>().equal_range( boost::make_tuple( int( b2), 123), Index1::hash_type(), Predicate());
    for( auto i = result.first; i != result.second; ++i)
    {
        std::cout << i->field << ' ';
        // expect item2 and item3
    }
}

Add accessors in Item and indexes in multi-index container for each bit from enumeration:

struct Item
{
    int field;
    int info;

    bool isB0() const { return ( field & b0) != 0; }
    bool isB1() const { return ( field & b1) != 0; }
    bool isB2() const { return ( field & b2) != 0; }
};


struct MyTag1;
struct MyTag2;
struct MyTag3;

using boost::multi_index_container;
using namespace boost::multi_index;

typedef composite_key<Item,   
    const_mem_fun<Item, bool, &Item::isB0>,
    member<Item, int, &Item::info> > CompositeKeyB0;
typedef composite_key<Item,   
    const_mem_fun<Item, bool, &Item::isB1>,
    member<Item, int, &Item::info> > CompositeKeyB1;
typedef composite_key<Item,   
    const_mem_fun<Item, bool, &Item::isB2>,
    member<Item, int, &Item::info> > CompositeKeyB2;
typedef hashed_non_unique< tag<MyTag1>, CompositeKeyB0> Index1;
typedef hashed_non_unique< tag<MyTag2>, CompositeKeyB1> Index2;
typedef hashed_non_unique< tag<MyTag3>, CompositeKeyB2> Index3;
typedef boost::multi_index_container<Item, indexed_by<Index1, Index2, Index3> > Container;

void f()
{
    Item item1 = { int( b0 | b1), 123};
    Item item2 = { int( b1 | b2), 123};
    Item item3 = { int( b2), 123};

    Container c;
    c.insert( item1);
    c.insert( item2);
    c.insert( item3);

    auto result = c.get<MyTag2>().equal_range( boost::make_tuple( true, 123));
    for( auto i = result.first; i != result.second; ++i)
    {
        std::cout << i->field << ' ';
        // expect item2 and item3
    }
}

what is the exact selection criteria? does searching for 0b0010 match key value 0b0011 ? — Richard Hodges, Jun 12 '16 at 14:26
Perhaps option 1 is not a possibility at all, but I'm not sure. — Flaviu, Jun 12 '16 at 15:03

score 1 · Answer 1 · answered Jun 12 '16 at 15:31

I think the way to think about this is:

"how would I create these indexes without boost?"

It would be something like this:

#include <unordered_map>
#include <list>
#include <functional>
#include <iostream>

enum Bit
{
    b0 = 1,
    b1 = 2,
    b2 = 4
};

struct Item
{
    int field;
    int info;

    bool isB0() const { return ( field & b0) != 0; }
    bool isB1() const { return ( field & b1) != 0; }
    bool isB2() const { return ( field & b2) != 0; }
};

template<int bit>
struct ItemHash
{
  bool operator()(const Item& item) const {
    auto accum = std::hash<int>()(item.info);
    if ((item.field & bit) == bit)
      accum += 1234567;
    return accum;
  }
};

template<int bit>
  struct ItemEqual
  {
    bool operator()(const Item& l, const Item& r) const 
    {
      auto lt = std::make_tuple((l.field & bit) == bit, l.info);
      auto rt = std::make_tuple((r.field & bit) == bit, r.info);
      return lt == rt;
    }
  };

struct Items
{
  Item& insert(Item item)
  {
    // store object
    auto i = _storage.insert(_storage.end(), item);

    // update indeces
    _by_b0.emplace(item, *i);
    _by_b1.emplace(item, *i);
    _by_b2.emplace(item, *i);

    _by_b2_and_b1.emplace(item, *i);

    return *i;
  };



  // object storage
  std::list<Item> _storage;

  // indexes we want to keep
  std::unordered_map<Item, std::reference_wrapper<Item>, ItemHash<b0>, ItemEqual<b0> > _by_b0;
  std::unordered_map<Item, std::reference_wrapper<Item>, ItemHash<b1>, ItemEqual<b1> > _by_b1;
  std::unordered_map<Item, std::reference_wrapper<Item>, ItemHash<b2>, ItemEqual<b2> > _by_b2;

  // multiple bits?
  std::unordered_map<Item, std::reference_wrapper<Item>, ItemHash<b2|b1>, ItemEqual<b2|b1> > _by_b2_and_b1;
};


int main()
{
    Item item1 = { int( b0 | b1), 123};
    Item item2 = { int( b1 | b2), 123};
    Item item3 = { int( b2), 123};

    Items c;
    c.insert( item1);
    c.insert( item2);
    c.insert( item3);

    auto result = c._by_b2.equal_range( Item { b2, 123 });
    for( auto i = result.first; i != result.second; ++i)
    {
        std::cout << i->second.get().field << ' ' << i->second.get().info << std::endl;
        // expect item2 and item3
    }
}

Looking at this, we'd begin to realise that these bits in field are actually attributes. Do we really want to create an index on every combination of attributes? It that really going to be more efficient and easier to maintain that simply storing our Items in a vector, and using a predicate in a for loop?

for example:

#include <vector>
#include <algorithm>
#include <iostream>
#include <tuple>

enum Bit
{
    b0 = 1,
    b1 = 2,
    b2 = 4
};

struct Item
{
    int field;
    int info;

    bool isB0() const { return ( field & b0) != 0; }
    bool isB1() const { return ( field & b1) != 0; }
    bool isB2() const { return ( field & b2) != 0; }
};

struct matches_bits_and_info {

    constexpr matches_bits_and_info(int bits, int info) : _bits(bits), _info(info) {}

    bool operator()(const Item& item) const {
        return std::make_tuple((item.field & _bits) , item.info)
        == std::tie(_bits, _info);
    }
    int _bits, _info;
};

int main()
{
    std::vector<Item> c;
    c.push_back({b0 | b1, 123});
    c.push_back({b0 | b2, 123});
    c.push_back({b2, 123});

    for (auto& item : c) {
        if (matches_bits_and_info(b2, 123)(item)) {
            std::cout << item.field << ' ' << item.info << std::endl;
        }
    }

}

expected results:

5 123
4 123

For this to be less efficient than the multi-index-container solution (or a hand-rolled solution), the data set will need to be massive.

Thank you! I just wanted the optimal solution using BMI. I hoped in a solution with a single index and a smart predicate but this is not possible. — Flaviu, Jun 13 '16 at 15:41

Joaquín M López Muñoz · Accepted Answer · 2016-06-13T14:15:01.533

Option 2 is perfectly fine, and in fact your solution works as intended. As for option 1, there is no way you can make it work, no matter how clever your Predicate class, an explanation follows: a non-unique hashed index groups (meaning it packs together) elements according to some equivalence criterion, which in your case is having the same value for both field and info data members. Let's forget about info for a while by pretending all elements have the same value for it, and concentrate on field: the six equivalence classes (clusters of elements) are then

0, 1, 2, 3, 4, 5, 6, 7

which are simply the different values that the three bits 0, 1 and 2 of field can take combinedly (these clusters do not necessarily appear in the order shown above). Now, if you try to do an equal_rangethat retrieves all elements with b2 set what you want is for the operation to return the elements in those clusters shown in bold letter below:

0, 1, 2, 3, 4, 5, 6, 7

which, in general, are not arranged together, so there is no way equal_range can return a couple of iterators ranging over these and only these elements.

score 1 · Answer 3 · answered Jun 13 '16 at 16:50

An alternative to 1 can be considered taking into account that the possible values of the field member are actually not that many (8), which is to pick up one by one the corresponding field values with a given bit set, see for_each_value in the following example:

Live Coliru Demo

#include <algorithm>
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/composite_key.hpp>

enum Bit
{
  b0 = 1,
  b1 = 2,
  b2 = 4
};

struct Item
{
  int field;
  int info;
};

struct MyTag;

using boost::multi_index_container;
using namespace boost::multi_index;

typedef composite_key<
  Item,   
  member<Item, int, &Item::field>,
  member<Item, int, &Item::info> > CompositeKey;
typedef hashed_non_unique< tag<MyTag>, CompositeKey> Index1;
typedef boost::multi_index_container<Item, indexed_by<Index1> > Container;

template<typename F>
F for_each_value(const Container& c,Bit b,int info,F f)
{
  for(auto field:{0,1,2,3,4,5,6,7}){
    if(field&b){
      auto r=c.equal_range(std::make_tuple(field,info));
      std::for_each(r.first,r.second,f);
    }
  }
  return f;
}

#include <iostream>

int main()
{
  Item item1 = { int( b0 | b1), 123};
  Item item2 = { int( b1 | b2), 123};
  Item item3 = { int( b2), 123};

  Container c;
  c.insert( item1);
  c.insert( item2);
  c.insert( item3);

  for_each_value(c,b2,123,[](const Item& i){
    std::cout << i.field << ' ';
  });
}

Output

4 6

This could be useful in memory constrained projects. Instead of having n indexes for each bit defined, the container is looked-up 2^n times. Thanks! — Flaviu, Jun 14 '16 at 06:31
Actually, it is looked up 2^(n-1) times, i.e. for half the possible values of the bitfield. — Joaquín M López Muñoz, Jun 14 '16 at 06:58

Searching bit-field data in Boost multi index container

3 Answers3