How to make the cartesian products of vectors, when the number of elements > 1000?

Question

I have 1,2,...,n piece of vectors. Every vector has more than 10000 elements and I have to get the cartesian product of these vectors. I have a code, what's working, but only under 1000 elements and under 4 vector. I'd like to write the cartesian product to a file, but if the output file is bigger than 1GB i got : "terminate called after throwing an instance of 'std::bad_alloc' what(): std::bad_alloc".

My primary question is, how can I fix this memory allocation error?

Here is a runable piece of my code:

#include <iostream>
#include <vector>
#include <algorithm>
#include <time.h>
#include <fstream>
#include <math.h>

using namespace std;

vector<double> makeVectorByRange(double min, double max, double step){
    vector<double> out = {};
    for( ; min <= max; min+=step){
        out.push_back(min);
    }
    return out;
} 


void cart_product_solve_and_write_to_file (const vector<vector<double>>& inpV) {
    vector<vector<double>> out = {{}};

    std::ofstream outputFile;
    std::fixed;

    for (auto& u : inpV) {
        vector<vector<double>> r;
        r.clear();
        for (auto& x : out) {

            //make/open file, append
            outputFile.open ("out.csv", std::ofstream::out | std::ofstream::app);
            outputFile.precision(8);

            for (auto y : u) {
                r.push_back(x);
                r.back().push_back(y);

                if( r.back().size() == inpV.size() ){

                    // write the input parameters of griewank to file
                    for(double asd : r.back()){
                        outputFile << asd << ";";
                    }

                    outputFile << "; \n";
                    outputFile << std::flush;

                    r.back().clear();
                }
            }
            //close file
            outputFile.close();
        }
        out.swap(r);
    }  
}

// Pure cartesian product. This function returns the cartesian product as a vector, but if the input vectors are too big, it has an error
/*
vector < vector<double> > cartesian_product (const vector< vector<double> >& inpV) {
    vector< vector<double> > out = {{}};
    for (auto& u : inpV) {
        vector< vector<double> > r;
        for (auto& x : out) {
            for (auto y : u) {
                r.push_back(x);
                r.back().push_back(y);
            }
        }
        out.swap(r);
    }
    return out;
}
*/

int main(){

    clock_t tStart = clock();

    // it works
    // const vector<vector<int> > test ={ {0,1,2,3,4}, {5,6,7}, {8,9,10,11,12,13} };
    // vector<vector<int> > cart_prod = cartesian_product(test);

    vector <vector<double> > test = {};
    test.push_back( makeVectorByRange( 0, 0.5, 0.001) );
    test.push_back( makeVectorByRange( 0, 0.5, 0.001) );
    test.push_back( makeVectorByRange( 0, 0.5, 0.001) );

    cart_product_solve_and_write_to_file(test);

    printf("Time taken: %.6fs\n", (double)(clock() - tStart)/CLOCKS_PER_SEC);

    return 0;
}

Answer 1

You need to iterate over all combinations of the resulting cartesian product. This is typically achieved by recursion. In each recursion level you then iterate over elements of one input vector.

Here is a sample solution for printing the resulting combinations to std::cout. You can easily modify it for printing to a file by providing an additional reference parameter to an opened std::ofstream object to the recursive function.

#include <iostream>
#include <vector>

template <typename T>
void vector_cartesian_product_helper(
  const std::vector<std::vector<T>>& v, std::vector<T>& combination, size_t level)
{
  if (level == v.size()) {
    for (auto elem : combination)
      std::cout << elem << ";";
    std::cout << "\n";
  }
  else {
    for (const auto& elem : v[level]) {
      combination[level] = elem;
      vector_cartesian_product_helper(v, combination, level + 1);
    }
  }
}

template <typename T>
void vector_cartesian_product(const std::vector<std::vector<T>>& v)
{
  std::vector<T> combination(v.size());
  vector_cartesian_product_helper(v, combination, 0);
}

int main(){
  std::vector<std::vector<int>> test = {{0,1,2,3,4}, {5,6,7}, {8,9,10,11,12,13}};
  vector_cartesian_product(test);
}

Live demo: https://wandbox.org/permlink/PoyEviWGDtEpvN1z

It works with vectors of any sizes and does uses additional memory only of O(N) where N is a number of vectors.