I'd need your help with the max number of connection allowed with mongodb. I'm getting this error:
AssertionError [ERR_ASSERTION]: null == 'MongoError: write EPIPE'
At first glance I thought of following this SO solution and increase the max connection variable in the conf. file. But then I thought, it's silly, I don't really need these many connections, I should be able to limit the number to not overload my server and I actually found another SO solution that goes in that direction.
So I went on in trying to figure out how can I create a new DB per instance of a variable, since it's what I'm trying to achieve and found this in the NodeJS driver doc. for MongoDB.
My code basically loop through 2 different Arrays. First: ccxt.exchanges (around 30 items) and then pairs (4 items) and I create a new connection each iteration of both of the arrays... Not sure if I'm clear but here's the code:
var _ = require("lodash");
var ccxt = require("ccxt");
var MongoClient = require("mongodb").MongoClient,
assert = require("assert");
let pairs = ["ETH/EUR", "BTC/EUR", "LTC/EUR", "BCH/EUR"];
let delay = 1200;
// NOTE: Server initiation:
setInterval(function() {
ccxt.exchanges.forEach(r => {
let exchange = new ccxt[r]();
if (exchange.hasFetchOrderBook) {
pairs.forEach(p => {
exchange
.fetchOrderBook(p)
.then(async order => {
if (_.isObject(order) && order.bids[0][1]) {
let now = Math.floor(new Date());
order.mkt = r;
order.pair = p;
let mkmodel = r.charAt(0).toUpperCase() + r.slice(1) + "order";
var url = `mongodb://localhost:27017/${"order" + p.slice(0, 3)}`;
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
var collection = db.collection(mkmodel);
collection.insert(order, function(err, result) {});
db.close();
});
let compOrder = {
mkt: order.mkt,
pair: order.pair,
aprice: order.asks[0][0],
avol: order.asks[0][1],
bprice: order.bids[0][0],
bvol: order.bids[0][1],
sn: order.timestamp,
ping: now - order.timestamp,
fees: exchange.fees
};
var irl = `mongodb://localhost:27017/${"comp" + p.slice(0, 3)}`;
MongoClient.connect(irl, function(err, db) {
assert.equal(null, err);
var collection = db.collection(mkmodel);
collection.insert(compOrder, function(err, result) {});
db.close();
});
console.log(compOrder);
await new Promise(resolve => setTimeout(resolve, delay));
} else {
}
})
.catch(e => {});
});
}
});
}, 2000);
To simplify my code, it goes something like this:
let array1 = [1, 2, 3, 4, 5, ..., 30];
let array2 = [1, 2, 3, 4];
array1.forEach(i => {
array2.forEach(p => {
new mongoConnection1 ==> Creates DB from Array2 (for instance: db = 1 on first iteration)
Each Array1 item creates a Collection
db.close();
someObjectManipulation
new mongoConnection2 ==> Create different DB from Array2 as well
Each Array1 item creates a Collection as well
db.close();
})
})
Now since my code is asynchronous (and I'd rather keep it this way) the number of connections is too high and I get the error. I would like to do something like that:
let array1 = [1, 2, 3, 4, 5, ..., 30];
let array2 = [1, 2, 3, 4];
new mongoConnection
array1.forEach(i => {
array2.forEach(p => {
Create DB from Array2 (for instance: db = 1 on first iteration)
Each Array1 item creates a Collection
someObjectManipulation
Create different DB from Array2 as well
Each Array1 item creates a Collection as well
})
})
db.close();
in order to limit the number of connections to 1... But I'm not sure if it's possible at all? If it were, I guess that I would have to use new Db(databaseName, topology, options)(as presented in the Node driver docs) somewhere in my loops but I'm not sure how to do that...
I hope my question is clear enough otherwise please let me know in the comments and I'll try to clarify as much as needed! Thanks in advance for your help folks!
EDIT:
In order to classify and treat my data correctly I would like to have a database for each pair stated in the Array. Obviously, I could re-write my app so that it is not needed and have one database with many collections and a key for the pairs inside each document but the data would just become messier.
Thanks @Alistair_Nelson for precising this part wasn't clear :)
