I have been encountering an issue for a long time but I haven't found any solution yet, despite reading other answers that related to this issue.
I have 2 different mongo dbs and I want to concat 2 streams that came from each one, and send the result to the client.
The problem is that the response contains 2 arrays (the first of db1, the second of db2) instead of one array that contains the data of both.
How can I concat the 2 arrays in the response into one array?
In order to simulate the problem I did the same with files.
This is my code:
const express = require('express');
const fs = require('fs');
const CombinedStream = require('combined-stream');
const MultiStream = require('multistream');
const JSONStream = require('jsonstream-next');
const app = express();
function concatStreams(streams) {
var combinedStream = CombinedStream.create();
streams.forEach(stream => combinedStream.append(stream));
return combinedStream;
};
function createDbStream(connection, dbName, colName1, colName2){
const collection = connection.useDb(dbName).db.collection(colName1);
const pipeline = [
{$match: {}},
{$project: { _id: 1, value: 1 }},
{$unionWith: {
coll: colName2,
pipeline: [ {$match: {}}, {$project: { _id: 1, value: 1 }} ]
}
}
];
const stream = collection.aggregate(pipeline).stream()
.pipe(JSONStream.stringify());
return stream;
}
async function getDbStreams(){
const connection1 = await mongoose.createConnection("dburl1", { useUnifiedTopology: true, useNewUrlParser: true, });
const connection2 = await mongoose.createConnection("dburl2", { useUnifiedTopology: true, useNewUrlParser: true, });
const stream1 = createDbStream(connection1, "dbName", "colName1", "colName2");
const stream2 = createDbStream(connection2, "dbName", "colName1", "colName2");
return [stream1, stream2];
}
app.post('/concat-file-stream1', (req, res) => {
const filePaths = ["/file1.json", "/file2.json"];
const streams = filePaths.map(path=>fs.createReadStream(path));
const result = concatStreams(streams);
result.pipe(res.type('json'));
});
app.post('/concat-file-stream2', (req, res) => {
const filePaths = ["/file1.json", "/file2.json"];
const streams = filePaths.map(path=>fs.createReadStream(path));
const result = MultiStream(streams)
result.pipe(res.type('json'));
});
app.post('/concat-db-stream1', async (req, res) => {
const streams = await getDbStreams();
const result = concatStreams(streams);
result.pipe(res.type('json'));
});
app.post('/concat-db-stream2', async (req, res) => {
const streams = await getDbStreams();
const result = new MultiStream(streams);
result.pipe(res.type('json'));
});
app.listen(3000, () => {
console.log(`Server running on port ${3000}.`);
});
My files:
file1.json:
[
{ "id": "1", "value": 1 },
{ "id": "2", "value": 2 },
{ "id": "3", "value": 3 }
]
file2.json:
[
{ "id": "4", "value": 4 },
{ "id": "5", "value": 5 },
{ "id": "6", "value": 6 }
]
The response:
