Get get unique ObjectID's from an array containing duplicates

Question

So I have an array of ObjectID's, for example:

console.log(objectIdArray);

gives [ObjectID, ObjectID, ObjectID].

However there are duplicates here, as seen when mapping to ID strings:

var idArray = objectIdArray.map(objectId => objectId.toString());
console.log(idArray);

gives ["5afa54e5516c5b57c0d43227", "5afa54e5516c5b57c0d43227", "5afa54f0516c5b57c0d43228"] where you can see the ID ending in 27 is duplicated twice.

How can I filter this array of ObjectID's to remove the duplicates (keeping the complete ObjectID objects, not just the ID string values)?

I often add the values of the array to a map ( `var map = {}; array.forEach(id => map[id] = true // or original object)` ) and then return `Object.keys(map)` or `Object.values(map)`. O(n) time. — jrasm91, May 21 '18 at 03:07

Pavlo · Accepted Answer · 2018-05-20T20:43:06.550

const removeDuplicates = inputArray => {
    const ids = [];
    return inputArray.reduce((sum, element) => {
       if(!ids.includes(element.toString()){
           sum.push(element);
           ids.push(element.toString());
       }
       return sum;
    }, []);
};

This solution will remove all the objects that aren't the first object with a certain Id.

We fill an Array with the ids, then we check if the ids already are filled in the current list.

The above solution can be potentially slow if there is a lot of elements since you need to check the list of ids which is O(n) for each iteration in the inputArray which would put the algorithm at O(n^2)+O(n)

So instead, we can sort it first based on toString() then we can just verify that the current id didn't match the last id we saw.

const removeDuplicates = inputArray => {
    const sortedArray = inputArray.sort((a,b) => (a.toString() > b.toString() ? 1 : (a.toString() < b.toString() ? -1 : 0)));

    let lastSeen = undefined;
    return sortedArray.reduce((sum, element) => {
       if(lastSeen !== element.toString()){
           sum.push(element);
       }
       lastSeen = element.toString();
       return sum;
    }, []);
};

Now the algorithm is O(n log n) + O(n) assuming sort uses Merge Sort

score 0 · Answer 2 · answered May 20 '18 at 20:04

0

If you use ES6 you can take Sajeetharan's answer, but create a set of objects, not their ids:

let nodupes = [...new Set(objectIdArray)];

answered May 20 '18 at 20:04

Andrew Svietlichnyy

743
1
6
13

They work on everything: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set – Andrew Svietlichnyy May 20 '18 at 20:06
I don't believe JavaScript Set's work with full objects, [see here](https://stackoverflow.com/questions/41404607/why-does-javascript-set-not-do-unique-objects). Doing what you mention above doesn't remove any ObjectID's. – Kevin Novak May 20 '18 at 20:10
That's because objects are not compared by their contents, they must be references to the same object. – Andrew Svietlichnyy May 20 '18 at 20:26

score 0 · Answer 3 · answered May 20 '18 at 20:39

I recommend MongoDB Aggregation Pipeline to prevent the situation of having a final result with duplicate ObjectId values.

However:

// Define callback Function to receive modified Array
var receiveModifiedArray = function(objectIdArray) {
    // log modified Array to console
    console.log(objectIdArray);
}

// Remove duplicate ObjectId values
function removeDuplicateObjectIdValues(objectIdArray, callback) {

    // Iterate through each ObjectId
    objectIdArray.forEach((currentValue, index, array) => {
        // evaluate Array items with index greater than 0
        if(index > 0) {

            // check ObjectId string values for type and value equality
            if(currentValue.toString() == array[index -1].toString()) {
                /**
                 * The splice() method changes the contents of an array
                 * by removing existing elements and/or adding new elements.
                 * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice
                 */
                objectIdArray.splice(index,1);
            }

            // If processing last item in Array, callback
            if(index == array.length +1) {
                callback(objectIdArray);
            }
        }
    });

    // Return to move on to next message in call stack
    return;
}

// Remove duplicate ObjectId values
removeDuplicateObjectIdValues(objectIdArray,receiveModifiedArray);

Get get unique ObjectID's from an array containing duplicates

3 Answers3