Deduplicate a merged array

Question

I have an app where I load data, store the data, and in some case, reload the data. But when I reload, I do not want to duplicate my data if they are already loaded, and, I do want to apply modification if there are.

I store the data 2 array of object, that looks like

[{ 
    created_at: "2019-02-20T18:18:37.19+09:00"
    display_order: 0
    h_id: "5c6d1b6d3b6fe20690e2ef80"
    is_related_post: false
    item_id: "5c5a865e3b6fe24da8de7f21"
    user_id: "5c468bfa3b6fe24f78a36cdc"
    _id: "5c6d1b6d3b6fe20690e2ef80"
}]

The code is as follows

var data1 = [{
  created_at: "2019-02-20T18:18:37.19+09:00",
  value: "HELLO",
  h_id: "5c6d1b6d3b6fe20690e2ef80",
  is_related_post: false,
  item_id: "5c5a865e3b6fe24da8de7f21",
  user_id: "5c468bfa3b6fe24f78a36cdc",
  _id: "5c6d1b6d3b6fe20690e2ef80"
}, {
  created_at: "2019-02-15T14:45:29.17+09:00",
  value: "HELLO",
  h_id: "5c6651f93b6fe24e40e2b0e9",
  is_related_post: false,
  item_id: "5c5a865e3b6fe24da8de7f22",
  user_id: "5c468bfa3b6fe24f78a36cdc",
  _id: "5c6651f93b6fe24e40e2b0e9"
}]
var data1Copy = [{
  created_at: "2019-02-15T14:45:29.17+09:00",
  value: "HELLO",
  h_id: "5c6651f93b6fe24e40e2b0e9",
  is_related_post: false,
  item_id: "5c5a865e3b6fe24da8de7f22",
  user_id: "5c468bfa3b6fe24f78a36cdc",
  _id: "5c6651f93b6fe24e40e2b0e9"
}]

var dataValueChange = [{
  created_at: "2019-02-15T14:45:29.17+09:00",
  value: "MODIFIED HELLO",
  h_id: "5c6651f93b6fe24e40e2b0e9",
  is_related_post: false,
  item_id: "5c5a865e3b6fe24da8de7f22",
  user_id: "5c468bfa3b6fe24f78a36cdc",
  _id: "5c6651f93b6fe24e40e2b0e9"
}]
var dataNewId = [{
  created_at: "2019-02-15T14:45:29.17+09:00",
  value: "MODIFIED HELLO",
  h_id: "5c6651f93b6fe24e40e2b0e9333",
  is_related_post: false,
  item_id: "5c5a865e3b6fe24da8de7f22",
  user_id: "5c468bfa3b6fe24f78a36cdc",
  _id: "5c6651f93b6fe24e40e2b0e9"
}]

var testFunction = (name, func, optParams1, optParams2) => {
  var t0 = performance.now();
  var result = func(optParams1, optParams2);
  var t1 = performance.now();
  console.log("Call " + name + " took " + (t1 - t0) + " milliseconds.");
  console.log("result :", result);
}


var DeduplicateArray = (array) => {
  var hashTable = {};

  return array.filter(function(el) {
    var key = JSON.stringify(el);
    var match = Boolean(hashTable[key]);
    return (match ? false : hashTable[key] = true);
  });
}

var DeduplicateArray2 = (from, array) => {
  var newStory = [];
  array.forEach(history => {
    let isAlreadyStored = false;
    from.forEach(storedHistory => {
      if (isAlreadyStored) return;
      if (history.h_id === storedHistory.h_id) {
        isAlreadyStored = true;
      }
    });
    if (!isAlreadyStored) {
      newStory.push(history);
    };
  });
  return newStory;
}

console.log("SHOULD RETURN THE DATA NOT DUPLICATED");
testFunction("sameObject1", DeduplicateArray, data1.concat(data1Copy));
testFunction("sameObject2", DeduplicateArray2, data1, data1.concat(data1Copy));
console.log("");
console.log("");

console.log("THIS SHOULD RETURN THE DATA NOT DUPLICATED, BUT MODIFIED");
testFunction("valueChange1", DeduplicateArray, data1.concat(dataValueChange));
testFunction("valueChange2", DeduplicateArray2, data1, data1.concat(dataValueChange));
console.log("");
console.log("");

console.log("THIS SHOULD ADD A DATA TO THE LIST");
testFunction("idChange1", DeduplicateArray, data1.concat(dataNewId));
testFunction("idChange2", DeduplicateArray2, data1, data1.concat(dataNewId));
console.log("");
console.log("");

https://codepen.io/anon/pen/KJYEOg?editors=0010

The problem I have with my two functions :

the forloop is very long to execute if I had a big amount of data, and do not return modification. If there is a way to make this nested forloop faster, I would use it cause I can do another way for the changes.

the stringify is very quick, works for exact match, but not when an object is modified, it will create a new object.

EDIT I USED THIS :

Converting lodash _.uniqBy() to native javascript

Instead of an array of objects, couldn't you store the objects by their h_id, like: `{"5c6d1b6d3b6fe20690e2ef80": {...}, "5c6651f93b6fe24e40e2b0e9": {...}` then see if that key already exists when reloading? — Charlie, Feb 21 '19 at 03:16

score 1 · Answer 1 · answered Feb 21 '19 at 03:05

1

I would suggest using lodash's _uniqBy function, if you are prepared to use a library. It would be simple as:

let unique = _.uniqBy(data, 'h_id');

You can be assured that lodash's implementation will be close to optimal.

answered Feb 21 '19 at 03:05

see sharper

11,505
8
46
65

following your answer I found this : https://stackoverflow.com/questions/40801349/converting-lodash-uniqby-to-native-javascript – Bobby Feb 21 '19 at 03:16

Shohel · Answer 2 · 2019-02-21T03:15:27.577

Here is the simple example of merging arrays

var array1 = ["Shohel","Rana"];
var array2 = ["Shipon", "Shohel"];
let noDuplicate = array1.filter ( i => array2.findIndex( a => i==a)==-1 );
let result = [...noDuplicate, ...array2];
console.log(result);

Your code could be written in this way

console.log("SHOULD RETURN THE DATA NOT DUPLICATED");
let noDuplicate = data1.filter ( i => data1Copy.findIndex(a => i.h_id==a.h_id)==-1 );
console.log(noDuplicate);

Deduplicate a merged array

2 Answers2