I am working with an API that gives me data with a limit per request (here 25). Therefore, I have to recursively make promises with fetch. However, while most of my logic works, when I try to return from the function it will return an empty array. The image below will make it more clear.
const url = (conf_name) => {
return (
"https://api.elsevier.com/content/search/scopus?view=COMPLETE&cursor=*&query=CONFNAME(" +
conf_name +
")%20AND%20DOCTYPE(cp)%20AND%20SRCTYPE(p)&field=dc:creator,dc:title,dc:description,affilname,affiliation-city,affiliation-country,authkeywords,prism:doi,prism:doi,prism:coverDate"
);
};
const savePapers = (json, conf_name) => {
let temp = new Array();
for (let i = 0; i < json["search-results"]["entry"].length; i++) {
temp[i] = {
title: json["search-results"]["entry"][i]["dc:title"],
author: json["search-results"]["entry"][i]["dc:creator"],
publication_date: json["search-results"]["entry"][i]["prism:coverDate"],
doi: json["search-results"]["entry"][i]["prism:doi"],
abstract: json["search-results"]["entry"][i]["dc:description"],
author_keywords: json["search-results"]["entry"][i]["authkeywords"],
proceeding: conf_name,
};
}
return temp;
};
async function getPapers(final, url, conf_name) {
let total_amount_of_papers;
let next;
let position = 2;
try {
let response = await fetch(url, options);
let json = await response.json();
total_amount_of_papers = json["search-results"]["opensearch:totalResults"];
if (json["search-results"]["link"][position]["@ref"] == "prev")
next = json["search-results"]["link"][position + 1]["@href"];
next = json["search-results"]["link"][position]["@href"];
final = final.concat(savePapers(json, conf_name));
if (final.length === 50) {
console.log("hey",final.length);
return final;
}
await getPapers(final, next, conf_name);
} catch (error) {
console.log(error);
}
}
const createNewConf = async (conferences) => {
let final = new Array();
try {
var temp = new Conference({
acronym: conferences.acronym,
name: conferences.fullname,
area: conferences.area,
subarea: conferences.subarea,
location: conferences.location,
url: conferences.url,
description: conferences.description,
papers: await getPapers(final, url(conferences.acronym),conferences.acronym),
});
console.log(temp.papers.length);
} catch (error) {
console.log(error);
}
return temp;
};
describe("Saving records", function () {
it("Saved records to the database", async function (done) {
var conferences = [];
try {
for (var i = 0; i <= 1; i++) {
conferences[i] = await createNewConf(json_conferences[i]);
conferences[i].save().then(function () {
assert(conferences[i].isNew === True);
done();
});
}
mongoose.connection.close();
} catch (error) {
console.log(error);
}
});
});
Below you can see the length of my final array after passing the if to stop fetching more. and the second number is what I receive in the initial call
Maybe anyone has an idea of the undefined behavior that occurs during return.
Your help is much appreciated.