Return list of Objects with Node js

Question

I recently started development of a Node js application and it uses Selenium in a controller to fetch list of items from a web page and I want to return the fetched list of items as a JSON response.

exports.read_all_products = function (req, res) {
    var driver = new webdriver.Builder().forBrowser('phantomjs').build();
    driver.get('https://www.test.com/products?PC=' +req.params.category);    
    driver.wait(until.elementLocated(By.className('product-slide-all')), 20000, 'Could not locate the element within the time specified');

    driver.findElements(By.className("product-slide-all")).then(function (elements) {
        var arr = [];
        elements.forEach(function (element) {
        element.getAttribute("innerHTML").then(function (html) {
            const dom = new JSDOM(html);
            var obj = new Object();
            obj.product_name = dom.window.document.querySelector(".product-name").textContent;
            obj.product_code = dom.window.document.querySelector(".product-code").textContent;
            obj.price = dom.window.document.querySelector(".product-price").textContent;
            arr.push(obj);
        });
    });
    res.json(arr);
    });
}

Issue is I am always getting an empty JSON response even though items were added to the array. I want to know the proper way of handling this scenario.

Thanks.

Can you try moving your `res.json(arr);` one level up? (before the `});`) ? — David R, May 25 '17 at 12:33
No then I am getting an error saying 'Can't set headers after they are sent'. — Lakmal, May 25 '17 at 12:35
Ohh. In that case, can you try rephrasing your `res.json(arr);` as `JSON.parse(JSON.stringify(arr))` ? — David R, May 25 '17 at 12:37
I am still getting the same error, I think it's because then **res.json(JSON.parse(JSON.stringify(arr)))** is inside the **forEach** loop. — Lakmal, May 25 '17 at 12:42
Okay, Can you please try moving your `var arr = [];` declaration below yor `var driver` variable declaration and try? — David R, May 25 '17 at 12:48
@David Still not working, I think this is an issue related to the promises. — Lakmal, May 25 '17 at 12:54

score 1 · Answer 1 · answered May 25 '17 at 15:07

It looks like the issue is because Selenium is running an async process, thus the response immediately returns because there is nothing blocking it.

findElements returns a Promise which you need to return the response from.

Take a look at How do I return the response from an asynchronous call?

Lakmal · Accepted Answer · 2017-05-26T07:28:39.683

Finally I was able to get it work with the help of webdriver.promise.map.

Moved web driver HTML extraction to separate function.

var findItems = function (category) {
var driver = new webdriver.Builder().forBrowser('phantomjs').build();
var map = webdriver.promise.map;
driver.get('https://www.test.com?PC=' + category);
driver.wait(until.elementLocated(By.className('product-slide-all')), 30000, 'Could not locate the element within the time specified');
    var elems = driver.findElements(By.className("product-slide-all"));
        return map(elems, elem => elem.getAttribute("innerHTML")).then(titles => {
        return titles;
    });
}

then call it from response handling function like bellow,

exports.read_all_products = function (req, res) {
findItems(req.params.category).then(function (html) {
    var value;
    var arr = [];
    Object.keys(html).forEach(function (key) {
        value = html[key];
        const dom = new JSDOM(value);
        var obj = new Object();
        obj.product_name = dom.window.document.querySelector(".product-name").textContent;
        obj.product_code = dom.window.document.querySelector(".product-code").textContent;
        obj.price = dom.window.document.querySelector(".product-price").textContent;
        arr.push(obj);
    });
    res.json(arr);
})
};

it's described in this stack overflow answers.

Return list of Objects with Node js

2 Answers2