I have a Node JS App Engine app that uses Request Promises Native to fetch external URLs. App Engine scripts launched from Cloud Tasks are supposed to have up to a 24 hour deadline, but all my tasks are being killed after 60 seconds. I know other frameworks have to use a built-in URLFetch library, and this gives them an automatic 60 second task deadline, but the docs don't state anything about Node.JS bases app engine tasks. This task fetches json from an external URL, then processes the results and inserts them into firebase, which is typically what is happening when the task is killed with an HTTP 504 Deadline Exceeded error
app.post('/tasks/import', async (req, res) => {
const paramsObj = JSON.parse(Buffer.from(req.body, 'base64').toString('utf-8'));
const page = paramsObj.page * 1;
const storeID = paramsObj.storeid;
const pageSize = 50;
const config: StoreConfig = await getConfigById(storeID);
const options = {
uri: EXTERNAL_URL,
json: true,
resolveWithFullResponse: false,
qs: {
limit: pageSize,
page: page,
},
};
try {
const results = await rp.get(options);
if (results.products.length === 0) {
return res.status(200).end();
}
const prodLooper = idx => {
const product = results.products[idx];
product.store_id = storeID;
product.body_html = cleanProductBody(product.body_html);
getUnusedUPC(storeID)
.then(upcID => {
product.upc = upcID;
fsdb
.collection('products')
.add(product)
.then(() => {
idx++;
if (idx < results.products.length) {
prodLooper(idx);
} else {
return res.send('OK').end();
}
});
})
.catch(err => {
console.log(err.code, ':', page, ':', idx);
if (err.code === 10) {
setTimeout(() => {
prodLooper(idx);
}, 10);
}
});
};
prodLooper(0);
} catch (error) {
console.log('caught error:');
console.log(error.statusCode);
console.log(error.message);
return res.status(500).end();
}
});
