0

thank you for ur attention, so i write a mini project that scrape news site and store main texts from them. i tried many solutions to add json in my project without have consol.log but always after scraping its show only one main text. so i show my code to you so u could help me how to have json with all three news.

const { Cluster } = require('puppeteer-cluster');
const fs = require('fs');


const launchOptions = {
    headless: false,
    args: [
        '--disable-gpu',
        '--disable-dev-shm-usage',
        '--disable-web-security',
        '--disable-xss-auditor',
        '--disable-accelerated-2d-canvas',
        '--ignore-certifcate-errors',
        '--ignore-certifcate-errors-spki-list',
        '--no-zygote',
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-webgl',
    ],
    ignoreHTTPSErrors: true,
    waitUntil: 'networkidle2',
};
(async() => {
    // Create a cluster with 2 workers
    const cluster = await Cluster.launch({
        monitor: true,
        concurrency: Cluster.CONCURRENCY_PAGE,
        maxConcurrency: 2,
        puppeteerOptions: launchOptions,
    });

    // Define a task (in this case: screenshot of page)
    await cluster.task(async({ page, data: url }) => {
        await page.setRequestInterception(true);
        page.on('request', (request) => {
            if (['stylesheet', 'font', 'image', 'styles','other', 'media'].indexOf(request.resourceType()) !== -1) {
                request.abort();
            } else {
                request.continue();
            }
        });
        await page.goto(url);
        const scrapedData = await page.$eval('div[class="entry-content clearfix"]', el => el.innerText)
        fs.writeFileSync('test.json', JSON.stringify(scrapedData, null, 2))
    });
    // Add some pages to queue
    cluster.queue('https://www.ettelaat.com/?p=526642');
    cluster.queue('https://www.ettelaat.com/?p=526640');
    cluster.queue('https://www.ettelaat.com/?p=526641');

    // Shutdown after everything is done
    await cluster.idle();
    await cluster.close();
})();

1 Answers1

0

for gather all outputs i had to put my fs in the bottom of cluster.close

kanopyDB = []
.
.
.
        kanopyDB = kanopyDB.concat(name);
.
.
.
    await cluster.idle();
    await cluster.close();
    fs.writeFileSync('output.json', kanopyDB, 'utf8');