how to get iframe content with chrome-remote-interface?

Question

I am building a crawler But I need to crawl the iframe contents chrome-remote-interface is not dumping the iframe content is there any way to do it.

Code

     CDP.New({'url':url},(err,target) => {
                if(!err){
                    CDP({target},(client) => {
                        const {Network, Page, Runtime} = client;
                        Network.setUserAgentOverride({'userAgent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36'});
                        Network.enable();
                        Page.enable();
                        Runtime.enable();
                        Page.navigate({url});
                            Page.loadEventFired(() => {
                                Runtime.evaluate({
//I have no idea what to do ..
                                    expression:'document.documentElement',
                                    returnValue:true})
                                    .then(({result}) => {
                                        cb(null,{html:result.value})
                                        CDP.Close({id:target.id})
                                    })
                            })


                    })

score 2 · Accepted Answer · answered Aug 10 '17 at 16:26

2

I think this is forbidden, what you can do instead is to use DOM.getDocument to obtain the whole tree, you cannot convert it directly to an HTML string though:

const {root} = await DOM.getDocument({depth: -1, pierce: true});

answered Aug 10 '17 at 16:26

cYrus

2,976
6
29
47

is there any other libraries could do this .? – Nane Aug 11 '17 at 06:15
1

Not that I'm aware of, sorry. – cYrus Aug 11 '17 at 11:05

score 0 · Answer 2 · edited Jun 20 '20 at 09:12

Use Chrome Launcher

https://www.npmjs.com/package/chrome-launcher

This is actually pretty easy to do if you use the chromeLauncher and pass in the '--disable-web-security' flag. Here is an example of how to set that up.

const chromeLauncher = require('chrome-launcher');
const CDP = require('chrome-remote-interface');


let launchChrome = () => {
  console.log('launchChrome..');
  return chromeLauncher.launch({
    chromeFlags: [
      '--disable-web-security', // Query within iframes
    ],
    logLevel: 'error'
  }).catch(function(e) {
    console.log('Error launching chrome: ' + e);
  });
}

let initChrome = async () => {
  console.log('initChrome..');
  const chrome = await launchChrome();
  const protocol = await CDP({port: chrome.port});

  const {Page, Runtime, Network} = protocol;
  const userAgent = 'Mozilla/5.0 (X11; Linux x86_64)AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36';
  await Promise.all([Page.enable(), Runtime.enable(), Network.setUserAgentOverride({userAgent})]);

  return {chrome: chrome, protocol: protocol, Page: Page, Runtime: Runtime}
}

let run = async () => {

  let {chrome, protocol, Page, Runtime} = await initChrome();

  try {

    await Page.navigate({url: 'https://www.example.com/'});
    await Page.loadEventFired();

    //////////////////////////////////
    // YOU CAN NOW QUERY IN IFRAMES //
    console.log(await Runtime.evaluate({expression: `document.querySelector('iframe')`, returnByValue: true}));
    //////////////////////////////////
    
    console.log('..Finished');
  } catch (err) {
    console.log(err);
  }

  protocol.close();
  chrome.kill();
}

how to get iframe content with chrome-remote-interface?

2 Answers2

Use Chrome Launcher