python html scraping result of JS function

Question

I'm trying to scrape a web page with JS script executed on it.

I can get the HTML with the none executed JS using from requests import get

But I can't have the JS function result like when I inspect the webpage with mozilla

here the function I would like to get the result from:

function showFlight(idPilote, idFlight, idActivite)

on this page

any idea how to do that?

I've tried this

def kmlScrape(target):
    curl = pycurl.Curl()
    curl.setopt(pycurl.CAINFO, certifi.where())
    curl.setopt(pycurl.SSL_VERIFYPEER, 0) 
    curl.setopt(pycurl.URL, target)
    curl.setopt(pycurl.WRITEFUNCTION, e.write)
    curl.perform()
    return e

or

response = get(target)
print(response.text)
html_soup = BeautifulSoup(response.text, 'html.parser')

what I'm trying to do is to write this curl command in pycurl format:

curl "https://www.syride.com/scripts/ajx_vols.php?p=/en/flights/&idPays=0&pseudo=0&typePratique=0&page=01&idSpot=0&recherche=&order=&tri=&l=en" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0" -H "Accept: */*" -H "Accept-Language: en-US,en;q=0.5" --compressed -H "X-Requested-With: XMLHttpRequest" -H "DNT: 1" -H "Connection: keep-alive" -H "Referer: https://www.syride.com/en/flights/&idPays=0&pseudo=0&typePratique=0&page=01&idSpot=0&recherche=&order=&tri=" -H "Cookie: instruments2=5ai6jgg9p6bpkh3m7slf0b2vq7; gb__ssyride=aufnlupslr259o49nc98qm1t55" -H "Cache-Control: max-age=0"

any help very welcome.

Thanks, Mat

score 0 · Answer 1 · edited Feb 02 '21 at 16:18

I found the answer:

def kmlScrape(target):
    curl = pycurl.Curl()
    curl.setopt(pycurl.CAINFO, certifi.where())
    curl.setopt(pycurl.SSL_VERIFYPEER, 0) 
    curl.setopt(pycurl.URL, target)
    opts={pycurl.USERAGENT:"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0", pycurl.HTTPHEADER:["Accept: */*", "Accept-Language: en-US,en;q=0.5", "Connection: keep-alive", "X-Requested-With: XMLHttpRequest", "DNT: 1", "Referer: https://www.syride.com/en/flights/&idPays=0&pseudo=0&typePratique=0&page=01&idSpot=0&recherche=&order=&tri=", "Cookie: instruments2=5ai6jgg9p6bpkh3m7slf0b2vq7; gb__ssyride=aufnlupslr259o49nc98qm1t55", "Cache-Control: max-age=0"]}
    for (key,value) in opts.items():
            curl.setopt(key,value)
    curl.setopt(pycurl.WRITEFUNCTION, e.write)
    curl.perform()
    return e

score 0 · Accepted Answer · answered Apr 07 '20 at 15:02

That's a direct call to the Back-End API.

import requests
import pandas as pd


params = {
    'l': 'en'
}


def main(url, params):
    with requests.Session() as req:
        allin = []
        for page in range(1, 41):
            print(F"Extracting Page# {page}")
            params['page'] = page
            r = req.get(url, params=params)
            df = pd.read_html(r.content, encoding="UTF-8",
                              header=0)[0][:-1]
            allin.append(df)
        new = pd.concat(allin)
        new.to_csv("data.csv", index=False)
        print("Done")


main("https://www.syride.com/scripts/ajx_vols.php", params)

python html scraping result of JS function

2 Answers2