0

Im trying to use crawlera alongside splash local instance, this is my lua script

function main(splash)
function use_crawlera(splash)

    local user = splash.args.crawlera_user

    local host = 'proxy.crawlera.com'
    local port = 8010
    local session_header = 'X-Crawlera-Session'
    local session_id = 'create'

    splash:on_request(function(request)
        request:set_header('X-Crawlera-Cookies', 'disable')
        request:set_header(session_header, session_id)
        request:set_proxy { host, port, username = user, password = '' }
    end)

    splash:on_response_headers(function(response)
        if type(response.headers[session_header]) ~= nil then
            session_id = response.headers[session_header]
        end
    end)
end

function main(splash)
    use_crawlera(splash)
    splash:go(splash.args.url)
    splash:wait(30)
    return splash:html()
end

end

and this is my start_request

yield SplashRequest(index_url,
                            self.parse_kawanlama_index,
                            endpoint='execute',
                            args={
                                'lua_source': lua_script,
                                'wait' : 5,
                                'html' : 1,
                                'url': index_url,
                                'timeout': 10,
                                'crawlera_user':self.crawlera_apikey
                            },
                            # tell Splash to cache the lua script, to avoid sending it for every request
                            cache_args=['lua_source'],
                            )

but it doesnt seems to work because response.body that i got in self.parse(response) is contains no html.

Farhan Muhammad
  • 13
  • 5
  • Did you test your Lua script by itself in the local Splash browser (http://localhost:8050/)? – malberts Mar 17 '19 at 15:08
  • @malberts yes i did, Im trying to scrape this page https://www.kawanlama.com/brands/krisbow, and local splash browser response is Splash Response: "" – Farhan Muhammad Mar 17 '19 at 15:15

0 Answers0