im trying to scroll more than once in same instance of webdriver using scrapy selenium .
as sending another SeleniumRequest will refresh the page.
here is my code :
class QuotesSpider(scrapy.Spider):
name = 'scss'
def start_requests(self):
url = 'https://quotes.toscrape.com/scroll'
yield SeleniumRequest(url=url, callback=self.parse, wait_time=5,
script="window.scrollTo(0, document.body.scrollHeight);")
def parse(self, response):
quote_item = QuoteItem()
yield SeleniumRequest(url=url, callback=self.parse, wait_time=5,
script="window.scrollTo(0, document.body.scrollHeight);")
also settings :
SELENIUM_DRIVER_NAME = 'chrome'
SELENIUM_DRIVER_EXECUTABLE_PATH = "/../chromedriver.exe"
SELENIUM_DRIVER_ARGUMENTS = ['--window-size=1920,1080']
DOWNLOADER_MIDDLEWARES = {
'scrapy_selenium.SeleniumMiddleware': 800
}