I am trying to use "pyvirtualdisplay" to scrape a dynamic Javascript website while using a headless Chrome. I am on Windows and installed "pyvirtualdisplay" via pip
My code to run the headless chrome is as follows:
from pyvirtualdisplay import Display
from selenium import webdriver
display = Display(visible=0, size=(1024, 768))
display.start()
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
driver = webdriver.Chrome(executable_path="C:/Users/chromedriver.exe", chrome_options=chrome_options)
But I get the below error:
2022-08-22 14:28:41 [scrapy.core.scraper] ERROR: Spider error processing <GET https://eservices.mas.gov.sg/statistics/fdanet/AverageDailyTurnoverVolume.aspx> (referer: http://www.google.com)
Traceback (most recent call last):
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\utils\defer.py", line 120, in iter_errback
yield next(it)
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\utils\python.py", line 353, in __next__
return next(self.data)
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\utils\python.py", line 353, in __next__
return next(self.data)
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\spidermiddlewares\offsite.py", line 29, in process_spider_output
for x in result:
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\spidermiddlewares\referer.py", line 340, in <genexpr>
return (_set_referer(r) for r in result or ())
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\spidermiddlewares\urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\spidermiddlewares\depth.py", line 58, in <genexpr>
return (r for r in result or () if _filter(r))
File "C:\Users\pearl\anaconda3\lib\site-packages\scrapy\core\spidermw.py", line 62, in _evaluate_iterable
for r in iterable:
File "C:\Users\pearl\mas_bond\mas_bond\spiders\turnover.py", line 133, in parse
display = Display(visible=0, size=(1024, 768))
File "C:\Users\pearl\anaconda3\lib\site-packages\pyvirtualdisplay\display.py", line 54, in __init__
self._obj = cls(
File "C:\Users\pearl\anaconda3\lib\site-packages\pyvirtualdisplay\xvfb.py", line 44, in __init__
AbstractDisplay.__init__(
File "C:\Users\pearl\anaconda3\lib\site-packages\pyvirtualdisplay\abstractdisplay.py", line 85, in __init__
helptext = get_helptext(program)
File "C:\Users\pearl\anaconda3\lib\site-packages\pyvirtualdisplay\util.py", line 13, in get_helptext
p = subprocess.Popen(
File "C:\Users\pearl\anaconda3\lib\subprocess.py", line 951, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "C:\Users\pearl\anaconda3\lib\subprocess.py", line 1420, in _execute_child
hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
FileNotFoundError: [WinError 2] The system cannot find the file specified
2022-08-22 14:28:41 [scrapy.core.engine] INFO: Closing spider (finished)
2022-08-22 14:28:41 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
