I try to use manager and queue to share the object of selenium in two subprocess, but it shows me the error "AttributeError: Can't pickle local object '_createenviron..encodekey'"
from selenium.webdriver import Chrome
from multiprocessing import Queue, Manager, Process
import time
def find(q):
driver = Chrome()
driver.get('https://querycourse.ntust.edu.tw/querycourse/api/courses')
q.put(driver)
def refresh(q):
driver = q.get()
while True:
time.sleep(9)
driver.refresh()
if __name__=='__main__':
with Manager() as manager:
q = manager.Queue()
p1 = Process(target=find, args=(q,))
p2 = Process(target=refresh, args=(q,))
p1.start()
time.sleep(3)
p2.start()
p1.join()
p2.join()
Process Process-2:
Traceback (most recent call last):
File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\process.py", line 314, in _bootstrap
self.run()
File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "D:\python_training\web crawler\kk_manager_queue.py", line 8, in find
q.put(driver)
File "<string>", line 2, in put
File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\managers.py", line 817, in _callmethod
conn.send((self._id, methodname, args, kwds))
File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\connection.py", line 211, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
AttributeError: Can't pickle local object '_createenviron.<locals>.encodekey'
Does any one know what's wrong with my code? Any feetbeck are appreciated.
I simplified the code, I originally expected to use the code grabbing the course I want. One subprocess is to refresh the course selecting system to prevent Connect Timeouts, the other is to detect if someone drops the course and then click the “select” button, so the two subprocesses need to share the object “driver”.
This is my original code:
import urllib.request as req
import json
from selenium import webdriver
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.support.wait import WebDriverWait
import time
from multiprocessing import Manager, Queue, Process
def find(url, requestData, chooseStudent, q):
while True:
request=req.Request(url, headers={
"content-type":"application/json; charset=utf-8"
}, data=json.dumps(requestData).encode("utf-8"))
with req.urlopen(request) as response:
result=response.read().decode("utf-8")
result=json.loads(result)
if int(result[0]["ChooseStudent"])<chooseStudent:
driver = q.get()
ADDgo = driver.find_element(By.ID, "SingleAdd")
ADDgo.click()
break
def refresh(q):
account='XXXXXXXX'
password='XXXXXXXX'
classid='GE3710302'
driver = Chrome()
driver.get("https://stuinfosys.ntust.edu.tw/NTUSTSSOServ/SSO/Login/CourseSelection")
UserName = WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.NAME,"UserName"))
UserName.send_keys(account)
Password = driver.find_element(By.NAME, "Password")
Password.send_keys(password)
btnLogIn = driver.find_element(By.NAME, "btnLogIn")
btnLogIn.click()
q.put(driver)
while True:
time.sleep(10)
driver.refresh()
if __name__=='__main__':
url="https://querycourse.ntust.edu.tw/querycourse/api/courses"
requestData={"Semester":"1112","CourseNo":"GE3710302","CourseName":"","CourseTeacher":"","Dimension":"","CourseNotes":"","ForeignLanguage":0,"OnlyGeneral":0,"OnleyNTUST":0,"OnlyMaster":0,"OnlyUnderGraduate":0,"OnlyNode":0,"Language":"zh"}
chooseStudent=51
with Manager() as manager:
q = manager.Queue()
p1 = Process(target=find, args=(url, requestData, chooseStudent, q,))
p2 = Process(target=refresh, args=(q,))
p1.start()
p2.start()
p1.join()
p2.join()