I'm trying to find best parameter in LGBM using GridSearch and here's my approach.
from sklearn.model_selection import GridSearchCV
import lightgbm as lgb
model = lgb.LGBMRegressor(random_state=1, objective='regression')
param_grid = {
'max_depth':[20, 30, 2],
'num_leaves': [700, 1000, 100],
'min_data_in_leaf': range(200, 500),
'n_estimators': [400, 800, 100],
'learning_rate':[0.05],
'feature_fraction':[0.4, 0.5],
'max_bin':[400, 500]}
model_l = GridSearchCV(model, param_grid, cv=5 , n_jobs=-1)
model_l.fit(df_train_robust, y_floor_train,
eval_set=[(df_test_robust, y_floor_test)],
eval_metric=['l1','l2_root','mape'])
After running for like a day I'm getting this error:
/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.
"timeout or by a memory leak.", UserWarning
exception calling callback for <Future at 0x7f2a65498ad0 state=finished raised PicklingError>
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 292, in save
wrapper.write_array(obj, self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 104, in write_array
pickler.file_handle.write(chunk.tostring('C'))
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/queues.py", line 150, in _feed
obj_ = dumps(obj, reducers=reducers)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/reduction.py", line 247, in dumps
dump(obj, buf, reducers=reducers, protocol=protocol)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/reduction.py", line 240, in dump
_LokyPickler(file, reducers=reducers, protocol=protocol).dump(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/cloudpickle/cloudpickle.py", line 482, in dump
return Pickler.dump(self, obj)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 890, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 819, in save_list
self._batch_appends(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 846, in _batch_appends
save(tmp[0])
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 789, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 819, in save_list
self._batch_appends(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 846, in _batch_appends
save(tmp[0])
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 638, in save_reduce
save(args)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 510, in save
rv = reduce(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/_memmapping_reducer.py", line 339, in __call__
for dumped_filename in dump(a, filename):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/_base.py", line 625, in _invoke_callbacks
callback(self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 340, in __call__
self.parallel.dispatch_next()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 769, in dispatch_next
if not self.dispatch_one_batch(self._original_iterator):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 835, in dispatch_one_batch
self._dispatch(tasks)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 754, in _dispatch
job = self._backend.apply_async(batch, callback=cb)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 551, in apply_async
future = self._workers.submit(SafeFunction(func))
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/reusable_executor.py", line 160, in submit
fn, *args, **kwargs)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 1047, in submit
self._ensure_executor_running()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 1021, in _ensure_executor_running
self._adjust_process_count()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 999, in _adjust_process_count
worker_exit_lock = self._context.BoundedSemaphore(1)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/context.py", line 220, in BoundedSemaphore
return BoundedSemaphore(value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 156, in __init__
SemLock.__init__(self, SEMAPHORE, value, value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 70, in __init__
unlink_now)
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/queues.py", line 150, in _feed
obj_ = dumps(obj, reducers=reducers)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/_base.py", line 625, in _invoke_callbacks
callback(self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 340, in __call__
self.parallel.dispatch_next()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 769, in dispatch_next
if not self.dispatch_one_batch(self._original_iterator):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 156, in __init__
SemLock.__init__(self, SEMAPHORE, value, value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 70, in __init__
unlink_now)
OSError: [Errno 28] No space left on device
exception calling callback for <Future at 0x7f2a651b3710 state=finished raised PicklingError>
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 292, in save
wrapper.write_array(obj, self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 104, in write_array
pickler.file_handle.write(chunk.tostring('C'))
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/queues.py", line 150, in _feed
obj_ = dumps(obj, reducers=reducers)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 156, in __init__
SemLock.__init__(self, SEMAPHORE, value, value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 70, in __init__
unlink_now)
OSError: [Errno 28] No space left on device
exception calling callback for <Future at 0x7f2a65158090 state=finished raised PicklingError>
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 292, in save
wrapper.write_array(obj, self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 104, in write_array
pickler.file_handle.write(chunk.tostring('C'))
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/queues.py", line 150, in _feed
obj_ = dumps(obj, reducers=reducers)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/_memmapping_reducer.py", line 339, in __call__
for dumped_filename in dump(a, filename):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/_base.py", line 625, in _invoke_callbacks
callback(self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 340, in __call__
self.parallel.dispatch_next()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 769, in dispatch_next
if not self.dispatch_one_batch(self._original_iterator):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 835, in dispatch_one_batch
self._dispatch(tasks)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 754, in _dispatch
job = self._backend.apply_async(batch, callback=cb)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 551, in apply_async
future = self._workers.submit(SafeFunction(func))
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/reusable_executor.py", line 160, in submit
fn, *args, **kwargs)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 1047, in submit
self._ensure_executor_running()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 1021, in _ensure_executor_running
self._adjust_process_count()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 999, in _adjust_process_count
worker_exit_lock = self._context.BoundedSemaphore(1)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/context.py", line 220, in BoundedSemaphore
return BoundedSemaphore(value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 156, in __init__
SemLock.__init__(self, SEMAPHORE, value, value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 70, in __init__
unlink_now)
OSError: [Errno 28] No space left on device
exception calling callback for <Future at 0x7f2a65158310 state=finished raised PicklingError>
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 156, in __init__
SemLock.__init__(self, SEMAPHORE, value, value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 70, in __init__
unlink_now)
OSError: [Errno 28] No space left on device
exception calling callback for <Future at 0x7f2a651824d0 state=finished raised PicklingError>
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 292, in save
wrapper.write_array(obj, self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 104, in write_array
pickler.file_handle.write(chunk.tostring('C'))
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 789, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 819, in save_list
self._batch_appends(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 846, in _batch_appends
save(tmp[0])
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 638, in save_reduce
save(args)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 510, in save
rv = reduce(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/_memmapping_reducer.py", line 339, in __call__
for dumped_filename in dump(a, filename):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 292, in save
wrapper.write_array(obj, self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 104, in write_array
pickler.file_handle.write(chunk.tostring('C'))
OSError: [Errno 28] No space left on device
"""
The above exception was the direct cause of the following exception:
PicklingError Traceback (most recent call last)
<ipython-input-37-ad2cda3e0f98> in <module>
1 gs_m2_rs_poly.fit(df_train_robust, y_floor_train,
2 eval_set=[(df_test_robust, y_floor_test)],
----> 3 eval_metric=['l1','l2_root','mape'])
~/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py in fit(self, X, y, groups, **fit_params)
708 return results
709
--> 710 self._run_search(evaluate_candidates)
711
712 # For multi-metric evaluation, store the best_index_, best_params_ and
~/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py in _run_search(self, evaluate_candidates)
1149 def _run_search(self, evaluate_candidates):
1150 """Search all candidates in param_grid"""
-> 1151 evaluate_candidates(ParameterGrid(self.param_grid))
1152
1153
~/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py in evaluate_candidates(candidate_params)
687 for parameters, (train, test)
688 in product(candidate_params,
--> 689 cv.split(X, y, groups)))
690
691 if len(out) < 1:
~/anaconda3/lib/python3.7/site-packages/joblib/parallel.py in __call__(self, iterable)
1015
1016 with self._backend.retrieval_context():
-> 1017 self.retrieve()
1018 # Make sure that we get a last message telling us we are done
1019 elapsed_time = time.time() - self._start_time
~/anaconda3/lib/python3.7/site-packages/joblib/parallel.py in retrieve(self)
907 try:
908 if getattr(self._backend, 'supports_timeout', False):
--> 909 self._output.extend(job.get(timeout=self.timeout))
910 else:
911 self._output.extend(job.get())
~/anaconda3/lib/python3.7/site-packages/joblib/_parallel_backends.py in wrap_future_result(future, timeout)
560 AsyncResults.get from multiprocessing."""
561 try:
--> 562 return future.result(timeout=timeout)
563 except LokyTimeoutError:
564 raise TimeoutError()
~/anaconda3/lib/python3.7/concurrent/futures/_base.py in result(self, timeout)
426 raise CancelledError()
427 elif self._state == FINISHED:
--> 428 return self.__get_result()
429
430 self._condition.wait(timeout)
~/anaconda3/lib/python3.7/concurrent/futures/_base.py in __get_result(self)
382 def __get_result(self):
383 if self._exception:
--> 384 raise self._exception
385 else:
386 return self._result
PicklingError: Could not pickle the task to send it to the workers.
I don't know what's going wrong after running for a while, initially I might be running out of storage so I increased the size of the instance but still running into same thing. Is it because I have too many parameter to train. I have around 1.6 million records for training this with 40 variables. Or is there a better way to find hyper parameter?
Instance:
ubuntu 20.04
m5a.8Xlarge
1000GB
