I have two Redshift Tables each containing a column which stores email_addresses. I am using the FuzzyWuzzy library for String Matching between these two columns. I have read about FuzzyWuzzy here and the PySpark logic here. This is my FuzzyWuzzy logic in Pyspark script :
schema = StructType([
StructField("max_name", StringType(), False),
StructField("max_score", IntegerType(), False)
])
second_table_email = second_redshift_table_df.select("email_address")
second_table_email_list = list(second_table_email)
def get_best_email_match(first_col):
choices = second_table_email_list
return process.extractOne(first_col, choices, scorer=fuzz.token_sort_ratio)
get_best_email_udf = udf(get_best_email, schema)
sdf_1 = first_redshift_table_df.withColumn('new_col', get_best_email_udf(first_redshift_table_df.email_address))
Error:
Traceback (most recent call last):
File "/usr/lib/spark/python/pyspark/serializers.py", line 437, in dumps
return cloudpickle.dumps(obj, pickle_protocol)
File "/usr/lib/spark/python/pyspark/cloudpickle/cloudpickle_fast.py", line 101, in dumps
cp.dump(obj)
File "/usr/lib/spark/python/pyspark/cloudpickle/cloudpickle_fast.py", line 540, in dump
return Pickler.dump(self, obj)
File "/usr/lib64/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib/spark/python/pyspark/cloudpickle/cloudpickle_fast.py", line 722, in save_function
*self._dynamic_function_reduce(obj), obj=obj
File "/usr/lib/spark/python/pyspark/cloudpickle/cloudpickle_fast.py", line 664, in _save_reduce_pickle5
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 819, in save_list
self._batch_appends(obj)
File "/usr/lib64/python3.7/pickle.py", line 846, in _batch_appends
save(tmp[0])
File "/usr/lib64/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 890, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 524, in save
rv = reduce(self.proto)
TypeError: can't pickle _thread.RLock objects
Traceback (most recent call last):
File "/usr/lib/spark/python/pyspark/serializers.py", line 437, in dumps
return cloudpickle.dumps(obj, pickle_protocol)
File "/usr/lib/spark/python/pyspark/cloudpickle/cloudpickle_fast.py", line 101, in dumps
cp.dump(obj)
File "/usr/lib/spark/python/pyspark/cloudpickle/cloudpickle_fast.py", line 540, in dump
return Pickler.dump(self, obj)
File "/usr/lib64/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib/spark/python/pyspark/cloudpickle/cloudpickle_fast.py", line 722, in save_function
*self._dynamic_function_reduce(obj), obj=obj
File "/usr/lib/spark/python/pyspark/cloudpickle/cloudpickle_fast.py", line 664, in _save_reduce_pickle5
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 819, in save_list
self._batch_appends(obj)
File "/usr/lib64/python3.7/pickle.py", line 846, in _batch_appends
save(tmp[0])
File "/usr/lib64/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 890, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/usr/lib64/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/usr/lib64/python3.7/pickle.py", line 524, in save
rv = reduce(self.proto)
TypeError: can't pickle _thread.RLock objects
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/spark/python/pyspark/sql/udf.py", line 199, in wrapper
return self(*args)
File "/usr/lib/spark/python/pyspark/sql/udf.py", line 177, in __call__
judf = self._judf
File "/usr/lib/spark/python/pyspark/sql/udf.py", line 161, in _judf
self._judf_placeholder = self._create_judf()
File "/usr/lib/spark/python/pyspark/sql/udf.py", line 170, in _create_judf
wrapped_func = _wrap_function(sc, self.func, self.returnType)
File "/usr/lib/spark/python/pyspark/sql/udf.py", line 34, in _wrap_function
pickled_command, broadcast_vars, env, includes = _prepare_for_python_RDD(sc, command)
File "/usr/lib/spark/python/pyspark/rdd.py", line 2814, in _prepare_for_python_RDD
pickled_command = ser.dumps(command)
File "/usr/lib/spark/python/pyspark/serializers.py", line 447, in dumps
raise pickle.PicklingError(msg)
_pickle.PicklingError: Could not serialize object: TypeError: can't pickle _thread.RLock objects
What am I doing wrong here? Please help.
