I am trying to execute below code to get the list of files from 2 nodes parallel and process them:
#!/usr/bin/python
import paramiko
import cffi
from multiprocessing import Process, Pool
key1 = paramiko.RSAKey.from_private_key_file("/Users//Downloads/test1.pem")
key2 = paramiko.RSAKey.from_private_key_file("/Users//Downloads/test2.pem")
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Connect/ssh to an instance
file_list = []
host1 = '3.93.XX.X'
host2 = '3.93.XX.X'
filepath="/home/ubuntu/test/"
def readFileNode(host,key):
try:
# Here 'ubuntu' is user name and 'instance_ip' is public IP of EC2
client.connect(hostname=host, username="ubuntu", pkey=key)
sftp = client.open_sftp()
file_list=sftp.listdir_attr(filepath)
client.close()
# break
except Exception:
print("Exception")
return file_list
def compareFilesOnNode(listoffiles):
fileNode1 = []
fileNode1 = listoffiles[0]
fileNode2 = []
fileNode2 = listoffiles[1]
client.connect(hostname=host1, username="ubuntu", pkey=key)
mysftp1 = client.open_sftp()
client.connect(hostname=host2, username="ubuntu", pkey=key)
mysftp2 = client.open_sftp()
print(fileNode1[0].filename)
print(fileNode2[0].filename)
FileCountNode1 = len(fileNode1)
FileCountNode2 = len(fileNode2)
print("Files in Node1:", FileCountNode1, "Total Files in Node2:", FileCountNode2)
# loop should be running number of times that is equal to node having less number of files
counter=min(len(fileNode1),len(fileNode2))
for i in range(0,counter ):
filePath1 = filepath + fileNode1[i].filename
filePath2 = filepath + fileNode2[i].filename
print("Comparing:", filePath1, "From host:", host1, "to", filePath2, "In Host:", host2)
s1 = mysftp1.open(filePath1)
s2 = mysftp2.open(filePath2)
#file metadata validation
print(s1.stat)
print(s2.stat)
if (s1.stat() == s2.stat()):
print("File Stats are Equal")
else:
print("Stats are not Equal")
# check integrity of file
print("Checksum file from Node 1 is", s1.__hash__())
print("Cheksum file from Node2 is ", s2.__hash__())
# match metadata of files from both nodes
if (s1.readlines() == s2.readlines()):
print("File content are Equal")
yield "pass"
else:
print("File Content are Not Equal")
yield "fail"
s1.close()
s2.close()
if __name__ == '__main__':
p = Pool(2)
#call readFileNode in Parallel
#filenames = p.map(readFileNode, [host1, host2])
filenames=p.starmap(readFileNode, [(host1, host2), (key1, key2)])
#Both nodes files will be stored in filenames as list
print(filenames)
#compare files across node and node2 and get the result as pass or fail
for res in compareFilesOnNode(filenames):
print(res)
But, I am getting the below error:
TypeError: cannot pickle '_cffi_backend.FFI' object
I also tried uninstalling several modules and installing it . Some examples are like uninstalling paramiko, cryptography module and reinstalling it, but it doesn't work.
Please help!