Still related to following Question... Parallel downloads with Multiprocessing and PySftp
I'd like to know how to print the successfully downloads? My intention behind that is actually append a record in a database table in order to create a log of downloaded files with the filename, date and time.
Any ideas? I've searched for some examples and made some tests, but it seems that my download module can't return anything or I'm not using the right code to read the results and print it.
DOWNLOAD function
import pysftp
import os
def fdownload(vfileaux):
vtmpspl = vfileaux.split(',')
vfile = vtmpspl[0]
vhost = vtmpspl[1]
vlogin = vtmpspl[2]
vpwd = vtmpspl[3]
vftppath = vtmpspl[4]
vlocalpath = vtmpspl[5]
os.chdir(vlocalpath)
os.getcwd()
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
vfilecheck = vlocalpath + '/' + vfile
if not os.path.isfile(vfilecheck):
vftpaux = pysftp.Connection(host=vhost, username=vlogin, password=vpwd, cnopts=cnopts)
vftpaux.cwd(vftppath)
vftpaux.get(vfile, preserve_mtime=True)
vftpaux.close()
return vnename + "_" + vdatetime
else:
pass
MAIN function
from datetime import *
from ffilelist import *
from ffilefilter import *
from developing.fdownload import *
import pymysql.cursors
from concurrent.futures import ThreadPoolExecutor, wait, as_completed
def main():
print(datetime.datetime.now(), 'Loading variables...')
vhostlist = {}
vloginlist = {}
vpwdlist = {}
vftppathlist = {}
vlocalpathlist = {}
vhostaux = '10.11.12.13'
vhostlist[vhostaux] = vhostaux
vloginlist[vhostaux] = 'admin'
vpwdlist[vhostaux] = 'pass1234'
vftppathlist[vhostaux] = '/export/home'
vlocalpathlist[vhostaux] = 'd:/test/'
vfilelist1 = []
global vfilelist2
vfilelist2 = []
for vhosttmp in vhostlist:
print(datetime.datetime.now(), 'Starting to process ' + vhosttmp + "...")
global vhost
global vlogin
global vpwd
global vftppath
global vlocalpath
vhost = vhostlist[vhosttmp]
vlogin = vloginlist[vhosttmp]
vpwd = vpwdlist[vhosttmp]
vftppath = vftppathlist[vhosttmp]
vlocalpath = vlocalpathlist[vhosttmp]
vfilelist1 = ffilelist(vhost, vlogin, vpwd, vftppath)
print(datetime.datetime.now(), 'Vectorizing download file list...')
for vfile in vfilelist1:
vfilelist2.append(vfile + ',' + vhost + ',' + vlogin + ',' + vpwd + ',' + vftppath + ',' + vlocalpath)
vfilelist0 = ffilefilter(vfilelist2)
print(datetime.datetime.now(), 'Starting simultaneous downloads...')
vpool = concurrent.futures.ProcessPoolExecutor(max_workers=8)
vpool.map(fdownload, vfilelist0)
vpool.shutdown()
print(datetime.datetime.now(), 'Downloads finished!')
The INSERT string for the log to be stored in a MARIADB, is something like this. Already tested and working. To be used in MAIN function as soon as I find a solution to get the list of downloaded files.
vconn = pymysql.connect(host='localhost', user='root', password='pass1234', db='test')
vcurs = vconn.cursor()
vsql = "INSERT INTO `logs_download` (`ne`, `datetime`) VALUES (\'" + vnename + "\', \'" + vdatetime + "\')"
vcurs.execute(vsql)
vconn.commit()
