1

I'm using the below code to read a TEXT file in a remote server using python FTLIB class(I only have FTP access to the server).

The below code is working fine in my local and in server it is throwing permission denied exception as i was downloading the TETX file to the server and process it in there.

So i want to make the script modified and instead of downloading the text file to the local,Is it possible only to read the contents of the text file and port that read data to the database(another remote server).

Thanks in advance!

Code:

    from psycopg2 import connect, sql 
    from time import perf_counter
    import psycopg2
    import time
    import os
    from ftplib import FTP
    from utils.config import Configuration as Config
    from utils.postgres_helper import get_connection
    from utils.utils import get_global_config

    start_time = time.perf_counter()

    # Postgresql connection
    try:
       cnx_psql = get_connection(get_global_config(), 'pg_server_1')
       print ("Server Connected")
    except psycopg2.Error as e:
       print('PSQL: Unable to connect!\n{0}').format(e)
       sys.exit(1)

    # Cursors initializations
    cur_psql = cnx_psql.cursor()

    def getFile(ftp, filename):
        try:
           local_filename = os.path.join(r"/home/tl/bi/csv_imports", filename)
            lf = open(local_filename, "wb")
            ftp.retrbinary("RETR " + filename ,lf.write)
            print("file copied")
        except (Exception, psycopg2.Error) as error:
            print("Error occured while copyting file from source to destination ")
            print ("Error: {}".format(error))
            quit()
    try:

        filePath='''/home/tl/bi/csv_imports/log'''
        #filePath='''/cmd/log/stk/log.txt''' 

table='staging.stock_dump'

        SQL="""DROP TABLE IF EXISTS """+  table + """;CREATE TABLE """+ table + """
        (created_date TEXT, product_sku TEXT, previous_stock TEXT, current_stock TEXT );"""


        cur_psql.execute(SQL)
        cnx_psql.commit()
        print("Table "+table + " Created at DWH")

        try:
         ftp = FTP('server.com','user','password')
         print("FTP connection succesful")
         ftp.cwd('/stockitem/')
        except (Exception, psycopg2.Error) as error:
         print ("Error: {}".format(error))
        getFile(ftp,'log')

        read_file = open(filePath, "r")

        my_file_data = read_file.readlines()

        for line in my_file_data:
            if 'Stock:' in line:
                fields=line.split(" ")
                date_part1=fields[0]
                date_part2=fields[1][:-1]
                sku=fields[3]
                prev_stock=fields[5]
                current_stock=fields[7]
                if prev_stock.strip()==current_stock.strip():
                    continue
                else:
                   cur_psql.execute("insert into " + table+"(created_date, product_sku, previous_stock , current_stock)" + " select CAST('" + date_part1+ " "+ date_part2 + "' AS TEXT)" +", CAST('"+sku+"' AS TEXT),CAST('" + prev_stock +"' AS TEXT),CAST('" +current_stock  + "' AS TEXT);")
        cnx_psql.commit()       
        cur_psql.close()
        cnx_psql.close()
        print("Data loaded to server from text file")
        print("Data porting took %s seconds to finish---" % (time.perf_counter() - start_time))          
    except (Exception, psycopg2.Error) as error:
            print ("Error: {}".format(error))
            print("Error adding  information.")
            quit()
    finally:
        ftp.close()
Linu
  • 589
  • 1
  • 10
  • 23

0 Answers0