I've multiple .npy
files in a directory which contain 2D
array of the same shape (row=500,column=55). Due to some constraints, every time I can load 2 files, stack them vertically and slice
to make a new array of shape (432,55). I want to slice files as follows in the comment, and I want to do it sequentially in a loop, so I tried with the following script (but could not fix the issue), but it does not give the required result.
Can anybody tell me why this script does not save even no. of files save only odd numbered files? Thanks in advance.
import numpy as np
import os, glob, re
import time
path = '/**array_*.npy' # 100 files
filenames = [os.path.basename(x) for x in glob.glob(path)]
filelist = sorted(filenames, key=lambda x:float(re.findall("(\d+)",x)[0]))
no_files = len(filelist)
t0 = time.time()
daily_data = 432
one_file_data = 500
k = 0
m = 1
for n in range(no_files):
f1 = filelist[n]
arr1 = np.load(f1)
print('First file Name: {} & Shape of Array/File_1:: {}'.format(f1, arr1.shape))
f2 = filelist[n + 1]
arr2 = np.load(f2)
print('Second file Name: {} & Shape of Array/File_2:: {}'.format(f2, arr2.shape))
# Stacking the two arrays vertically
my_arr = np.vstack((arr1, arr2))
del arr1; del arr2
print("Shape of vertically stacked array:\n ", my_arr.shape)
subtract_file = one_file_data * n
nxt_day_dat = my_arr[(daily_data * k - subtract_file):(daily_data * (k + 1) - subtract_file),
:] # for the 0-th file
# Numpy File saving
np.save('test_{}.npy'.format(m), nxt_day_dat)
print('Time spent= {} seconds'.format(time.time() - t0))
k = k + 1
m = m + 1
if (daily_data * k + daily_data) < one_file_data * (k + 1):
k = k + 1
t1 = time.time()
nxt_day_dat = my_arr[(daily_data * k - subtract_file):(daily_data * (k + 1) - subtract_file),:]
del my_arr
m = m + 1
# Numpy File saving
np.save('test_{}.npy'.format(m), nxt_day_dat)
print('Time spent= {} seconds'.format(time.time() - t1))
del my_arr