Move files from an Azure file share directory to unix directory and to a backup directory (within Azure file share)

Question

Need to move all the files from a directory on Azure file share to Unix directory. Once it is moved take a backup of these files in a backup directory.

I have written a method which move the file from Azure file share directory to unix directory based on file names. But i need to change it so that it moves all the files and take backup. Source directory address looks like this:- Z:\Business Backup directory is already created which is:- Z:\Business\Backup And there are no subdirectory under Business just the files and name start with Data_Files_yyyymmdd.

In second step, Need to move all the files from the directory to unix directory.

Edit:1- I have edited the code a bit, since I am running it inside a tool. And calling the code as:- maincode(AzureStorageConnectionString);

But I am getting error as:- [ERROR] com.microsoft.azure.storage.StorageException: The specifed resource name contains invalid characters. I tried to fix it but not able to. I tried changing backupFileShareName to different name like below but both is not working. try 1) static String backupFileShareName = "Business/Backup"; try 2) static String backupFileShareName = "Backup";

static String connectionString = "DefaultEndpointsProtocol=https;AccountName=elkdemmastershare;AccountKey=ZdqwMyhGDBVJWy85IapP5CnzavK2cGzVUCqyQIKwhdcWbI0bGE/WNkQsW+CPWWRJN1JITFkYaWm0bGqOIEJnUg==;EndpointSuffix=core.windows.net";
static String fileShareName = "Business";
static String localRootDirPath = "/cogn_shared/TgtFiles/test_data/";
static String backupFileShareName = "Business/Backup";

public static void download(CloudFileDirectory root, CloudFileDirectory backup)throws StorageException, URISyntaxException, FileNotFoundException {
    System.out.println("=>\t" + root.getName());
    ResultSegment < ListFileItem > list = root.listFilesAndDirectoriesSegmented();
    for (ListFileItem item: list.getResults()) {
        URI uri = item.getUri();
        //Need to move all the files from a directory on Azure file share to Unix directory.Once it is moved take a backup of these files in a backup directory.
        //I have written a method which move the file from Azure file share directory to unix directory based on file names.But i need to change it so that it moves all the files and take backup.
        //Need to move all the files from the directory to unix directory.
        String path = uri.getPath();
        String localPath = localRootDirPath + path;
        String itemName = new File(path).getName();
        boolean flag = isDir(root, itemName);
        System.out.println(item.getUri() + "\t" + path + "\t" + itemName + "\t" + flag);
        if (flag) {
            // Create local directory
            new File(localPath).mkdirs();
            CloudFileDirectory next = root.getDirectoryReference(itemName);
            // Create cloud directory for backup
            CloudFileDirectory backupNext = backup.getDirectoryReference(itemName);
            backupNext.createIfNotExists();
            // Recursion
            download(next, backupNext);
        } else {
            // Download file to local
            FileOutputStream fos = new FileOutputStream(localPath);
            CloudFile file = root.getFileReference(itemName);
            file.download(fos);
            // Start Copy to cloud directory for backup without upload again
            CloudFile backupFile = backup.getFileReference(itemName);
            backupFile.startCopy(file);
            System.out.println("Downloaded " + path);
        }
    }
}

public static boolean isDir(CloudFileDirectory root, String itemName)throws URISyntaxException, StorageException {
    CloudFileDirectory dir = root.getDirectoryReference(itemName);
    boolean flag = true;
    try {
        dir.listFilesAndDirectoriesSegmented();
    } catch (StorageException e) {
        flag = false;
    }
    return flag;
}

public static void maincode(String connectionString) {

    try {
        CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
        CloudFileClient fileClient = account.createCloudFileClient();
        CloudFileShare share = fileClient.getShareReference(fileShareName);
        CloudFileDirectory rootDir = share.getRootDirectoryReference();
        CloudFileShare backupShare = fileClient.getShareReference(backupFileShareName);
        backupShare.createIfNotExists();
        CloudFileDirectory backupRootDir = backupShare.getRootDirectoryReference();
        download(rootDir, backupRootDir);
    } catch (Exception e) {
        e.printStackTrace();
        //System.out.println(e.getMessage());
    }
}

Answer 1

It sounds like you want to download all files in an Azure File Share to local directory and backup them to another Azure File Share.

Here is my sample code with Azure Storage SDK v8 for Java (I see you used the same SDK version) for your needs.

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;

import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.ResultSegment;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.file.CloudFile;
import com.microsoft.azure.storage.file.CloudFileClient;
import com.microsoft.azure.storage.file.CloudFileDirectory;
import com.microsoft.azure.storage.file.CloudFileShare;
import com.microsoft.azure.storage.file.ListFileItem;

public class DownloadFilesFromFileShare {

    private static final String connectionString = "DefaultEndpointsProtocol=https;AccountName=<your account name>;AccountKey=<your account key>;EndpointSuffix=core.windows.net;";

    private static final String fileShareName = "<source file share>";
    private static final String localRootDirPath = "<local directory like D:/backup or /home/user/backup>";

    private static final String backupFileShareName = "<backup file share>";

    public static boolean isDir(CloudFileDirectory root, String itemName) throws URISyntaxException, StorageException {
        CloudFileDirectory dir = root.getDirectoryReference(itemName);
        boolean flag = true;
        try {
            dir.listFilesAndDirectoriesSegmented();
        } catch (StorageException e) {
            flag = false;
        }
        return flag;
    }

    public static void download(CloudFileDirectory root, CloudFileDirectory backup) throws StorageException, URISyntaxException, FileNotFoundException {
        System.out.println("=>\t"+root.getName());
        ResultSegment<ListFileItem> list = root.listFilesAndDirectoriesSegmented();
        for (ListFileItem item : list.getResults()) {
            URI uri = item.getUri();
            String path = uri.getPath();
            String localPath = localRootDirPath + path;
            String itemName = new File(path).getName();
            boolean flag = isDir(root, itemName);
            System.out.println(item.getUri() + "\t" + path +"\t"+itemName + "\t" + flag);
            if(flag) {
                // Create local directory
                new File(localPath).mkdirs();
                CloudFileDirectory next = root.getDirectoryReference(itemName);
                // Create cloud directory for backup
                CloudFileDirectory backupNext = backup.getDirectoryReference(itemName);
                backupNext.createIfNotExists();
                // Recursion
                download(next, backupNext);
            } else {
                // Download file to local
                FileOutputStream fos = new FileOutputStream(localPath);
                CloudFile file = root.getFileReference(itemName);
                file.download(fos);
                // Start Copy to cloud directory for backup without upload again
                CloudFile backupFile = backup.getFileReference(itemName);
                backupFile.startCopy(file);
                System.out.println("Downloaded " + path);
            }
        }
    }

    public static void main(String[] args) throws InvalidKeyException, URISyntaxException, StorageException, FileNotFoundException {
        CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
        CloudFileClient fileClient = account.createCloudFileClient();
        CloudFileShare share = fileClient.getShareReference(fileShareName);
        CloudFileDirectory rootDir = share.getRootDirectoryReference();
        CloudFileShare backupShare = fileClient.getShareReference(backupFileShareName);
        backupShare.createIfNotExists();
        CloudFileDirectory backupRootDir = backupShare.getRootDirectoryReference();
        download(rootDir, backupRootDir);
    }

}

I have test it working on my local environment.

Hope it helps.

---

Update:

For the issue of invalid characters used in resource name, please refer to Naming and Referencing Shares, Directories, Files, and Metadata to know it, and to fix it by encoding, for example use url-encoding for /.