How to implement Parallel Zip Creation with ScatterZipOutputStream with Zip64 Support?

Question

I am wondering if any one can help implementing Parallel Zip Creation using ScatterZipOutputStream . I have searched a lot but no where I am finding an example for the same.

https://commons.apache.org/proper/commons-compress/zip.html

I have tried making Zip, Zipping a directory etc with ZipArchiveOutputStream . Now, I am trying to do that in parallel.

public static void makeZip(String filename) throws IOException,
        ArchiveException {
    File sourceFile = new File(filename);

    final OutputStream out = new FileOutputStream(filename.substring(0, filename.lastIndexOf('.')) + ".zip");
    ZipArchiveOutputStream os = new ZipArchiveOutputStream(out);
    os.setUseZip64(Zip64Mode.AsNeeded);

    os.putArchiveEntry(new ZipArchiveEntry(sourceFile.getName()));
    IOUtils.copy(new FileInputStream(sourceFile), os);
    os.closeArchiveEntry();
    os.close();
}

It should be able to process individual files as thread and then combine it to write the result zip.

Answer 1

Following is the working code of both zip and unzip:
1. Change path for sourceFolder and zipFilePath
2. Zipping only *.text type of files it can be any type or all the files
3. Unzipped files at sourceFolder/unzip/

Import following dependency in build.gradle or in pom.xml

implementation("org.apache.commons:commons-compress:1.18")
implementation("commons-io:commons-io:2.6")

Ref: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.18 https://mvnrepository.com/artifact/commons-io/commons-io/2.6

//code

import org.apache.commons.compress.archivers.zip.*;
import org.apache.commons.compress.parallel.InputStreamSupplier;
import org.apache.commons.io.FileUtils;

import java.io.*;
import java.nio.file.Files;
import java.util.Iterator;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

public class ZipMain {

static ParallelScatterZipCreator scatterZipCreator = new ParallelScatterZipCreator();
static ScatterZipOutputStream dirs;

static {
    try {
        dirs = ScatterZipOutputStream.fileBased(File.createTempFile("java-zip-dirs", "tmp"));
    } catch (IOException e) {
        e.printStackTrace();
    }
}

public static void main(String[] args) throws IOException {
    String sourceFolder = "/Users/<user>/Desktop/";
    String zipFilePath = "/Users/<user>/Desktop/Desk.zip";
    String fileTypesToBeAddedToZip = "txt";
    zip(sourceFolder, zipFilePath, fileTypesToBeAddedToZip);
    unzip(zipFilePath, sourceFolder + "/unzip/");
}

private static void zip(String sourceFolder, String zipFilePath, String fileTypesToBeAddedToZip) throws IOException {
    OutputStream outputStream = null;
    ZipArchiveOutputStream zipArchiveOutputStream = null;
    try {

        File srcFolder = new File(sourceFolder);
        if (srcFolder.isDirectory()) {
            // uncomment following code if you want to add all files under srcFolder
            //Iterator<File> fileIterator = Arrays.asList(srcFolder.listFiles()).iterator();
            Iterator<File> fileIterator = FileUtils.iterateFiles(srcFolder, new String[]{fileTypesToBeAddedToZip}, true);

            File zipFile = new File(zipFilePath);
            zipFile.delete();
            outputStream = new FileOutputStream(zipFile);

            zipArchiveOutputStream = new ZipArchiveOutputStream(outputStream);
            zipArchiveOutputStream.setUseZip64(Zip64Mode.AsNeeded);

            int srcFolderLength = srcFolder.getAbsolutePath().length() + 1;  // +1 to remove the last file separator

            while (fileIterator.hasNext()) {
                File file = fileIterator.next();

                // uncomment following code if you want to add all files under srcFolder
                //if (file.isDirectory()) {
                //        continue;
                //    }

                String relativePath = file.getAbsolutePath().substring(srcFolderLength);


                InputStreamSupplier streamSupplier = () -> {
                    InputStream is = null;
                    try {
                        is = Files.newInputStream(file.toPath());
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                    return is;
                };
                ZipArchiveEntry zipArchiveEntry = new ZipArchiveEntry(relativePath);
                zipArchiveEntry.setMethod(ZipEntry.DEFLATED);
                scatterZipCreator.addArchiveEntry(zipArchiveEntry, streamSupplier);
            }
            scatterZipCreator.writeTo(zipArchiveOutputStream);
            }
            if (zipArchiveOutputStream != null) {
                zipArchiveOutputStream.close();
            }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (outputStream != null) {
            outputStream.close();
        }
    }
}

private static void unzip(String zipFilePath, String destDir) {
    File dir = new File(destDir);
    // create output directory if it doesn't exist
    if (!dir.exists()) {
        dir.mkdirs();
    } else {
        dir.delete();
    }

    FileInputStream fis;
    //buffer for read and write data to file
    byte[] buffer = new byte[1024];
    try {
        fis = new FileInputStream(zipFilePath);
        ZipInputStream zis = new ZipInputStream(fis);
        ZipEntry ze = zis.getNextEntry();
        while (ze != null) {
            String fileName = ze.getName();

            File newFile = new File(destDir + File.separator + fileName);

            System.out.println("Unzipping to " + newFile.getAbsolutePath());

            //create directories for sub directories in zip
            String parentFolder = newFile.getParent();
            File folder = new File(parentFolder);
            folder.mkdirs();

            FileOutputStream fos = new FileOutputStream(newFile);
            int len;
            while ((len = zis.read(buffer)) > 0) {
                fos.write(buffer, 0, len);
            }
            fos.close();
            //close this ZipEntry
            zis.closeEntry();
            ze = zis.getNextEntry();
        }
        //close last ZipEntry
        zis.closeEntry();
        zis.close();
        fis.close();
    } catch (IOException e) {
        e.printStackTrace();
    }

}
}

Ref: Fast zipping folder using java ParallelScatterZipCreator