Parsing multiple gzipped json files that are contained in a zip inputstream without saving any files to disk (because of google app engine)

Question

I'm trying to parse multiple gzipped json files that are contained in one zip file through an InputStream from a http connection.

I've managed to read the first file but not more. Sometimes it fails and does not read the whole (first) file. I have checked content-length header on the connection and it is the same even when I'm failing to read the whole file.

I'm using goole app engine which doesn't allow me to save files locally which most of the examples I've found are doing.

I'm using ZipArchiveInputStream from https://commons.apache.org/proper/commons-compress/ for the Zip file.

This is the most closely related question I've been able to find: How to read from file containing multiple GzipStreams

private static ArrayList<RawEvent> parseAmplitudeEventArchiveData(HttpURLConnection connection)
        throws IOException, ParseException {
    String name, line;
    ArrayList<RawEvent> events = new ArrayList<>();

    try (ZipArchiveInputStream zipInput =
                 new ZipArchiveInputStream(connection.getInputStream(), null, false, true);) {

        ZipArchiveEntry zipEntry = zipInput.getNextZipEntry();
        if (zipEntry != null) {

            try(GZIPInputStream gzipInputStream = new GZIPInputStream(connection.getInputStream());
            BufferedReader reader = new BufferedReader(new InputStreamReader(gzipInputStream))) {

                name = zipEntry.getName();
                log.info("Parsing file: " + name);

                while ((line = reader.readLine()) != null) {
                    events.add(parseJsonLine(line));
                }
                log.info("Events size: " + events.size());
            }
        }
    }
    return events;
}

Answer 1

This works for me:

public class UnzipZippedFiles {

    public static void main(String[] args) throws IOException, ParseException {
        FileInputStream inputStream = new FileInputStream("/home/me/dev/scratchpad/src/main/resources/files.zip");
        unzipFile(inputStream);
    }

    private static void unzipFile(InputStream inputStream)
            throws IOException, ParseException {
        try (ZipArchiveInputStream zipInput =
                     new ZipArchiveInputStream(inputStream, null, false, true);) {

            ZipArchiveEntry zipEntry;

            while ((zipEntry = zipInput.getNextZipEntry()) != null) {
                System.out.println("File: " + zipEntry.getName());

                byte[] fileBytes = readDataFromZipStream(zipInput, zipEntry);

                ByteArrayInputStream byteIn = new ByteArrayInputStream(fileBytes);
                unzipGzipArchiveAndPrint(byteIn);
            }
        }
    }

    private static byte[] readDataFromZipStream(ZipArchiveInputStream zipStream, ZipArchiveEntry entry) throws IOException {
        byte[] data = new byte[(int) entry.getSize()];
        zipStream.read(data);

        return data;
    }

    private static void unzipGzipArchiveAndPrint(InputStream inputStream) throws IOException {
        System.out.println("Content:");
        try (GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream);
             BufferedReader reader = new BufferedReader(new InputStreamReader(gzipInputStream))) {

            String line;
            while ((line = reader.readLine()) != null) {
                System.out.println(line);
            }
        }
    }
}