1

Consider the following code that depends on zstandard library from pip:

import zstandard
import contextlib

old_open = open


@contextlib.contextmanager
def open(*, fileobj=None, file=None, mode='r'):
    if mode not in ('r', 'w'):
        raise ValueError('Mode should be "r" or "w")')
    if not fileobj and not file:
        raise ValueError('either fileobj or file must be provided')
    if mode == "r":
        cctx = zstandard.ZstdDecompressor()
        if file:
            with old_open(file, 'rb') as fh, cctx.stream_reader(fh) as reader:
                yield reader
        else:
            yield cctx.stream_reader(fileobj)
    elif mode == "w":
        cctx = zstandard.ZstdCompressor()
        if file:
            with old_open(file, 'wb') as fh, cctx.stream_writer(fh) as writer:
                yield writer
        else:
            yield cctx.stream_writer(fileobj)


if __name__ == '__main__':
    import io
    with open(file='/tmp/a.zst', mode='w') as f:
        f.write(b'asd')

    with open(file='/tmp/a.zst', mode='r') as f:
        for line in f:
            print(line)

I'm getting the following error:

Traceback (most recent call last):
  File "zstdopen.py", line 35, in <module>
    for line in f:
io.UnsupportedOperation

Given that read() is already there and works, is there a way to wrap this in an interface that would automatically provide readline() for me?

d33tah
  • 10,999
  • 13
  • 68
  • 158
  • Does this answer your question? [subclassing file objects (to extend open and close operations) in python 3](https://stackoverflow.com/questions/16085292/subclassing-file-objects-to-extend-open-and-close-operations-in-python-3) – mkrieger1 Jul 31 '20 at 19:32
  • Note that you don't need to define `old_open`; the built-in function is always available via the `builtins` module. – chepner Jul 31 '20 at 19:43

1 Answers1

2

It looks like subclassing io.RawIOBase helped:

import zstandard
import contextlib
import io

old_open = open


class NewClass(io.RawIOBase):

    def __init__(self, f):
        self.f = f

    def read(self, *args, **kwargs):
        return self.f.read(*args, **kwargs)


@contextlib.contextmanager
def open(*, fileobj=None, file=None, mode='r'):
    if mode not in ('r', 'w'):
        raise ValueError('Mode should be "r" or "w")')
    if not fileobj and not file:
        raise ValueError('either fileobj or file must be provided')
    if mode == "r":
        cctx = zstandard.ZstdDecompressor()
        if file:
            with old_open(file, 'rb') as fh, cctx.stream_reader(fh) as reader:
                yield NewClass(reader)
        else:
            yield NewClass(cctx.stream_reader(fileobj))
    elif mode == "w":
        cctx = zstandard.ZstdCompressor()
        if file:
            with old_open(file, 'wb') as fh, cctx.stream_writer(fh) as writer:
                yield writer
        else:
            yield cctx.stream_writer(fileobj)


if __name__ == '__main__':
    with open(file='/tmp/a.zst', mode='w') as f:
        f.write(b'asd\nasd2')

    with open(file='/tmp/a.zst', mode='r') as f:
        for line in f:
            print(repr(line))
d33tah
  • 10,999
  • 13
  • 68
  • 158
  • 1
    Its great to see someone come up with a problem and then work out a way to solve their own question. Kudos to you @d33tah. – Joe Ferndz Jul 31 '20 at 21:07
  • 1
    It seems they intentionally dropped support for `__iter__` https://github.com/indygreg/python-zstandard/commit/690d534b2b205cefd81a2f06b1b175075c7a4978#diff-fcac7b72af0a3f9e15d70841e946246bR503 – Thomas Grainger Aug 01 '20 at 00:45