How can I async the following code, without first downloading the entire file:
import gzip
import urllib.request
def gunzip_url(url: str):
with gzip.open(urllib.request.urlopen(url), 'rt') as f:
return f.read()
The following works, but it is not on the fly (downloads entire gz file before decompressing):
import aiohttp
import asyncio
import gzip
async def gunzip_url(client: aiohttp.ClientSession, url: str):
async with client.get(url) as resp:
gz = await resp.read()
return gzip.decompress(gz)
async def main():
async with aiohttp.ClientSession() as client:
coros = [gunzip_url(client, 'http://some.file/1.gz'),
gunzip_url(client, 'http://some.file/2.gz')]
return await asyncio.gather(*coros)
data = asyncio.run(main())
The following works, but is also not on the fly:
from aioify import aioify
agzip = aioify(obj=gzip, name='agzip')
async def gunzip_url(client: aiohttp.ClientSession, url):
async with client.get(url) as resp:
return await agzip.decompress(await resp.read())