I need to download many different users data in a very slow API, so I decided to make parallel requests. After trying multithreads and multiprocess I decided asynco download would be the best in my case, and my code works for some cases I have tried without the header that includes the Bearer Auth... But in this case it gives ClientConnectorCertificateError... Any ideas what I am doing wrong?
import aiohttp
import asyncio
import time
start_time = time.time()
headers = {'accept': 'application/json','Authorization': "Bearer {}".format(token)}
async def get_cch(session, url):
async with session.get(url, headers=headers) as resp:
cch = await resp.json()
x= cch['data']['contractId']
return open(f'{x}.json', 'w').write(json.dumps(cch))
async def main():
async with aiohttp.ClientSession(headers=headers) as session:
tasks = []
for contract in range(len(file_name_list)):
start_date = '2018-01-01'
end_date = '2018-01-05'
url = 'https://apinergia.somenergia.coop/cch/'+file_name_list[contract]+'?tariff=None&type=tg_cchfact&from_='+start_date+'&to_='+end_date+'&limit=30000'
tasks.append(asyncio.ensure_future(get_cch(session, url)))
original_cch = await asyncio.gather(*tasks)
for cch in original_cch:
print(cch)
await main()
print("--- %s seconds ---" % (time.time() - start_time))