I've scoured StackOverflow for similar questions and haven't found an answer that fits my need. I'm trying to use MalwareBaazar's API to get info about a number of file hashes via POST requests (via API spec). The API has no rate limiting nor a limit on the number of concurrent requests. I'm using aiohttp along with asyncio and I believe my code is correct; however, I randomly get a RuntimeError: await wasn't used with future every few runs. I am not sure why and was hoping someone might offer suggestions as to where to look for debugging. Thanks!
async def hash_info_async(file_hashes: List[str], sleep_on_error=300) -> List[str]:
results = []
# set number of concurrent requests to host
#connector = aiohttp.TCPConnector(limit_per_host=6, force_close=False)
# create asynchronous HTTP client session
HEADER_DATA = {'Accept': 'application/json'}
try:
async with aiohttp.ClientSession(headers=HEADER_DATA) as session:
calls = [session.post("http://mb-api.abuse.ch/api/v1/", data={'query': 'get_info', 'hash': h}) for h in file_hashes]
responses = await asyncio.gather(*calls)
for resp in responses:
decoded_resp = await resp.json(encoding='utf-8', content_type='application/json')
if decoded_resp['query_status'] != 'ok':
logging.warning(f"MalwareBazaar error : {decoded_resp['query_status']}")
results.append(None)
else:
results.append(decoded_resp)
return results
except aiohttp.ServerConnectionError as e:
logging.error(e)
return []
# async test
print('!!! STARTING ASYNC !!!')
data_async = asyncio.run(hash_info_async(hashes))
if len(data_async) != len(hashes):
print('ASYNC error')
!!! STARTING ASYNC !!!
Traceback (most recent call last):
File "/Users/me/Library/Application Support/JetBrains/PyCharm2022.2/scratches/scratch.py", line 135, in <module>
data_async = asyncio.run(hash_info_async(hashes))
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/asyncio/base_events.py", line 642, in run_until_complete
return future.result()
File "/Users/me/Library/Application Support/JetBrains/PyCharm2022.2/scratches/scratch.py", line 118, in hash_info_async
responses = await asyncio.gather(*calls)
File "/Users/me/lib/python3.9/site-packages/aiohttp/client.py", line 1125, in send
return self._coro.send(arg)
File "/Users/me/lib/python3.9/site-packages/aiohttp/client.py", line 536, in _request
conn = await self._connector.connect(
File "/Users/me/lib/python3.9/site-packages/aiohttp/connector.py", line 540, in connect
proto = await self._create_connection(req, traces, timeout)
File "/Users/me/lib/python3.9/site-packages/aiohttp/connector.py", line 901, in _create_connection
_, proto = await self._create_direct_connection(req, traces, timeout)
File "/Users/me/lib/python3.9/site-packages/aiohttp/connector.py", line 1175, in _create_direct_connection
transp, proto = await self._wrap_create_connection(
File "/Users/me/lib/python3.9/site-packages/aiohttp/connector.py", line 980, in _wrap_create_connection
return await self._loop.create_connection(*args, **kwargs) # type: ignore[return-value] # noqa
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/asyncio/base_events.py", line 1041, in create_connection
sock = await self._connect_sock(
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/asyncio/base_events.py", line 955, in _connect_sock
await self.sock_connect(sock, address)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/asyncio/selector_events.py", line 502, in sock_connect
return await fut
RuntimeError: await wasn't used with future
