I tested various proxies to query a public api using the Python module requests. My requests came through fine.
Unfortunately using aiohttp with asyncio, I get the following error:
ClientHttpProxyError: 403, message='Forbidden', url=URL('http://165.225.56.117:10605')
My code looks like so:
def fire_requests(urls, proxy):
results = []
async def gather_with_concurrency(n, proxy):
semaphore = asyncio.Semaphore(n)
session = aiohttp.ClientSession(connector=conn, trust_env=True)
async def get(url, proxy):
async with semaphore:
async with session.get(url, ssl=False, proxy=proxy) as response:
obj = json.loads(await response.read())
results.append(obj)
await asyncio.gather(*(get(url, proxy) for url in urls))
await session.close()
# Initialize connection pool
conn = aiohttp.TCPConnector(limit_per_host=100, limit=0, ttl_dns_cache=300)
PARALLEL_REQUESTS = 240
loop = asyncio.get_event_loop()
loop.run_until_complete(gather_with_concurrency(PARALLEL_REQUESTS, proxy))
request_finished = datetime.now()
conn.close()
return request_finished, results