Async http requests with queue
examples/async/http_requests_async_queue.py
import aiohttp import asyncio import sys from bs4 import BeautifulSoup async def fetch(session, urls, results): while urls: url = urls.pop(0) async with session.get(url) as response: content = await response.text() soup = BeautifulSoup(content, 'html.parser') results[url] = None if soup.title is None else soup.title.string async def main(parallel, urls): results = {} async with aiohttp.ClientSession() as session: tasks = [] for _ in range(parallel): tasks.append(asyncio.create_task(fetch(session, urls, results))) await asyncio.gather(*tasks) return results def setup(): if len(sys.argv) < 4: exit(f"Usage: {sys.argv[0]} LIMIT PARALLEL FILENAME") # examples/parallel/url.txt or wikipedia.txt limit = int(sys.argv[1]) parallel = int(sys.argv[2]) filename = sys.argv[3] with open(filename) as fh: urls = list(map(lambda url: url.rstrip('\n'), fh.readlines())) if len(urls) > limit: urls = urls[0:limit] # use asyncio.Queue instead results = asyncio.run(main(parallel, urls)) for url, title in results.items(): print(f"{url} - {title}") setup()