Sync http requests
examples/async/http_requests_sync.py
import requests import sys from bs4 import BeautifulSoup def fetch(url): response = requests.get(url) content = response.text return content def main(): if len(sys.argv) < 2: exit(f"Usage: {sys.argv[0]} FILENAME") # examples/parallel/url.txt filename = sys.argv[1] with open(filename) as fh: urls = list(map(lambda url: url.rstrip('\n'), fh.readlines())) #urls = urls[0:10] #urls = ['https://httpbin.org/get'] #print(urls) tasks = [] for url in urls: # tasks.append([url, fetch(url)]) # for task in tasks: # url, content = task content = fetch(url) print(url) #print(content) soup = BeautifulSoup(content, 'html.parser') print('Missing' if soup.title is None else soup.title.string) #print(task) main()