import requests
from joblib import Parallel, delayed
from tqdm import tqdm
def save_html(url):
r = requests.get(url)
assert r.status_code == 200, "Bad request"
# Save HTML somewhere
# ...%%time
url = "https://quotes.toscrape.com/"
if __name__ == "__main__":
Parallel(n_jobs=10, backend="multiprocessing")(
delayed(save_html)(url) for i in tqdm(range(1000))
)[OUT]: Wall time: 2min 37sec