我不知道如何加速我的解析器
import asyncio
import aiohttp
from bs4 import BeautifulSoup
import time
import requests
async def get_html(url):
async with aiohttp.ClientSession() as client:
async with client.get(url) as r:
return await r.text()
async def main():
t0 = time.time()
tasks = []
count = 0
for i in range(1,201):
url = "https://cspromogame.ru/avatars/?page="
url = url + str(i)
task = asyncio.create_task(get_html(url))
tasks.append(task)
p = await asyncio.gather(*tasks)
for text in p:
soup = BeautifulSoup(text, 'html.parser')
for a in soup.findAll('a',class_='avatars__link'):
link = a.get("href")
await makePhoto(link)
count += 1
print(time.time()-t0)
print(count)
async def makePhoto(url):
name = str(url.split("/")[-1:]).replace(".jpg","")
print(url)
async with aiohttp.ClientSession(raise_for_status=True) as client:
async with client.get(url) as r:
with open(f"{name}.jpg","wb") as f:
f.write(await r.text())
if __name__ == "__main__":
asyncio.run(main())
