98 lines
2.9 KiB
Python
98 lines
2.9 KiB
Python
import asyncio
|
|
|
|
import aiohttp
|
|
|
|
from api import fetch_filter_count, fetch_main, fetch_single_page
|
|
from config import PAGE_SIZE, PROFILE_CONCURRENCY, TG_CONCURRENCY
|
|
from girl_parser import parse_girl_links, parse_single_girl
|
|
from page_data import fetch_args, fetch_nonce
|
|
from storage import ResultWriter
|
|
from telegram_checker import check_telegram
|
|
|
|
|
|
async def process_girl(session, link, writer, profile_sem, tg_sem):
|
|
try:
|
|
async with profile_sem:
|
|
girl = await parse_single_girl(session, link)
|
|
if not girl:
|
|
return None
|
|
|
|
tg_link = girl.get("telegram")
|
|
if not tg_link:
|
|
return None
|
|
|
|
async with tg_sem:
|
|
free_username = await check_telegram(session, tg_link)
|
|
if not free_username:
|
|
return None
|
|
|
|
await writer.add(girl["profile_url"], free_username)
|
|
return free_username
|
|
|
|
except Exception as e:
|
|
print(f"Профиль {link.get('url')} - ошибка: {type(e).__name__}: {e}")
|
|
return None
|
|
|
|
|
|
async def collect_links(session):
|
|
html_content = await fetch_main(session)
|
|
if not html_content:
|
|
raise ValueError("Пустой ответ от главной страницы каталога.")
|
|
|
|
nonce = fetch_nonce(html_content)
|
|
args = fetch_args(html_content)
|
|
print("Nonce:", nonce)
|
|
|
|
total = await fetch_filter_count(session, nonce)
|
|
print("Всего анкет по фильтру:", total)
|
|
|
|
pages_count = -(-total // PAGE_SIZE) # деление с округлением вверх без math.ceil
|
|
page_ids = range(1, pages_count + 1)
|
|
print(f"Страниц к обходу: {pages_count}")
|
|
|
|
pages = await asyncio.gather(
|
|
*(fetch_single_page(session, page_id, nonce, args) for page_id in page_ids)
|
|
)
|
|
|
|
links = []
|
|
seen = set()
|
|
for page_html in pages:
|
|
for item in parse_girl_links(page_html):
|
|
if item["url"] not in seen:
|
|
seen.add(item["url"])
|
|
links.append(item)
|
|
|
|
print(f"Уникальных анкет: {len(links)}")
|
|
return links
|
|
|
|
|
|
async def main():
|
|
async with aiohttp.ClientSession() as session:
|
|
links = await collect_links(session)
|
|
|
|
writer = ResultWriter()
|
|
writer.reset()
|
|
|
|
profile_sem = asyncio.Semaphore(PROFILE_CONCURRENCY)
|
|
tg_sem = asyncio.Semaphore(TG_CONCURRENCY)
|
|
|
|
tasks = [
|
|
asyncio.create_task(
|
|
process_girl(session, link, writer, profile_sem, tg_sem)
|
|
)
|
|
for link in links
|
|
]
|
|
|
|
found = []
|
|
for task in asyncio.as_completed(tasks):
|
|
username = await task
|
|
if username:
|
|
found.append(username)
|
|
print(f"Свободный юзернейм: @{username}")
|
|
|
|
print(f"Итого свободных: {len(found)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|