import asyncio import aiohttp from api import fetch_filter_count, fetch_main, fetch_single_page from config import PAGE_SIZE, PROFILE_CONCURRENCY, TG_CONCURRENCY from girl_parser import parse_girl_links, parse_single_girl from page_data import fetch_args, fetch_nonce from storage import ResultWriter from telegram_checker import check_telegram async def process_girl(session, link, writer, profile_sem, tg_sem): try: async with profile_sem: girl = await parse_single_girl(session, link) if not girl: return None tg_link = girl.get("telegram") if not tg_link: return None async with tg_sem: free_username = await check_telegram(session, tg_link) if not free_username: return None await writer.add(girl["profile_url"], free_username) return free_username except Exception as e: print(f"Профиль {link.get('url')} - ошибка: {type(e).__name__}: {e}") return None async def collect_links(session): html_content = await fetch_main(session) if not html_content: raise ValueError("Пустой ответ от главной страницы каталога.") nonce = fetch_nonce(html_content) args = fetch_args(html_content) print("Nonce:", nonce) total = await fetch_filter_count(session, nonce) print("Всего анкет по фильтру:", total) pages_count = -(-total // PAGE_SIZE) # деление с округлением вверх без math.ceil page_ids = range(1, pages_count + 1) print(f"Страниц к обходу: {pages_count}") pages = await asyncio.gather( *(fetch_single_page(session, page_id, nonce, args) for page_id in page_ids) ) links = [] seen = set() for page_html in pages: for item in parse_girl_links(page_html): if item["url"] not in seen: seen.add(item["url"]) links.append(item) print(f"Уникальных анкет: {len(links)}") return links async def main(): async with aiohttp.ClientSession() as session: links = await collect_links(session) writer = ResultWriter() writer.reset() profile_sem = asyncio.Semaphore(PROFILE_CONCURRENCY) tg_sem = asyncio.Semaphore(TG_CONCURRENCY) tasks = [ asyncio.create_task( process_girl(session, link, writer, profile_sem, tg_sem) ) for link in links ] found = [] for task in asyncio.as_completed(tasks): username = await task if username: found.append(username) print(f"Свободный юзернейм: @{username}") print(f"Итого свободных: {len(found)}") if __name__ == "__main__": asyncio.run(main())