Refactor: Replace httpx with curl-cffi for HTTP requests and update dependencies

pyfragment/core/constants.py

@@ -24,15 +24,14 @@ BASE_HEADERS: dict[str, str] = {
     "content-type": "application/x-www-form-urlencoded; charset=UTF-8",
     "origin": FRAGMENT_BASE_URL,
     "priority": "u=1, i",
-    "sec-ch-ua": '"Google Chrome";v="147", "Not.A/Brand";v="8", "Chromium";v="147"',
+    "sec-ch-ua": '"Not;A=Brand";v="8", "Chromium";v="150", "Google Chrome";v="150"',
     "sec-ch-ua-mobile": "?1",
     "sec-ch-ua-platform": '"Android"',
     "sec-fetch-dest": "empty",
     "sec-fetch-mode": "cors",
     "sec-fetch-site": "same-origin",
     "user-agent": (
-        "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Mobile Safari/537.36"
+        "Mozilla/5.0 (Linux; Android 15; Pixel 9) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Mobile Safari/537.36"
     ),
     "x-requested-with": "XMLHttpRequest",
 }

pyfragment/core/transport.py

@@ -5,35 +5,25 @@ import random
 import re
 from typing import Any, cast
 
-import httpx
+from curl_cffi.requests import AsyncSession, Response
 
-from pyfragment.core.constants import DEFAULT_TIMEOUT, FRAGMENT_BASE_URL
+from pyfragment.core.constants import FRAGMENT_BASE_URL
 from pyfragment.exceptions import FragmentPageError, ParseError
 
 
 async def get_fragment_hash(
-    cookies: dict[str, Any],
+    session: AsyncSession[Any],  # type: ignore[type-arg]
     headers: dict[str, str],
     page_url: str,
-    timeout: float = DEFAULT_TIMEOUT,
 ) -> str:
-    page_headers = {
-        k: v
-        for k, v in headers.items()
-        if k not in ("accept", "accept-encoding", "content-type", "x-requested-with", "x-aj-referer")
-    }
-    page_headers.update(
-        {
-            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-            "referer": f"{FRAGMENT_BASE_URL}/",
-            "sec-fetch-dest": "document",
-            "sec-fetch-mode": "navigate",
-            "upgrade-insecure-requests": "1",
-        }
-    )
+    # Derive the natural referer: strip the last path segment (e.g. /stars/buy → /stars)
+    parent_url = page_url.rsplit("/", 1)[0] or FRAGMENT_BASE_URL
 
-    async with httpx.AsyncClient(cookies=cookies, timeout=timeout) as session:
-        response = await session.get(page_url, headers=page_headers)
+    page_headers = {k: v for k, v in headers.items() if k not in ("content-type", "origin")}
+    page_headers["referer"] = parent_url
+    page_headers["x-aj-referer"] = parent_url
+
+    response = await session.get(page_url, headers=page_headers)
 
     if response.status_code != 200:
         raise FragmentPageError(FragmentPageError.BAD_STATUS.format(status=response.status_code, url=page_url))
@@ -45,15 +35,15 @@ async def get_fragment_hash(
     return match.group(1)
 
 
-def parse_json_response(response: httpx.Response, context: str) -> dict[str, Any]:
+def parse_json_response(response: Response, context: str) -> dict[str, Any]:
     try:
-        return cast(dict[str, Any], response.json())
+        return cast(dict[str, Any], response.json())  # type: ignore[no-untyped-call]
     except Exception as exc:
         raise ParseError(ParseError.UNPARSEABLE.format(context=context, exc=exc)) from exc
 
 
 async def fragment_request(
-    session: httpx.AsyncClient,
+    session: AsyncSession[Any],  # type: ignore[type-arg]
     fragment_hash: str,
     headers: dict[str, str],
     data: dict[str, Any],

pyfragment/domains/base.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING, Any
 
-import httpx
+from curl_cffi.requests import AsyncSession
 
 from pyfragment.core.constants import BASE_HEADERS
 from pyfragment.core.transport import fragment_request, get_fragment_hash
@@ -27,8 +27,8 @@ async def raw_api_call(
     call_headers = {**base, "referer": page_url, "x-aj-referer": page_url}
     logger.debug("Starting Fragment API call '%s' on %s", method, page_url)
     try:
-        async with httpx.AsyncClient(cookies=cookies, timeout=timeout) as session:
-            fragment_hash = await get_fragment_hash(cookies, call_headers, page_url, timeout)
+        async with AsyncSession(cookies=cookies, timeout=timeout, impersonate="chrome") as session:
+            fragment_hash = await get_fragment_hash(session, call_headers, page_url)
             response = await fragment_request(session, fragment_hash, call_headers, payload)
             logger.debug("Completed Fragment API call '%s' with response keys: %s", method, sorted(response.keys()))
             return response

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Typing :: Typed",
 ]
-dependencies = ["httpx>=0.25", "tonutils>=2.0.1"]
+dependencies = ["curl-cffi>=0.7", "tonutils>=2.0.1"]
 
 [project.optional-dependencies]
 dev = ["pytest", "pytest-asyncio", "pytest-mock", "mypy", "ruff"]

tests/008_test_call.py

@@ -2,8 +2,8 @@
 
 from unittest.mock import AsyncMock, MagicMock, patch
 
-import httpx
 import pytest
+from curl_cffi.requests import AsyncSession, Response
 
 from pyfragment import FragmentClient, FragmentPageError
 from pyfragment.core.transport import fragment_request
@@ -67,10 +67,10 @@ async def test_call_merges_extra_data(client: FragmentClient) -> None:
 
 @pytest.mark.asyncio
 async def test_fragment_request_non_200_raises() -> None:
-    response = MagicMock(spec=httpx.Response)
+    response = MagicMock(spec=Response)
     response.status_code = 429
 
-    session = AsyncMock(spec=httpx.AsyncClient)
+    session = AsyncMock(spec=AsyncSession)
     session.post = AsyncMock(return_value=response)
 
     with pytest.raises(FragmentPageError, match="429"):