Playwright ve AI Vision ile Dinamik Web Kazıma ve Anti-Bot Korumalarını Aşma

import asyncio
from playwright.async_api import async_playwright

async def scrape_dynamic_catalog():
    async with async_playwright() as p:
        # Gerçek tarayıcı parmak iziyle başlatma
        browser = await p.chromium.launch(
            headless=True,
            args=[
                "--disable-blink-features=AutomationControlled",
                "--no-sandbox"
            ]
        )
        context = await browser.new_context(
            viewport={"width": 1920, "height": 1080},
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
        )
        page = await context.new_page()

        # Network dinleme: HTML parse etmek yerine doğrudan API cevabını yakalama
        captured_data = []
        async def handle_response(response):
            if "api/v1/products" in response.url and response.status == 200:
                try:
                    payload = await response.json()
                    captured_data.extend(payload.get("items", []))
                except Exception:
                    pass

        page.on("response", handle_response)
        await page.goto("https://target-catalog.com/electronics", wait_until="networkidle")
        
        print(f"Başarıyla yakalanan ürün adedi: {len(captured_data)}")
        await browser.close()

asyncio.run(scrape_dynamic_catalog())