update the code to handle ads and cloudflare challenge

app.py

@@ -20,6 +20,53 @@ from pathlib import Path
 import subprocess
 import traceback
 
+# =============================================
+# CONFIGURATION
+# =============================================
+
+BLOCK_PATTERNS = ["doubleclick", "adservice", "googlesyndication", "ads", "adserver", "cookie", "consent"]
+PAGE_TIMEOUT = 60000  # 60 seconds
+WAIT_FOR_LOAD_TIMEOUT = 10000  # 10 seconds extra wait after load
+CLOUDFLARE_CHECK_KEYWORDS = ["Checking your browser", "Just a moment", "Cloudflare"]
+
+# =============================================
+# HELPER FUNCTIONS
+# =============================================
+
+def ensure_http(url):
+    if not url.startswith(('http://', 'https://')):
+        return 'http://' + url
+    return url
+
+def sanitize_filename(url):
+    return re.sub(r'[^\w\-_\. ]', '_', url)
+
+def block_ads_and_cookies(page):
+    def route_intercept(route):
+        if any(resource in route.request.url.lower() for resource in BLOCK_PATTERNS):
+            route.abort()
+        else:
+            route.continue_()
+    page.route("**/*", route_intercept)
+
+def wait_for_page_stable(page):
+    try:
+        page.wait_for_load_state('networkidle', timeout=PAGE_TIMEOUT)
+        time.sleep(WAIT_FOR_LOAD_TIMEOUT / 1000)  # extra wait
+    except Exception as e:
+        print(f"⚠️  Page not fully stable: {e}")
+
+def detect_and_bypass_cloudflare(page):
+    try:
+        content = page.content()
+        if any(keyword.lower() in content.lower() for keyword in CLOUDFLARE_CHECK_KEYWORDS):
+            print("⚡ Detected Cloudflare challenge, waiting 5 seconds...")
+            time.sleep(5)
+            page.reload()
+            wait_for_page_stable(page)
+    except Exception as e:
+        print(f"⚠️  Failed to bypass Cloudflare: {e}")
+
 # --- Setup ---
 
 # Device setup
@@ -175,29 +222,53 @@ def clean_text(text):
 
 # Fungsi untuk mengambil screenshot viewport
 def take_screenshot(url):
-    filename = url.replace('https://', '').replace('http://', '').replace('/', '_').replace('.', '_') + '.png'
+    url = ensure_http(url)
+    filename = sanitize_filename(url) + '.png'
     filepath = os.path.join(SCREENSHOT_DIR, filename)
     
     try:
         print(f"\n=== [START SCREENSHOT] URL: {url} ===")
-        from playwright.sync_api import sync_playwright
-
         with sync_playwright() as p:
             print("Launching Playwright Chromium...")
-            browser = p.chromium.launch()
-            page = browser.new_page(
+            browser = p.chromium.launch(
+                args=['--disable-features=IsolateOrigins,site-per-process']  # Disable site isolation
+            )
+            
+            context = browser.new_context(
                 viewport={"width": 1280, "height": 800},
                 user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
             )
-            page.set_default_timeout(60000)
+            
+            page = context.new_page()
+            page.set_default_timeout(PAGE_TIMEOUT)
             page.set_extra_http_headers({"Accept-Language": "en-US,en;q=0.9"})
 
-            print("Navigating to URL...")
-            page.goto(url, wait_until="networkidle", timeout=60000)
-            page.wait_for_timeout(3000)
+            # Block ads and tracking
+            print("Setting up ad and tracking blockers...")
+            block_ads_and_cookies(page)
 
-            print("Taking screenshot (viewport only)...")
-            page.screenshot(path=filepath)
+            try:
+                print("Navigating to URL...")
+                page.goto(url, wait_until="domcontentloaded", timeout=PAGE_TIMEOUT)
+                
+                # Handle potential Cloudflare protection
+                detect_and_bypass_cloudflare(page)
+                
+                # Wait for page to be stable
+                wait_for_page_stable(page)
+                
+                print("Taking screenshot (viewport only)...")
+                page.screenshot(path=filepath)
+                
+            except Exception as nav_error:
+                print(f"Navigation error: {nav_error}")
+                # Try to take screenshot anyway if page partially loaded
+                if page.url != "about:blank":
+                    page.screenshot(path=filepath)
+                else:
+                    raise nav_error
+
+            context.close()
             browser.close()
             print(f"Screenshot saved to {filepath}")