update the code to handle ads and cloudflare challenge fixed3

app.py

@@ -243,7 +243,16 @@ def create_browser_context(playwright):
             '--disable-setuid-sandbox',
             '--no-sandbox',
             '--disable-gpu',
-            '--disable-dev-shm-usage'
+            '--disable-dev-shm-usage',
+            '--disable-extensions',
+            '--disable-plugins',
+            '--disable-background-timer-throttling',
+            '--disable-backgrounding-occluded-windows',
+            '--disable-renderer-backgrounding',
+            '--no-first-run',
+            '--no-default-browser-check',
+            '--disable-translate',
+            '--disable-ipc-flooding-protection'
         ]
     ).new_context(
         viewport={"width": 1280, "height": 800},
@@ -255,7 +264,8 @@ def create_browser_context(playwright):
             "Accept-Language": "en-US,en;q=0.9",
             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
             "Connection": "keep-alive",
-            "DNT": "1"
+            "DNT": "1",
+            "Cache-Control": "no-cache"
         }
     )
 
@@ -292,75 +302,129 @@ def setup_request_interception(page):
     page.on("response", handle_response)
     page.route("**/*", handle_request)
 
+def try_navigation_strategies(page, url):
+    strategies = [
+        {"wait_until": "commit", "timeout": 15000},
+        {"wait_until": "domcontentloaded", "timeout": 10000},
+        {"wait_until": "load", "timeout": 20000},
+        {"wait_until": "networkidle", "timeout": 30000}
+    ]
+    
+    for i, strategy in enumerate(strategies):
+        try:
+            print(f"Trying navigation strategy {i+1}: {strategy}")
+            response = page.goto(url, **strategy)
+            print(f"Navigation successful with strategy {i+1}")
+            return response
+        except Exception as e:
+            print(f"Strategy {i+1} failed: {e}")
+            if "ERR_TOO_MANY_REDIRECTS" in str(e):
+                print(f"Redirect error detected, trying next strategy...")
+                continue
+            elif i == len(strategies) - 1:  # Last strategy
+                raise e
+            continue
+    
+    raise Exception("All navigation strategies failed")
+
 def take_screenshot(url):
     url = ensure_http(url)
     filename = sanitize_filename(url) + '.png'
     filepath = os.path.join(SCREENSHOT_DIR, filename)
     
-    try:
-        print(f"\n=== [START SCREENSHOT] URL: {url} ===")
-        
-        with sync_playwright() as p:
-            print("Launching browser with custom configuration...")
-            context = create_browser_context(p)
-            page = context.new_page()
-            
-            print("Setting up request interception...")
-            setup_request_interception(page)
+    max_retries = 3
+    
+    for attempt in range(max_retries):
+        try:
+            print(f"\n=== [SCREENSHOT ATTEMPT {attempt + 1}/{max_retries}] URL: {url} ===")
             
-            try:
-                print("Attempting to navigate to URL...")
-                response = page.goto(
-                    url,
-                    wait_until="commit",
-                    timeout=PAGE_TIMEOUT
-                )
+            with sync_playwright() as p:
+                print("Launching browser with aggressive configuration...")
+                context = create_browser_context(p)
+                page = context.new_page()
                 
-                if not response:
-                    print("No response received, attempting to continue...")
-                elif response.status >= 300 and response.status <= 399:
-                    print(f"Received redirect status code: {response.status}")
-                elif response.status >= 400:
-                    print(f"Received error status code: {response.status}")
+                # Only set up basic request blocking for this attempt
+                if attempt == 0:
+                    print("Setting up basic request interception...")
+                    def simple_block(route):
+                        url_lower = route.request.url.lower()
+                        if any(pattern in url_lower for pattern in BLOCK_PATTERNS):
+                            route.abort()
+                        else:
+                            route.continue_()
+                    page.route("**/*", simple_block)
                 
-                # Try to wait for the page to be more stable
                 try:
-                    wait_for_page_stable(page)
-                except Exception as e:
-                    print(f"Page stability warning: {e}")
-                
-                # Take screenshot even if page might not be fully loaded
-                print("Taking screenshot...")
-                page.screenshot(path=filepath)
-                
-            except Exception as nav_error:
-                print(f"Navigation error: {nav_error}")
-                # Try to take screenshot anyway if we have any content
-                try:
-                    if page.url != "about:blank":
-                        print("Taking screenshot of partial page...")
-                        page.screenshot(path=filepath)
+                    # Try different navigation strategies
+                    if attempt == 0:
+                        # First attempt: aggressive but safe
+                        response = try_navigation_strategies(page, url)
+                    elif attempt == 1:
+                        # Second attempt: minimal approach
+                        print("Trying minimal navigation approach...")
+                        response = page.goto(url, wait_until="commit", timeout=10000)
                     else:
-                        raise nav_error
-                except:
-                    raise nav_error
-            finally:
-                try:
+                        # Third attempt: just try to load anything
+                        print("Trying basic navigation...")
+                        response = page.goto(url, timeout=15000)
+                    
+                    if response:
+                        print(f"Response status: {response.status}")
+                    
+                    # Try to wait for some content
+                    try:
+                        page.wait_for_timeout(3000)  # Just wait 3 seconds
+                        if attempt == 0:
+                            wait_for_page_stable(page)
+                    except Exception as e:
+                        print(f"Page stability warning: {e}")
+                    
+                    # Take screenshot
+                    print("Taking screenshot...")
+                    page.screenshot(path=filepath)
+                    
+                    # If we get here, screenshot was successful
                     context.close()
-                except Exception as close_error:
-                    print(f"Warning: Error while closing context: {close_error}")
-        
-        if os.path.exists(filepath):
-            print(f"Screenshot saved successfully to {filepath}")
-            return filepath
-        else:
-            raise Exception("Screenshot file was not created")
+                    print(f"Screenshot saved successfully to {filepath}")
+                    return filepath
+                    
+                except Exception as nav_error:
+                    print(f"Navigation error on attempt {attempt + 1}: {nav_error}")
+                    
+                    # Try to take screenshot of whatever we have
+                    try:
+                        if page.url != "about:blank":
+                            print("Taking screenshot of partial page...")
+                            page.screenshot(path=filepath)
+                            context.close()
+                            if os.path.exists(filepath):
+                                print(f"Partial screenshot saved to {filepath}")
+                                return filepath
+                    except Exception as screenshot_error:
+                        print(f"Failed to take partial screenshot: {screenshot_error}")
+                    
+                    context.close()
+                    
+                    # If this is the last attempt, raise the error
+                    if attempt == max_retries - 1:
+                        raise nav_error
+                    else:
+                        print(f"Retrying with different approach...")
+                        time.sleep(2)  # Wait before retry
+                        continue
 
-    except Exception as e:
-        print(f"[ERROR] Failed to take screenshot for URL: {url}")
-        print(f"Exception: {e}")
-        traceback.print_exc()
-        return None
+        except Exception as e:
+            print(f"[ERROR] Attempt {attempt + 1} failed: {e}")
+            if attempt == max_retries - 1:
+                print(f"All {max_retries} attempts failed for URL: {url}")
+                traceback.print_exc()
+                return None
+            else:
+                print("Waiting before next attempt...")
+                time.sleep(3)
+                continue
+    
+    return None
 
 def resize_if_needed(image_path, max_mb=1, target_width=720):
     file_size = os.path.getsize(image_path) / (1024 * 1024)  # dalam MB