app.py

import os
import json
import random
import re
import requests
import asyncio
from fastapi import FastAPI, WebSocket, HTTPException, Depends
from fastapi.responses import HTMLResponse, Response
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from threading import Lock
import uvicorn
from faker import Faker
from bs4 import BeautifulSoup
import time

app = FastAPI()
faker = Faker()
valid_proxies = {}
invalid_proxies = {}
proxies_lock = Lock()
basic_auth = HTTPBasic()

raw_proxy_urls = [
    os.getenv("PROXY_API_URL", "http://pubproxy.com/api/proxy?format=txt&level=anonymous,elite&type=http,socks4,socks5&last_check=60&speed=25&limit=1&post=true&user_agent=true&cookies=true&referer=true"),
    'https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt',
    'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',
    'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt',
    'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt',
    'https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/proxy.txt',
    'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',
    "https://storage.googleapis.com/river-treat-249913.appspot.com/p.txt",
    "https://storage.googleapis.com/river-treat-249913.appspot.com/proxy.txt",
    "https://storage.googleapis.com/river-treat-249913.appspot.com/ultimate.txt",
    'https://raw.githubusercontent.com/proxylist/proxylist/master/proxy.txt',
    'https://raw.githubusercontent.com/scrapfly/proxy-list/main/proxies.txt',
    'https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTP.txt',
    'https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4.txt',
    'https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5.txt',
    'https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS.txt',
    'https://raw.githubusercontent.com/roosterkid/openproxylist/main/ALL.txt',
    'https://raw.githubusercontent.com/proxylist/proxylist/master/https.txt',
    'https://raw.githubusercontent.com/proxylist/proxylist/master/socks4.txt',
    'https://raw.githubusercontent.com/proxylist/proxylist/master/socks5.txt',
    'https://raw.githubusercontent.com/proxylist/proxylist/master/http.txt',
    'https://raw.githubusercontent.com/proxylist/proxylist/master/all.txt',
    'https://raw.githubusercontent.com/jetlore/proxies/master/proxy-list.txt',
    'https://raw.githubusercontent.com/hookzof/proxy-list/main/proxy.txt',
    'https://raw.githubusercontent.com/zzlol123/proxy-list/main/proxies.txt',
    'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/http.txt',
    'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/https.txt',
    'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/socks4.txt',
    'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/socks5.txt',
    'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/all.txt',
    'https://www.proxy-list.download/api/v1/get?type=https',
    'https://www.proxy-list.download/api/v1/get?type=http',
    'https://www.proxy-list.download/api/v1/get?type=socks4',
    'https://www.proxy-list.download/api/v1/get?type=socks5',
    'https://www.proxy-list.download/api/v1/get?type=all',
    'https://www.sslproxies.org/',
    'https://www.us-proxy.org/',
    'https://free-proxy-list.net/',
    'https://www.proxy-list.download/',
    'https://www.proxy-list.org/eng/proxylist.txt',
    "https://api.proxyscrape.com/v2/?request=displayproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all",
    'https://api.openproxylist.xyz/http.txt',
    'https://api.proxyscrape.com/?request=displayproxies&proxytype=http&timeout=10000',
    'https://proxylist.geonode.com/api/proxy-list?limit=500&page=1&sort_by=lastChecked&sort_type=desc',
    'https://proxyocean.net/api/v1/গেট-প্রক্সি?api_key=demo',
    'https://www.freeproxychecker.com/result/?protocol=http&anonymity=elite&anonymity=anonymous',
    'https://premproxy.com/proxy-by-country/GB.htm',
    'https://premproxy.com/list/time-01.htm',
    'https://www.my-proxy.com/free-proxy-list.html',
    'https://proxyservers.pro/proxy/list/protocol/http',
    'https://proxyservers.pro/proxy/list/protocol/socks4',
    'https://proxyservers.pro/proxy/list/protocol/socks5',
    'https://spys.me/proxy.txt',
    'https://www.proxynova.com/proxy-server-list/anonymous-proxies/',
    'https://www.proxynova.com/proxy-server-list/elite-proxies/',
    'https://www.proxy-daily.com/https_proxies.txt',
    'https://www.proxy-daily.com/http_proxies.txt',
    'https://www.proxy-daily.com/socks4_proxies.txt',
    'https://www.proxy-daily.com/socks5_proxies.txt',
    'https://free-proxy-list.net/anonymous-proxy.html',
    'https://www.ip-adress.com/proxy-list/',
    'https://hidemy.name/en/proxy-list/',
    'https://www.kproxy.com/en/proxylist.html',
    'https://www.megaproxy.com/frees proxies/',
    'https://proxy-store.com/free-proxy-list',
    'https://proxylists.net/http_anonymous_proxylist.txt',
    'https://proxylists.net/http_elite_proxylist.txt',
    'https://proxylists.net/socks4_proxylist.txt',
    'https://proxylists.net/socks5_proxylist.txt',
    'http://www.httptunnel.ge/ProxyListForFree.aspx',
    'https://open-proxy.net/anonymous-proxies',
    'https://open-proxy.net/elite-proxies',
    'https://premproxy.com/socks-proxy/',
    'https://www.socks-proxy.net/',
    'https://www.freeproxylists.net/',
    'https://www.proxies.net/',
    'https://www.proxy-rack.com/free-proxy-list/',
    'https://www.vipsocks24.net/proxylist-http',
    'https://www.vipsocks24.net/proxylist-socks4',
    'https://www.vipsocks24.net/proxylist-socks5',
    'https://www.cool-proxy.net/proxies/http_proxy_list',
    'https://www.cool-proxy.net/proxies/socks4_proxy_list',
    'https://www.cool-proxy.net/proxies/socks5_proxy_list',
    'https://proxygather.com/proxylist/anonymity/anonymous',
    'https://proxygather.com/proxylist/anonymity/elite',
    'https://www.preproxy.com/proxy-servers',
    'https://www.preproxy.com/socks5-proxies',
    'https://www.preproxy.com/socks4-proxies',
    'https://www.proxy-listen.de/en/proxy-server-list/http.html',
    'https://www.proxy-listen.de/en/proxy-server-list/https.html',
    'https://www.proxy-listen.de/en/proxy-server-list/socks4.html',
    'https://www.proxy-listen.de/en/proxy-server-list/socks5.html',
    'https://www.freeproxylists.net/en/http_proxies.html',
    'https://www.freeproxylists.net/en/https_proxies.html',
    'https://www.freeproxylists.net/en/socks4_proxies.html',
    'https://www.freeproxylists.net/en/socks5_proxies.html',
    'https://www.xroxy.com/proxylist.htm',
    'https://proxylist.me/api/v1/getProxy?anon=elite',
    'https://proxylist.me/api/v1/getProxy?anon=anonymous',
    'https://proxylist.me/api/v1/getProxy?anon=transparent',
    'https://proxylist.me/api/v1/getProxy?type=http',
    'https://proxylist.me/api/v1/getProxy?type=https',
    'https://proxylist.me/api/v1/getProxy?type=socks4',
    'https://proxylist.me/api/v1/getProxy?type=socks5',
    'https://www.proxyscan.io/api/proxy?type=http',
    'https://www.proxyscan.io/api/proxy?type=https',
    'https://www.proxyscan.io/api/proxy?type=socks4',
    'https://www.proxyscan.io/api/proxy?type=socks5',
    'https://www.proxyscan.io/api/proxy?last_check=60',
    'https://openproxies.pl/proxylist.txt',
    'https://openproxies.pl/socks4.txt',
    'https://openproxies.pl/socks5.txt',
    'https://www.gatherproxy.com/zh/proxylist/country/?c=United%20States',
    'https://www.free-proxy-cz.com/en/proxylist/country/all/http/ping/all',
    'https://www.fre-proxy.ru/en/proxy/country/ru',
    'https://www.proxydocker.com/en/proxylist/country/all',
    'https://proxydb.net/?protocol=http&anonlvl=4',
    'https://proxydb.net/?protocol=socks4&anonlvl=4',
    'https://proxydb.net/?protocol=socks5&anonlvl=4',
    'https://www.ip-tracker.org/proxies/https-proxy-list.php',
    'https://www.ip-tracker.org/proxies/socks4-proxy-list.php',
    'https://www.ip-tracker.org/proxies/socks5-proxy-list.php',
    'https://www.proxy-server-list.com/country/us',
    'https://www.7proxies.net/index.php/proxy-list/proxies.html',
    'https://www.best-proxy.net/en/country/code/',
    'https://www.haiproxy.com/en/free-proxy-list/',
    'https://www.bonanza.com/items/like/228248525/VPN-Proxy-List-100000-Fresh-Daily-Updated-Private-Proxies-HTTP-SOCKS-Proxy',
    'https://www.megaproxylist.net/anonymous-proxy-list.html',
    'https://www.megaproxylist.net/elite-proxy-list.html',
    'https://www.proxy-ipv4.net/en/proxy-list-anonymous.html',
    'https://www.proxy-ipv4.net/en/proxy-list-elite-anonymous.html',
    'https://www.proxiesforfree.com/proxies/http_proxies.html',
    'https://www.proxiesforfree.com/proxies/socks_proxies.html',
    'https://www.proxy-list.download/HTTPS',
    'https://www.proxy-list.download/HTTP',
    'https://www.proxy-list.download/SOCKS4',
    'https://www.proxy-list.download/SOCKS5',
    'https://www.freeproxyme.org/proxy-list.html',
    'https://proxy-hub.com/en/us-anonymous-proxy-list.html',
    'https://www.ipaddress.com/proxy-list/',
    'https://www.proxylists.net/http.txt',
    'https://www.proxylists.net/socks4.txt',
    'https://www.proxylists.net/socks5.txt',
    'https://www.proxy-my-ip.com/free-proxy-list.html',
    'https://www.proxy-german.de/proxy-list/',
    'https://proxylist.cc/free-proxy-list/',
    'https://www.proxy-checker.net/proxy_list.php?type=https',
    'https://www.proxy-checker.net/proxy_list.php?type=http',
    'https://www.proxydrop.com/en/us-proxy-list/',
    'https://www.proxydrop.com/en/elite-proxy-list/',
    'https://www.proxydrop.com/en/anonymous-proxy-list/',
    'https://www.proxy-store.com/proxylist/country/us',
    'https://www.proxy-israel.com/en/proxy-list/',
    'https://www.proxies-socks5.net/proxy-lists/country-us',
    'https://www.proxies-socks5.net/proxy-lists/country-de',
    'https://www.proxies-socks5.net/proxy-lists/type-socks5',
    'https://www.proxies-socks5.net/proxy-lists/type-socks4',
    'https://www.proxy-list.org/en/socks-list/country-US.html',
    'https://www.proxy-ipv6.net/index_anonymous.html',
    'https://www.proxy-ipv6.net/index_elite.html',
    'https://www.proxy-ipv6.net/index_transparent.html',
    'https://www.proxy-ipv6.net/index_ssl.html',
    'https://www.proxies-socks5.net/proxy-lists/anonymous-elite',
    'https://www.proxies-socks5.net/proxy-lists/anonymous',
    'https://www.proxies-socks5.net/proxy-lists/transparent',
    'https://www.proxynow.top/proxy-list-anonymous',
    'https://www.proxynow.top/proxy-list-elite',
    'https://www.proxy-server.net/proxies/anonymous-elite-proxy-list.aspx',
    'https://www.proxy-server.net/proxies/anonymous-proxy-list.aspx',
    'https://www.proxy-server.net/proxies/transparent-proxy-list.aspx',
    'https://www.proxydaddy.com/proxylist.php?protocol=HTTP',
    'https://www.proxydaddy.com/proxylist.php?protocol=HTTPS',
    'https://www.proxydaddy.com/proxylist.php?protocol=SOCKS4',
    'https://www.proxydaddy.com/proxylist.php?protocol=SOCKS5',
    'https://www.proxy4free.com/list/webproxy_anon.html',
    'https://www.proxy4free.com/list/webproxy_elite.html',
    'https://www.freeproxylist.co/anonymous.html',
    'https://www.freeproxylist.co/elite.html',
    'https://www.proxylisty.com/anonymous-proxies',
    'https://www.proxylisty.com/elite-proxies',
    'https://www.aliveproxy.com/anonymous-proxy-list.aspx',
    'https://www.aliveproxy.com/elite-proxy-list.aspx',
    'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/https.php',
    'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/http.php',
    'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/socks5.php',
    'https://proxyservers.net/proxy-list/sort/country/order/asc',
    'https://www.ip2location.com/free/proxy-list',
    'https://www.yougetproxy.com/zh-cn/https-proxy-list/',
    'https://www.yougetproxy.com/zh-cn/http-proxy-list/',
    'https://www.ip-finder.net/en/proxylist/anonymous-proxies',
    'https://www.ip-finder.net/en/proxylist/elite-proxies',
    'https://www.rapidunblock.com/proxy-list/',
    'https://www.privateproxyguide.com/anonymous-proxy-list/',
    'https://www.privateproxyguide.com/elite-proxy-list/',
    'https://www.proxy-german.de/',
    'https://www.proxy-italia.it/proxy-list/',
    'https://www.proxylistchecker.pro/https_proxies.txt',
    'https://www.proxylistchecker.pro/http_proxies.txt',
    'https://www.proxylistchecker.pro/socks4_proxies.txt',
    'https://www.proxylistchecker.pro/socks5_proxies.txt',
    'https://www.proxy-list.org/en/socks-list-1.html',
    'https://www.proxy-list.org/en/https-anonymous-elite-proxy-list-1.html',
    'https://www.proxy-list.org/en/https-anonymous-proxy-list-1.html',
    'https://www.proxy-list.org/en/http-anonymous-elite-proxy-list-1.html',
    'https://www.proxy-list.org/en/http-anonymous-proxy-list-1.html',
    'https://www.proxyserverlist24.top/proxylists/http',
    'https://www.proxyserverlist24.top/proxylists/https',
    'https://www.proxyserverlist24.top/proxylists/socks4',
    'https://www.proxyserverlist24.top/proxylists/socks5',
    'https://www.proxy-ipv4.net/',
    'https://www.proxy-ipv6.net/',
    'https://www.proxy-list.org/',
    'https://www.proxyserverlist24.top/',
    'https://www.freeproxy.world/',
    'https://www.free-proxy-server.net/',
    'https://www.proxylists.me/',
    'https://www.proxy-directory.com/',
    'https://www.proxy-provider.net/',
    'https://www.proxy-server.net/',
    'https://www.proxy-list.me/',
    'https://www.proxy-daily.com/',
    'https://www.proxy-store.com/',
    'https://www.proxy-israel.com/',
    'https://www.proxy-german.de/',
    'https://www.proxy-italia.it/',
    'https://www.proxynow.top/',
    'https://www.proxydaddy.com/',
    'https://www.proxy4free.com/',
    'https://www.freeproxylist.co/',
    'https://www.proxylisty.com/',
    'https://www.aliveproxy.com/',
    'https://proxyscrape.com/free-proxy-list',
    'https://www.goldproxy.jp/proxylist/anonymous.html',
    'https://www.goldproxy.jp/proxylist/country/US.html',
    'https://www.goldproxy.jp/proxylist/elite.html',
    'https://www.ip-port.net/free-proxy-anonymous-server-list',
    'https://www.ip-port.net/free-proxy-elite-server-list',
    'https://www.ip-port.net/free-proxy-transparent-server-list',
    'https://www.gatherproxy.com/',
    'https://incloak.com/proxy-list/',
    'https://proxyservers.pro/',
    'https://www.proxy-anonymous.com/en/us-free-proxy-list.shtml',
    'https://www.proxy-anonymous.com/en/country_US-01.shtml',
    'https://www.proxy-anonymous.com/en/http-ssl-anonymous-proxy-list.shtml',
    'https://www.proxy-anonymous.com/en/socks-anonymous-proxy-list.shtml',
    'https://www.checkerproxy.net/api/anon/http',
    'https://www.checkerproxy.net/api/anon/https',
    'https://www.checkerproxy.net/api/anon/socks4',
    'https://www.checkerproxy.net/api/anon/socks5',
    'https://www.proxyscout.io/en/https',
    'https://www.proxyscout.io/en/socks4',
    'https://www.proxyscout.io/en/socks5',
    'https://www.proxy-list.download/SOCKS',
    'https://www.proxy-list.org/english/index.php?p=http',
    'https://www.proxy-list.org/english/index.php?p=https',
    'https://www.proxy-list.org/english/index.php?p=socks4',
    'https://www.proxy-list.org/english/index.php?p=socks5',
    'https://proxylists.net/',
    'https://www.rapidunblock.com/',
    'https://www.privateproxyguide.com/',
    'https://www.proxy-directory.com/proxylists.php',
    'https://www.proxy-provider.net/proxylists.php',
    'https://www.proxy-server.net/proxylists.php',
    'https://www.proxy-list.me/proxylists.php',
    'https://www.proxy-daily.com/proxylists.php',
    'https://www.proxy-store.com/proxylists.php',
    'https://www.proxy-israel.com/proxylists.php',
    'https://www.proxy-german.de/proxylists.php',
    'https://www.proxy-italia.it/proxylists.php',
    'https://www.proxynow.top/proxylists.php',
    'https://www.proxydaddy.com/proxylists.php',
    'https://www.proxy4free.com/proxylists.php',
    'https://www.freeproxylist.co/proxylists.php',
    'https://www.proxylisty.com/proxylists.php',
    'https://www.aliveproxy.com/proxylists.aspx',
    'https://www.proxyscrape.com/proxylists.php',
    'https://www.goldproxy.jp/proxylists.php',
    'https://www.ip-port.net/proxylists.php',
    'https://www.gatherproxy.com/proxylists.php',
    'https://incloak.com/proxylists.php',
    'https://www.proxyservers.pro/proxylists.php',
    'https://www.proxy-anonymous.com/proxylists.php',
    'https://www.checkerproxy.net/proxylists.php',
    'https://www.proxyscout.io/proxylists.php',
    'https://www.proxy-list.download/proxylists.php',
    'https://www.proxy-list.org/proxylists.php',
    'https://www.proxyserverlist24.top/proxylists.php',
    'https://www.freeproxy.world/proxylists.php',
    'https://www.free-proxy-server.net/proxylists.php',
    'https://www.proxylists.me/proxylists.php',
    'https://www.ip2location.com/proxylists.php',
    'https://www.yougetproxy.com/proxylists.php',
    'https://www.ip-finder.net/proxylists.php',
    'https://www.freeproxylists.net/proxylists.php',
    'https://www.xroxy.com/proxylists.php',
    'https://www.sslproxies.org/proxylists.php',
    'https://www.us-proxy.org/proxylists.php',
    'https://www.socks-proxy.net/proxylists.php',
    'https://www.httptunnel.ge/proxylists.php',
    'https://open-proxy.net/proxylists.php',
    'https://premproxy.com/proxylists.php',
    'https://www.vipsocks24.net/proxylists.php',
    'https://www.cool-proxy.net/proxylists.php',
]

all_proxy_urls = list(set(raw_proxy_urls))
print(f"Total unique proxy URLs after deduplication: {len(all_proxy_urls)}")

def create_headers():
    user_agent = faker.user_agent()
    random_ip = faker.ipv4()
    return {"User-Agent": user_agent, "X-Forwarded-For": random_ip, "Client-IP": random_ip, "X-Real-IP": random_ip}

def is_valid_proxy(proxy: str) -> bool:
    ip_port_pattern = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5}$')
    if not ip_port_pattern.match(proxy):
        return False
    try:
        ip, port_str = proxy.split(':')
        port = int(port_str)
        if not (0 < port < 65536):
            return False
        parts = ip.split('.')
        if len(parts) != 4:
            return False
        for part in parts:
            if not (0 <= int(part) <= 255):
                return False
    except ValueError:
        return False
    return True


async def verify_proxy(proxy: str):
    if not is_valid_proxy(proxy):
        with proxies_lock:
            invalid_proxies[proxy] = True
        return proxy, False

    test_urls = ["https://google.com", "http://httpbin.org/ip"]
    proxy_dict = {"http": f"http://{proxy}", "https": f"http://{proxy}"}
    headers = create_headers()

    success = False
    try:
        for url in test_urls:
            try:
                response = await asyncio.to_thread(requests.get, url, proxies=proxy_dict, headers=headers, timeout=10)
                response.raise_for_status()
                success = True
                break
            except (requests.RequestException, ValueError):
                continue

        if success:
             with proxies_lock:
                valid_proxies[proxy] = True
                invalid_proxies.pop(proxy, None)
             return proxy, True
        else:
            with proxies_lock:
                valid_proxies.pop(proxy, None)
                invalid_proxies[proxy] = True
            return proxy, False

    except Exception:
        with proxies_lock:
            valid_proxies.pop(proxy, None)
            invalid_proxies[proxy] = True
        return proxy, False

async def verify_proxies_in_background():
    while True:
        proxies_to_verify = []
        with proxies_lock:
            if valid_proxies:
                proxies_to_verify = list(valid_proxies.keys())
            else:
                if invalid_proxies:
                    sample_size = min(len(invalid_proxies), 50)
                    proxies_to_verify = random.sample(list(invalid_proxies.keys()), sample_size)


        if proxies_to_verify:
            tasks = [verify_proxy(proxy) for proxy in proxies_to_verify]
            await asyncio.gather(*tasks)

        await asyncio.sleep(60)

async def fetch_proxies_from_sources():
    proxies = set()
    for url in all_proxy_urls:
        response_text = fetch_response(url)
        if response_text:
            found_in_source = set()
            try:
                data = json.loads(response_text)
                if isinstance(data, list):
                    for item in data:
                        if isinstance(item, str) and ':' in item:
                             found_in_source.add(item.strip())
                        elif isinstance(item, dict):
                             ip = item.get('ip') or item.get('Ip') or item.get('IP') or item.get('proxy') or item.get('addr') or item.get('PX')
                             port = item.get('port') or item.get('Port')
                             if ip and port:
                                 found_in_source.add(f"{ip}:{port}")

                elif isinstance(data, dict):
                    if 'data' in data and isinstance(data['data'], list):
                        for item in data['data']:
                            if isinstance(item, dict):
                                ip = item.get('ip') or item.get('proxy')
                                port = item.get('port')
                                if ip and port:
                                    found_in_source.add(f"{ip}:{port}")
                                elif 'ipPort' in item:
                                     found_in_source.add(item['ipPort'])
                    elif 'proxies' in data and isinstance(data['proxies'], list):
                         for item in data['proxies']:
                             if isinstance(item, dict):
                                 ip = item.get('ip') or item.get('addr')
                                 port = item.get('port')
                                 if ip and port:
                                     found_in_source.add(f"{ip}:{port}")
                             elif isinstance(item, str) and ':' in item:
                                  found_in_source.add(item.strip())
                    elif 'PXs' in data and isinstance(data['PXs'], list):
                         for item in data['PXs']:
                             if isinstance(item, dict):
                                 ip = item.get('PX')
                                 port = item.get('Port')
                                 if ip and port:
                                     found_in_source.add(f"{ip}:{port}")


            except (json.JSONDecodeError, ValueError, KeyError, TypeError, IndexError):
                 pass

            try:
                soup = BeautifulSoup(response_text, 'html.parser')
                tables = soup.find_all('table')
                for table in tables:
                     rows = table.find_all('tr')
                     for row in rows[1:]:
                         cols = row.find_all(['td', 'th'])
                         if len(cols) >= 2:
                             ip = cols[0].get_text(strip=True)
                             port = cols[1].get_text(strip=True)
                             if re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', ip) and port.isdigit():
                                 found_in_source.add(f"{ip}:{port}")

                textareas = soup.find_all('textarea')
                for textarea in textareas:
                    lines = textarea.get_text().splitlines()
                    for line in lines:
                        line = line.strip()
                        if is_valid_proxy(line):
                            found_in_source.add(line)

                pres = soup.find_all('pre')
                for pre in pres:
                    lines = pre.get_text().splitlines()
                    for line in lines:
                        line = line.strip()
                        if is_valid_proxy(line):
                            found_in_source.add(line)

                found_in_source.update(re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5})', response_text))

            except Exception:
                pass

            lines = response_text.splitlines()
            for line in lines:
                line = line.strip()
                parts = re.split(r'[:\s]+', line)
                if len(parts) >= 2 and re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', parts[0]) and parts[1].isdigit():
                    found_in_source.add(f"{parts[0]}:{parts[1]}")
                elif is_valid_proxy(line):
                    found_in_source.add(line)


            for proxy in found_in_source:
                 if is_valid_proxy(proxy):
                     proxies.add(proxy)

        await asyncio.sleep(1)

    print(f"Fetched {len(proxies)} potential proxies from sources.")

    initial_verification_tasks = [verify_proxy(p) for p in proxies if p not in valid_proxies and p not in invalid_proxies]
    if initial_verification_tasks:
        print(f"Performing initial verification for {len(initial_verification_tasks)} new proxies.")
        await asyncio.gather(*initial_verification_tasks)
        print("Initial verification complete.")

    with proxies_lock:
        print(f"Total valid proxies: {len(valid_proxies)}, Total invalid proxies: {len(invalid_proxies)}")
        return valid_proxies, invalid_proxies


@app.on_event("startup")
async def on_startup():
    global valid_proxies, invalid_proxies
    print("Starting proxy fetch and initial verification...")
    await fetch_proxies_from_sources()
    print("Initial fetch and verification complete. Starting background verification task.")
    asyncio.create_task(verify_proxies_in_background())


def fetch_response(url):
    headers = create_headers()
    try:
        response = requests.get(url, headers=headers, timeout=20)
        response.raise_for_status()
        try:
            return response.content.decode('utf-8')
        except UnicodeDecodeError:
            try:
                return response.content.decode('iso-8859-1')
            except UnicodeDecodeError:
                return response.text
    except requests.RequestException as e:
        print(f"Failed to fetch URL: {url} - Error: {e}")
        return None
    except Exception as e:
        print(f"An unexpected error occurred fetching {url}: {e}")
        return None


def get_current_user(credentials: HTTPBasicCredentials = Depends(basic_auth)):
    correct_username = os.environ.get("PROXY_API_USERNAME", "proxyuser")
    correct_password = os.environ.get("PROXY_API_PASSWORD", "proxypassword")

    is_user_valid = credentials.username == correct_username
    is_pass_valid = credentials.password == correct_password

    if not (is_user_valid and is_pass_valid):
        raise HTTPException(
            status_code=401,
            detail="Incorrect username or password",
            headers={"WWW-Authenticate": "Basic"},
        )
    return credentials.username

@app.get("/")
async def root():
    html_content = """
    <!DOCTYPE html>
    <html>
    <head>
        <title>Proxy Service</title>
        <style>
            body { font-family: sans-serif; padding: 20px; }
            h1 { color: #333; }
            ul { list-style: none; padding: 0; }
            li { margin-bottom: 10px; }
            a { text-decoration: none; color: #007bff; }
            a:hover { text-decoration: underline; }
            .endpoint { background-color: #f8f9fa; border: 1px solid #dee2e6; border-radius: 4px; padding: 15px; margin-bottom:15px; }
            code { background-color: #e9ecef; padding: 2px 4px; border-radius: 3px; }
        </style>
    </head>
    <body>
        <h1>Proxy List Service</h1>
        <p>This service scrapes and verifies public proxies.</p>

        <h2>Available Endpoints:</h2>

        <div class="endpoint">
            <strong>GET /valid</strong>
            <p>Returns a sample list (up to 50) of currently valid proxies in plain text format.</p>
            <p>Example: <a href="/valid" target="_blank">/valid</a></p>
        </div>

         <div class="endpoint">
            <strong>GET /all-valid</strong>
            <p>Returns the complete list of currently valid proxies in plain text format.</p>
            <p>Example: <a href="/all-valid" target="_blank">/all-valid</a></p>
        </div>

        <div class="endpoint">
            <strong>GET /valid-auth</strong>
            <p>Returns a sample list (up to 50) of currently valid proxies in plain text format. Requires Basic Authentication.</p>
            <p>Credentials are set via environment variables <code>PROXY_API_USERNAME</code> and <code>PROXY_API_PASSWORD</code> (defaults: proxyuser/proxypassword).</p>
            <p>Example: <a href="/valid-auth" target="_blank">/valid-auth</a> (will prompt for credentials)</p>
        </div>

        <div class="endpoint">
            <strong>GET /invalid</strong>
             <p>Returns a sample list (up to 50) of proxies that recently failed verification, in plain text format.</p>
            <p>Example: <a href="/invalid" target="_blank">/invalid</a></p>
        </div>

        <div class="endpoint">
            <strong>GET /stats</strong>
             <p>Returns the current count of valid and invalid proxies in JSON format.</p>
            <p>Example: <a href="/stats" target="_blank">/stats</a></p>
        </div>

        <div class="endpoint">
            <strong>WebSocket /ws</strong>
            <p>Streams the current list of valid proxies. The list is sent periodically.</p>
            <p>Use a WebSocket client to connect to <code>ws://your-server-address/ws</code>.</p>
        </div>

    </body>
    </html>
    """
    return HTMLResponse(content=html_content, status_code=200)

@app.get("/valid")
async def valid_proxies_endpoint():
    with proxies_lock:
        valid_proxies_list = list(valid_proxies.keys())
        count = len(valid_proxies_list)
        if count > 50:
             sample_list = random.sample(valid_proxies_list, 50)
        else:
             sample_list = valid_proxies_list
    response_content = "\n".join(sample_list)
    return Response(content=response_content, media_type='text/plain')

@app.get("/all-valid")
async def all_valid_proxies_endpoint():
    with proxies_lock:
        valid_proxies_list = list(valid_proxies.keys())
    response_content = "\n".join(valid_proxies_list)
    return Response(content=response_content, media_type='text/plain')


@app.get("/invalid")
async def invalid_proxies_endpoint():
    with proxies_lock:
        invalid_proxies_list = list(invalid_proxies.keys())
        count = len(invalid_proxies_list)
        if count > 50:
            sample_list = random.sample(invalid_proxies_list, 50)
        else:
            sample_list = invalid_proxies_list
    response_content = "\n".join(sample_list)
    return Response(content=response_content, media_type='text/plain')

@app.get("/valid-auth")
async def valid_proxies_auth_endpoint(username: str = Depends(get_current_user)):
    with proxies_lock:
        valid_proxies_list = list(valid_proxies.keys())
        count = len(valid_proxies_list)
        if count > 50:
            sample_list = random.sample(valid_proxies_list, 50)
        else:
            sample_list = valid_proxies_list
    response_content = "\n".join(sample_list)
    return Response(content=response_content, media_type='text/plain')

@app.get("/stats")
async def get_stats():
    with proxies_lock:
        stats = {
            "valid_count": len(valid_proxies),
            "invalid_count": len(invalid_proxies)
        }
    return stats

@app.websocket("/ws")
async def websocket_endpoint(websocket: WebSocket):
    await websocket.accept()
    try:
        while True:
            with proxies_lock:
                proxies_list = list(valid_proxies.keys())
            await websocket.send_text("\n".join(proxies_list))
            await asyncio.sleep(10)
    except Exception as e:
        print(f"WebSocket Error: {e}")
    finally:
        print("WebSocket connection closed")


if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port="7860") # Changed this line