import os import json import random import re import requests import asyncio from fastapi import FastAPI, WebSocket, HTTPException, Depends from fastapi.responses import HTMLResponse, Response from fastapi.security import HTTPBasic, HTTPBasicCredentials from threading import Lock import uvicorn from faker import Faker from bs4 import BeautifulSoup import time app = FastAPI() faker = Faker() valid_proxies = {} invalid_proxies = {} proxies_lock = Lock() basic_auth = HTTPBasic() raw_proxy_urls = [ os.getenv("PROXY_API_URL", "http://pubproxy.com/api/proxy?format=txt&level=anonymous,elite&type=http,socks4,socks5&last_check=60&speed=25&limit=1&post=true&user_agent=true&cookies=true&referer=true"), 'https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt', 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt', 'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt', 'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt', 'https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/proxy.txt', 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt', "https://storage.googleapis.com/river-treat-249913.appspot.com/p.txt", "https://storage.googleapis.com/river-treat-249913.appspot.com/proxy.txt", "https://storage.googleapis.com/river-treat-249913.appspot.com/ultimate.txt", 'https://raw.githubusercontent.com/proxylist/proxylist/master/proxy.txt', 'https://raw.githubusercontent.com/scrapfly/proxy-list/main/proxies.txt', 'https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTP.txt', 'https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4.txt', 'https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5.txt', 'https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS.txt', 'https://raw.githubusercontent.com/roosterkid/openproxylist/main/ALL.txt', 'https://raw.githubusercontent.com/proxylist/proxylist/master/https.txt', 'https://raw.githubusercontent.com/proxylist/proxylist/master/socks4.txt', 'https://raw.githubusercontent.com/proxylist/proxylist/master/socks5.txt', 'https://raw.githubusercontent.com/proxylist/proxylist/master/http.txt', 'https://raw.githubusercontent.com/proxylist/proxylist/master/all.txt', 'https://raw.githubusercontent.com/jetlore/proxies/master/proxy-list.txt', 'https://raw.githubusercontent.com/hookzof/proxy-list/main/proxy.txt', 'https://raw.githubusercontent.com/zzlol123/proxy-list/main/proxies.txt', 'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/http.txt', 'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/https.txt', 'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/socks4.txt', 'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/socks5.txt', 'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/all.txt', 'https://www.proxy-list.download/api/v1/get?type=https', 'https://www.proxy-list.download/api/v1/get?type=http', 'https://www.proxy-list.download/api/v1/get?type=socks4', 'https://www.proxy-list.download/api/v1/get?type=socks5', 'https://www.proxy-list.download/api/v1/get?type=all', 'https://www.sslproxies.org/', 'https://www.us-proxy.org/', 'https://free-proxy-list.net/', 'https://www.proxy-list.download/', 'https://www.proxy-list.org/eng/proxylist.txt', "https://api.proxyscrape.com/v2/?request=displayproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all", 'https://api.openproxylist.xyz/http.txt', 'https://api.proxyscrape.com/?request=displayproxies&proxytype=http&timeout=10000', 'https://proxylist.geonode.com/api/proxy-list?limit=500&page=1&sort_by=lastChecked&sort_type=desc', 'https://proxyocean.net/api/v1/গেট-প্রক্সি?api_key=demo', 'https://www.freeproxychecker.com/result/?protocol=http&anonymity=elite&anonymity=anonymous', 'https://premproxy.com/proxy-by-country/GB.htm', 'https://premproxy.com/list/time-01.htm', 'https://www.my-proxy.com/free-proxy-list.html', 'https://proxyservers.pro/proxy/list/protocol/http', 'https://proxyservers.pro/proxy/list/protocol/socks4', 'https://proxyservers.pro/proxy/list/protocol/socks5', 'https://spys.me/proxy.txt', 'https://www.proxynova.com/proxy-server-list/anonymous-proxies/', 'https://www.proxynova.com/proxy-server-list/elite-proxies/', 'https://www.proxy-daily.com/https_proxies.txt', 'https://www.proxy-daily.com/http_proxies.txt', 'https://www.proxy-daily.com/socks4_proxies.txt', 'https://www.proxy-daily.com/socks5_proxies.txt', 'https://free-proxy-list.net/anonymous-proxy.html', 'https://www.ip-adress.com/proxy-list/', 'https://hidemy.name/en/proxy-list/', 'https://www.kproxy.com/en/proxylist.html', 'https://www.megaproxy.com/frees proxies/', 'https://proxy-store.com/free-proxy-list', 'https://proxylists.net/http_anonymous_proxylist.txt', 'https://proxylists.net/http_elite_proxylist.txt', 'https://proxylists.net/socks4_proxylist.txt', 'https://proxylists.net/socks5_proxylist.txt', 'http://www.httptunnel.ge/ProxyListForFree.aspx', 'https://open-proxy.net/anonymous-proxies', 'https://open-proxy.net/elite-proxies', 'https://premproxy.com/socks-proxy/', 'https://www.socks-proxy.net/', 'https://www.freeproxylists.net/', 'https://www.proxies.net/', 'https://www.proxy-rack.com/free-proxy-list/', 'https://www.vipsocks24.net/proxylist-http', 'https://www.vipsocks24.net/proxylist-socks4', 'https://www.vipsocks24.net/proxylist-socks5', 'https://www.cool-proxy.net/proxies/http_proxy_list', 'https://www.cool-proxy.net/proxies/socks4_proxy_list', 'https://www.cool-proxy.net/proxies/socks5_proxy_list', 'https://proxygather.com/proxylist/anonymity/anonymous', 'https://proxygather.com/proxylist/anonymity/elite', 'https://www.preproxy.com/proxy-servers', 'https://www.preproxy.com/socks5-proxies', 'https://www.preproxy.com/socks4-proxies', 'https://www.proxy-listen.de/en/proxy-server-list/http.html', 'https://www.proxy-listen.de/en/proxy-server-list/https.html', 'https://www.proxy-listen.de/en/proxy-server-list/socks4.html', 'https://www.proxy-listen.de/en/proxy-server-list/socks5.html', 'https://www.freeproxylists.net/en/http_proxies.html', 'https://www.freeproxylists.net/en/https_proxies.html', 'https://www.freeproxylists.net/en/socks4_proxies.html', 'https://www.freeproxylists.net/en/socks5_proxies.html', 'https://www.xroxy.com/proxylist.htm', 'https://proxylist.me/api/v1/getProxy?anon=elite', 'https://proxylist.me/api/v1/getProxy?anon=anonymous', 'https://proxylist.me/api/v1/getProxy?anon=transparent', 'https://proxylist.me/api/v1/getProxy?type=http', 'https://proxylist.me/api/v1/getProxy?type=https', 'https://proxylist.me/api/v1/getProxy?type=socks4', 'https://proxylist.me/api/v1/getProxy?type=socks5', 'https://www.proxyscan.io/api/proxy?type=http', 'https://www.proxyscan.io/api/proxy?type=https', 'https://www.proxyscan.io/api/proxy?type=socks4', 'https://www.proxyscan.io/api/proxy?type=socks5', 'https://www.proxyscan.io/api/proxy?last_check=60', 'https://openproxies.pl/proxylist.txt', 'https://openproxies.pl/socks4.txt', 'https://openproxies.pl/socks5.txt', 'https://www.gatherproxy.com/zh/proxylist/country/?c=United%20States', 'https://www.free-proxy-cz.com/en/proxylist/country/all/http/ping/all', 'https://www.fre-proxy.ru/en/proxy/country/ru', 'https://www.proxydocker.com/en/proxylist/country/all', 'https://proxydb.net/?protocol=http&anonlvl=4', 'https://proxydb.net/?protocol=socks4&anonlvl=4', 'https://proxydb.net/?protocol=socks5&anonlvl=4', 'https://www.ip-tracker.org/proxies/https-proxy-list.php', 'https://www.ip-tracker.org/proxies/socks4-proxy-list.php', 'https://www.ip-tracker.org/proxies/socks5-proxy-list.php', 'https://www.proxy-server-list.com/country/us', 'https://www.7proxies.net/index.php/proxy-list/proxies.html', 'https://www.best-proxy.net/en/country/code/', 'https://www.haiproxy.com/en/free-proxy-list/', 'https://www.bonanza.com/items/like/228248525/VPN-Proxy-List-100000-Fresh-Daily-Updated-Private-Proxies-HTTP-SOCKS-Proxy', 'https://www.megaproxylist.net/anonymous-proxy-list.html', 'https://www.megaproxylist.net/elite-proxy-list.html', 'https://www.proxy-ipv4.net/en/proxy-list-anonymous.html', 'https://www.proxy-ipv4.net/en/proxy-list-elite-anonymous.html', 'https://www.proxiesforfree.com/proxies/http_proxies.html', 'https://www.proxiesforfree.com/proxies/socks_proxies.html', 'https://www.proxy-list.download/HTTPS', 'https://www.proxy-list.download/HTTP', 'https://www.proxy-list.download/SOCKS4', 'https://www.proxy-list.download/SOCKS5', 'https://www.freeproxyme.org/proxy-list.html', 'https://proxy-hub.com/en/us-anonymous-proxy-list.html', 'https://www.ipaddress.com/proxy-list/', 'https://www.proxylists.net/http.txt', 'https://www.proxylists.net/socks4.txt', 'https://www.proxylists.net/socks5.txt', 'https://www.proxy-my-ip.com/free-proxy-list.html', 'https://www.proxy-german.de/proxy-list/', 'https://proxylist.cc/free-proxy-list/', 'https://www.proxy-checker.net/proxy_list.php?type=https', 'https://www.proxy-checker.net/proxy_list.php?type=http', 'https://www.proxydrop.com/en/us-proxy-list/', 'https://www.proxydrop.com/en/elite-proxy-list/', 'https://www.proxydrop.com/en/anonymous-proxy-list/', 'https://www.proxy-store.com/proxylist/country/us', 'https://www.proxy-israel.com/en/proxy-list/', 'https://www.proxies-socks5.net/proxy-lists/country-us', 'https://www.proxies-socks5.net/proxy-lists/country-de', 'https://www.proxies-socks5.net/proxy-lists/type-socks5', 'https://www.proxies-socks5.net/proxy-lists/type-socks4', 'https://www.proxy-list.org/en/socks-list/country-US.html', 'https://www.proxy-ipv6.net/index_anonymous.html', 'https://www.proxy-ipv6.net/index_elite.html', 'https://www.proxy-ipv6.net/index_transparent.html', 'https://www.proxy-ipv6.net/index_ssl.html', 'https://www.proxies-socks5.net/proxy-lists/anonymous-elite', 'https://www.proxies-socks5.net/proxy-lists/anonymous', 'https://www.proxies-socks5.net/proxy-lists/transparent', 'https://www.proxynow.top/proxy-list-anonymous', 'https://www.proxynow.top/proxy-list-elite', 'https://www.proxy-server.net/proxies/anonymous-elite-proxy-list.aspx', 'https://www.proxy-server.net/proxies/anonymous-proxy-list.aspx', 'https://www.proxy-server.net/proxies/transparent-proxy-list.aspx', 'https://www.proxydaddy.com/proxylist.php?protocol=HTTP', 'https://www.proxydaddy.com/proxylist.php?protocol=HTTPS', 'https://www.proxydaddy.com/proxylist.php?protocol=SOCKS4', 'https://www.proxydaddy.com/proxylist.php?protocol=SOCKS5', 'https://www.proxy4free.com/list/webproxy_anon.html', 'https://www.proxy4free.com/list/webproxy_elite.html', 'https://www.freeproxylist.co/anonymous.html', 'https://www.freeproxylist.co/elite.html', 'https://www.proxylisty.com/anonymous-proxies', 'https://www.proxylisty.com/elite-proxies', 'https://www.aliveproxy.com/anonymous-proxy-list.aspx', 'https://www.aliveproxy.com/elite-proxy-list.aspx', 'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/https.php', 'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/http.php', 'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/socks5.php', 'https://proxyservers.net/proxy-list/sort/country/order/asc', 'https://www.ip2location.com/free/proxy-list', 'https://www.yougetproxy.com/zh-cn/https-proxy-list/', 'https://www.yougetproxy.com/zh-cn/http-proxy-list/', 'https://www.ip-finder.net/en/proxylist/anonymous-proxies', 'https://www.ip-finder.net/en/proxylist/elite-proxies', 'https://www.rapidunblock.com/proxy-list/', 'https://www.privateproxyguide.com/anonymous-proxy-list/', 'https://www.privateproxyguide.com/elite-proxy-list/', 'https://www.proxy-german.de/', 'https://www.proxy-italia.it/proxy-list/', 'https://www.proxylistchecker.pro/https_proxies.txt', 'https://www.proxylistchecker.pro/http_proxies.txt', 'https://www.proxylistchecker.pro/socks4_proxies.txt', 'https://www.proxylistchecker.pro/socks5_proxies.txt', 'https://www.proxy-list.org/en/socks-list-1.html', 'https://www.proxy-list.org/en/https-anonymous-elite-proxy-list-1.html', 'https://www.proxy-list.org/en/https-anonymous-proxy-list-1.html', 'https://www.proxy-list.org/en/http-anonymous-elite-proxy-list-1.html', 'https://www.proxy-list.org/en/http-anonymous-proxy-list-1.html', 'https://www.proxyserverlist24.top/proxylists/http', 'https://www.proxyserverlist24.top/proxylists/https', 'https://www.proxyserverlist24.top/proxylists/socks4', 'https://www.proxyserverlist24.top/proxylists/socks5', 'https://www.proxy-ipv4.net/', 'https://www.proxy-ipv6.net/', 'https://www.proxy-list.org/', 'https://www.proxyserverlist24.top/', 'https://www.freeproxy.world/', 'https://www.free-proxy-server.net/', 'https://www.proxylists.me/', 'https://www.proxy-directory.com/', 'https://www.proxy-provider.net/', 'https://www.proxy-server.net/', 'https://www.proxy-list.me/', 'https://www.proxy-daily.com/', 'https://www.proxy-store.com/', 'https://www.proxy-israel.com/', 'https://www.proxy-german.de/', 'https://www.proxy-italia.it/', 'https://www.proxynow.top/', 'https://www.proxydaddy.com/', 'https://www.proxy4free.com/', 'https://www.freeproxylist.co/', 'https://www.proxylisty.com/', 'https://www.aliveproxy.com/', 'https://proxyscrape.com/free-proxy-list', 'https://www.goldproxy.jp/proxylist/anonymous.html', 'https://www.goldproxy.jp/proxylist/country/US.html', 'https://www.goldproxy.jp/proxylist/elite.html', 'https://www.ip-port.net/free-proxy-anonymous-server-list', 'https://www.ip-port.net/free-proxy-elite-server-list', 'https://www.ip-port.net/free-proxy-transparent-server-list', 'https://www.gatherproxy.com/', 'https://incloak.com/proxy-list/', 'https://proxyservers.pro/', 'https://www.proxy-anonymous.com/en/us-free-proxy-list.shtml', 'https://www.proxy-anonymous.com/en/country_US-01.shtml', 'https://www.proxy-anonymous.com/en/http-ssl-anonymous-proxy-list.shtml', 'https://www.proxy-anonymous.com/en/socks-anonymous-proxy-list.shtml', 'https://www.checkerproxy.net/api/anon/http', 'https://www.checkerproxy.net/api/anon/https', 'https://www.checkerproxy.net/api/anon/socks4', 'https://www.checkerproxy.net/api/anon/socks5', 'https://www.proxyscout.io/en/https', 'https://www.proxyscout.io/en/socks4', 'https://www.proxyscout.io/en/socks5', 'https://www.proxy-list.download/SOCKS', 'https://www.proxy-list.org/english/index.php?p=http', 'https://www.proxy-list.org/english/index.php?p=https', 'https://www.proxy-list.org/english/index.php?p=socks4', 'https://www.proxy-list.org/english/index.php?p=socks5', 'https://proxylists.net/', 'https://www.rapidunblock.com/', 'https://www.privateproxyguide.com/', 'https://www.proxy-directory.com/proxylists.php', 'https://www.proxy-provider.net/proxylists.php', 'https://www.proxy-server.net/proxylists.php', 'https://www.proxy-list.me/proxylists.php', 'https://www.proxy-daily.com/proxylists.php', 'https://www.proxy-store.com/proxylists.php', 'https://www.proxy-israel.com/proxylists.php', 'https://www.proxy-german.de/proxylists.php', 'https://www.proxy-italia.it/proxylists.php', 'https://www.proxynow.top/proxylists.php', 'https://www.proxydaddy.com/proxylists.php', 'https://www.proxy4free.com/proxylists.php', 'https://www.freeproxylist.co/proxylists.php', 'https://www.proxylisty.com/proxylists.php', 'https://www.aliveproxy.com/proxylists.aspx', 'https://www.proxyscrape.com/proxylists.php', 'https://www.goldproxy.jp/proxylists.php', 'https://www.ip-port.net/proxylists.php', 'https://www.gatherproxy.com/proxylists.php', 'https://incloak.com/proxylists.php', 'https://www.proxyservers.pro/proxylists.php', 'https://www.proxy-anonymous.com/proxylists.php', 'https://www.checkerproxy.net/proxylists.php', 'https://www.proxyscout.io/proxylists.php', 'https://www.proxy-list.download/proxylists.php', 'https://www.proxy-list.org/proxylists.php', 'https://www.proxyserverlist24.top/proxylists.php', 'https://www.freeproxy.world/proxylists.php', 'https://www.free-proxy-server.net/proxylists.php', 'https://www.proxylists.me/proxylists.php', 'https://www.ip2location.com/proxylists.php', 'https://www.yougetproxy.com/proxylists.php', 'https://www.ip-finder.net/proxylists.php', 'https://www.freeproxylists.net/proxylists.php', 'https://www.xroxy.com/proxylists.php', 'https://www.sslproxies.org/proxylists.php', 'https://www.us-proxy.org/proxylists.php', 'https://www.socks-proxy.net/proxylists.php', 'https://www.httptunnel.ge/proxylists.php', 'https://open-proxy.net/proxylists.php', 'https://premproxy.com/proxylists.php', 'https://www.vipsocks24.net/proxylists.php', 'https://www.cool-proxy.net/proxylists.php', ] all_proxy_urls = list(set(raw_proxy_urls)) print(f"Total unique proxy URLs after deduplication: {len(all_proxy_urls)}") def create_headers(): user_agent = faker.user_agent() random_ip = faker.ipv4() return {"User-Agent": user_agent, "X-Forwarded-For": random_ip, "Client-IP": random_ip, "X-Real-IP": random_ip} def is_valid_proxy(proxy: str) -> bool: ip_port_pattern = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5}$') if not ip_port_pattern.match(proxy): return False try: ip, port_str = proxy.split(':') port = int(port_str) if not (0 < port < 65536): return False parts = ip.split('.') if len(parts) != 4: return False for part in parts: if not (0 <= int(part) <= 255): return False except ValueError: return False return True async def verify_proxy(proxy: str): if not is_valid_proxy(proxy): with proxies_lock: invalid_proxies[proxy] = True return proxy, False test_urls = ["https://google.com", "http://httpbin.org/ip"] proxy_dict = {"http": f"http://{proxy}", "https": f"http://{proxy}"} headers = create_headers() success = False try: for url in test_urls: try: response = await asyncio.to_thread(requests.get, url, proxies=proxy_dict, headers=headers, timeout=10) response.raise_for_status() success = True break except (requests.RequestException, ValueError): continue if success: with proxies_lock: valid_proxies[proxy] = True invalid_proxies.pop(proxy, None) return proxy, True else: with proxies_lock: valid_proxies.pop(proxy, None) invalid_proxies[proxy] = True return proxy, False except Exception: with proxies_lock: valid_proxies.pop(proxy, None) invalid_proxies[proxy] = True return proxy, False async def verify_proxies_in_background(): while True: proxies_to_verify = [] with proxies_lock: if valid_proxies: proxies_to_verify = list(valid_proxies.keys()) else: if invalid_proxies: sample_size = min(len(invalid_proxies), 50) proxies_to_verify = random.sample(list(invalid_proxies.keys()), sample_size) if proxies_to_verify: tasks = [verify_proxy(proxy) for proxy in proxies_to_verify] await asyncio.gather(*tasks) await asyncio.sleep(60) async def fetch_proxies_from_sources(): proxies = set() for url in all_proxy_urls: response_text = fetch_response(url) if response_text: found_in_source = set() try: data = json.loads(response_text) if isinstance(data, list): for item in data: if isinstance(item, str) and ':' in item: found_in_source.add(item.strip()) elif isinstance(item, dict): ip = item.get('ip') or item.get('Ip') or item.get('IP') or item.get('proxy') or item.get('addr') or item.get('PX') port = item.get('port') or item.get('Port') if ip and port: found_in_source.add(f"{ip}:{port}") elif isinstance(data, dict): if 'data' in data and isinstance(data['data'], list): for item in data['data']: if isinstance(item, dict): ip = item.get('ip') or item.get('proxy') port = item.get('port') if ip and port: found_in_source.add(f"{ip}:{port}") elif 'ipPort' in item: found_in_source.add(item['ipPort']) elif 'proxies' in data and isinstance(data['proxies'], list): for item in data['proxies']: if isinstance(item, dict): ip = item.get('ip') or item.get('addr') port = item.get('port') if ip and port: found_in_source.add(f"{ip}:{port}") elif isinstance(item, str) and ':' in item: found_in_source.add(item.strip()) elif 'PXs' in data and isinstance(data['PXs'], list): for item in data['PXs']: if isinstance(item, dict): ip = item.get('PX') port = item.get('Port') if ip and port: found_in_source.add(f"{ip}:{port}") except (json.JSONDecodeError, ValueError, KeyError, TypeError, IndexError): pass try: soup = BeautifulSoup(response_text, 'html.parser') tables = soup.find_all('table') for table in tables: rows = table.find_all('tr') for row in rows[1:]: cols = row.find_all(['td', 'th']) if len(cols) >= 2: ip = cols[0].get_text(strip=True) port = cols[1].get_text(strip=True) if re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', ip) and port.isdigit(): found_in_source.add(f"{ip}:{port}") textareas = soup.find_all('textarea') for textarea in textareas: lines = textarea.get_text().splitlines() for line in lines: line = line.strip() if is_valid_proxy(line): found_in_source.add(line) pres = soup.find_all('pre') for pre in pres: lines = pre.get_text().splitlines() for line in lines: line = line.strip() if is_valid_proxy(line): found_in_source.add(line) found_in_source.update(re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5})', response_text)) except Exception: pass lines = response_text.splitlines() for line in lines: line = line.strip() parts = re.split(r'[:\s]+', line) if len(parts) >= 2 and re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', parts[0]) and parts[1].isdigit(): found_in_source.add(f"{parts[0]}:{parts[1]}") elif is_valid_proxy(line): found_in_source.add(line) for proxy in found_in_source: if is_valid_proxy(proxy): proxies.add(proxy) await asyncio.sleep(1) print(f"Fetched {len(proxies)} potential proxies from sources.") initial_verification_tasks = [verify_proxy(p) for p in proxies if p not in valid_proxies and p not in invalid_proxies] if initial_verification_tasks: print(f"Performing initial verification for {len(initial_verification_tasks)} new proxies.") await asyncio.gather(*initial_verification_tasks) print("Initial verification complete.") with proxies_lock: print(f"Total valid proxies: {len(valid_proxies)}, Total invalid proxies: {len(invalid_proxies)}") return valid_proxies, invalid_proxies @app.on_event("startup") async def on_startup(): global valid_proxies, invalid_proxies print("Starting proxy fetch and initial verification...") await fetch_proxies_from_sources() print("Initial fetch and verification complete. Starting background verification task.") asyncio.create_task(verify_proxies_in_background()) def fetch_response(url): headers = create_headers() try: response = requests.get(url, headers=headers, timeout=20) response.raise_for_status() try: return response.content.decode('utf-8') except UnicodeDecodeError: try: return response.content.decode('iso-8859-1') except UnicodeDecodeError: return response.text except requests.RequestException as e: print(f"Failed to fetch URL: {url} - Error: {e}") return None except Exception as e: print(f"An unexpected error occurred fetching {url}: {e}") return None def get_current_user(credentials: HTTPBasicCredentials = Depends(basic_auth)): correct_username = os.environ.get("PROXY_API_USERNAME", "proxyuser") correct_password = os.environ.get("PROXY_API_PASSWORD", "proxypassword") is_user_valid = credentials.username == correct_username is_pass_valid = credentials.password == correct_password if not (is_user_valid and is_pass_valid): raise HTTPException( status_code=401, detail="Incorrect username or password", headers={"WWW-Authenticate": "Basic"}, ) return credentials.username @app.get("/") async def root(): html_content = """ Proxy Service

Proxy List Service

This service scrapes and verifies public proxies.

Available Endpoints:

GET /valid

Returns a sample list (up to 50) of currently valid proxies in plain text format.

Example: /valid

GET /all-valid

Returns the complete list of currently valid proxies in plain text format.

Example: /all-valid

GET /valid-auth

Returns a sample list (up to 50) of currently valid proxies in plain text format. Requires Basic Authentication.

Credentials are set via environment variables PROXY_API_USERNAME and PROXY_API_PASSWORD (defaults: proxyuser/proxypassword).

Example: /valid-auth (will prompt for credentials)

GET /invalid

Returns a sample list (up to 50) of proxies that recently failed verification, in plain text format.

Example: /invalid

GET /stats

Returns the current count of valid and invalid proxies in JSON format.

Example: /stats

WebSocket /ws

Streams the current list of valid proxies. The list is sent periodically.

Use a WebSocket client to connect to ws://your-server-address/ws.

""" return HTMLResponse(content=html_content, status_code=200) @app.get("/valid") async def valid_proxies_endpoint(): with proxies_lock: valid_proxies_list = list(valid_proxies.keys()) count = len(valid_proxies_list) if count > 50: sample_list = random.sample(valid_proxies_list, 50) else: sample_list = valid_proxies_list response_content = "\n".join(sample_list) return Response(content=response_content, media_type='text/plain') @app.get("/all-valid") async def all_valid_proxies_endpoint(): with proxies_lock: valid_proxies_list = list(valid_proxies.keys()) response_content = "\n".join(valid_proxies_list) return Response(content=response_content, media_type='text/plain') @app.get("/invalid") async def invalid_proxies_endpoint(): with proxies_lock: invalid_proxies_list = list(invalid_proxies.keys()) count = len(invalid_proxies_list) if count > 50: sample_list = random.sample(invalid_proxies_list, 50) else: sample_list = invalid_proxies_list response_content = "\n".join(sample_list) return Response(content=response_content, media_type='text/plain') @app.get("/valid-auth") async def valid_proxies_auth_endpoint(username: str = Depends(get_current_user)): with proxies_lock: valid_proxies_list = list(valid_proxies.keys()) count = len(valid_proxies_list) if count > 50: sample_list = random.sample(valid_proxies_list, 50) else: sample_list = valid_proxies_list response_content = "\n".join(sample_list) return Response(content=response_content, media_type='text/plain') @app.get("/stats") async def get_stats(): with proxies_lock: stats = { "valid_count": len(valid_proxies), "invalid_count": len(invalid_proxies) } return stats @app.websocket("/ws") async def websocket_endpoint(websocket: WebSocket): await websocket.accept() try: while True: with proxies_lock: proxies_list = list(valid_proxies.keys()) await websocket.send_text("\n".join(proxies_list)) await asyncio.sleep(10) except Exception as e: print(f"WebSocket Error: {e}") finally: print("WebSocket connection closed") if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port="7860") # Changed this line