|
|
import os |
|
|
import json |
|
|
import random |
|
|
import re |
|
|
import requests |
|
|
import asyncio |
|
|
from fastapi import FastAPI, WebSocket, HTTPException, Depends |
|
|
from fastapi.responses import HTMLResponse, Response |
|
|
from fastapi.security import HTTPBasic, HTTPBasicCredentials |
|
|
from threading import Lock |
|
|
import uvicorn |
|
|
from faker import Faker |
|
|
from bs4 import BeautifulSoup |
|
|
import time |
|
|
|
|
|
app = FastAPI() |
|
|
faker = Faker() |
|
|
valid_proxies = {} |
|
|
invalid_proxies = {} |
|
|
proxies_lock = Lock() |
|
|
basic_auth = HTTPBasic() |
|
|
|
|
|
raw_proxy_urls = [ |
|
|
os.getenv("PROXY_API_URL", "http://pubproxy.com/api/proxy?format=txt&level=anonymous,elite&type=http,socks4,socks5&last_check=60&speed=25&limit=1&post=true&user_agent=true&cookies=true&referer=true"), |
|
|
'https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt', |
|
|
'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt', |
|
|
'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt', |
|
|
'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt', |
|
|
'https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/proxy.txt', |
|
|
'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt', |
|
|
"https://storage.googleapis.com/river-treat-249913.appspot.com/p.txt", |
|
|
"https://storage.googleapis.com/river-treat-249913.appspot.com/proxy.txt", |
|
|
"https://storage.googleapis.com/river-treat-249913.appspot.com/ultimate.txt", |
|
|
'https://raw.githubusercontent.com/proxylist/proxylist/master/proxy.txt', |
|
|
'https://raw.githubusercontent.com/scrapfly/proxy-list/main/proxies.txt', |
|
|
'https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTP.txt', |
|
|
'https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4.txt', |
|
|
'https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5.txt', |
|
|
'https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS.txt', |
|
|
'https://raw.githubusercontent.com/roosterkid/openproxylist/main/ALL.txt', |
|
|
'https://raw.githubusercontent.com/proxylist/proxylist/master/https.txt', |
|
|
'https://raw.githubusercontent.com/proxylist/proxylist/master/socks4.txt', |
|
|
'https://raw.githubusercontent.com/proxylist/proxylist/master/socks5.txt', |
|
|
'https://raw.githubusercontent.com/proxylist/proxylist/master/http.txt', |
|
|
'https://raw.githubusercontent.com/proxylist/proxylist/master/all.txt', |
|
|
'https://raw.githubusercontent.com/jetlore/proxies/master/proxy-list.txt', |
|
|
'https://raw.githubusercontent.com/hookzof/proxy-list/main/proxy.txt', |
|
|
'https://raw.githubusercontent.com/zzlol123/proxy-list/main/proxies.txt', |
|
|
'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/http.txt', |
|
|
'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/https.txt', |
|
|
'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/socks4.txt', |
|
|
'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/socks5.txt', |
|
|
'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/all.txt', |
|
|
'https://www.proxy-list.download/api/v1/get?type=https', |
|
|
'https://www.proxy-list.download/api/v1/get?type=http', |
|
|
'https://www.proxy-list.download/api/v1/get?type=socks4', |
|
|
'https://www.proxy-list.download/api/v1/get?type=socks5', |
|
|
'https://www.proxy-list.download/api/v1/get?type=all', |
|
|
'https://www.sslproxies.org/', |
|
|
'https://www.us-proxy.org/', |
|
|
'https://free-proxy-list.net/', |
|
|
'https://www.proxy-list.download/', |
|
|
'https://www.proxy-list.org/eng/proxylist.txt', |
|
|
"https://api.proxyscrape.com/v2/?request=displayproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all", |
|
|
'https://api.openproxylist.xyz/http.txt', |
|
|
'https://api.proxyscrape.com/?request=displayproxies&proxytype=http&timeout=10000', |
|
|
'https://proxylist.geonode.com/api/proxy-list?limit=500&page=1&sort_by=lastChecked&sort_type=desc', |
|
|
'https://proxyocean.net/api/v1/গেট-প্রক্সি?api_key=demo', |
|
|
'https://www.freeproxychecker.com/result/?protocol=http&anonymity=elite&anonymity=anonymous', |
|
|
'https://premproxy.com/proxy-by-country/GB.htm', |
|
|
'https://premproxy.com/list/time-01.htm', |
|
|
'https://www.my-proxy.com/free-proxy-list.html', |
|
|
'https://proxyservers.pro/proxy/list/protocol/http', |
|
|
'https://proxyservers.pro/proxy/list/protocol/socks4', |
|
|
'https://proxyservers.pro/proxy/list/protocol/socks5', |
|
|
'https://spys.me/proxy.txt', |
|
|
'https://www.proxynova.com/proxy-server-list/anonymous-proxies/', |
|
|
'https://www.proxynova.com/proxy-server-list/elite-proxies/', |
|
|
'https://www.proxy-daily.com/https_proxies.txt', |
|
|
'https://www.proxy-daily.com/http_proxies.txt', |
|
|
'https://www.proxy-daily.com/socks4_proxies.txt', |
|
|
'https://www.proxy-daily.com/socks5_proxies.txt', |
|
|
'https://free-proxy-list.net/anonymous-proxy.html', |
|
|
'https://www.ip-adress.com/proxy-list/', |
|
|
'https://hidemy.name/en/proxy-list/', |
|
|
'https://www.kproxy.com/en/proxylist.html', |
|
|
'https://www.megaproxy.com/frees proxies/', |
|
|
'https://proxy-store.com/free-proxy-list', |
|
|
'https://proxylists.net/http_anonymous_proxylist.txt', |
|
|
'https://proxylists.net/http_elite_proxylist.txt', |
|
|
'https://proxylists.net/socks4_proxylist.txt', |
|
|
'https://proxylists.net/socks5_proxylist.txt', |
|
|
'http://www.httptunnel.ge/ProxyListForFree.aspx', |
|
|
'https://open-proxy.net/anonymous-proxies', |
|
|
'https://open-proxy.net/elite-proxies', |
|
|
'https://premproxy.com/socks-proxy/', |
|
|
'https://www.socks-proxy.net/', |
|
|
'https://www.freeproxylists.net/', |
|
|
'https://www.proxies.net/', |
|
|
'https://www.proxy-rack.com/free-proxy-list/', |
|
|
'https://www.vipsocks24.net/proxylist-http', |
|
|
'https://www.vipsocks24.net/proxylist-socks4', |
|
|
'https://www.vipsocks24.net/proxylist-socks5', |
|
|
'https://www.cool-proxy.net/proxies/http_proxy_list', |
|
|
'https://www.cool-proxy.net/proxies/socks4_proxy_list', |
|
|
'https://www.cool-proxy.net/proxies/socks5_proxy_list', |
|
|
'https://proxygather.com/proxylist/anonymity/anonymous', |
|
|
'https://proxygather.com/proxylist/anonymity/elite', |
|
|
'https://www.preproxy.com/proxy-servers', |
|
|
'https://www.preproxy.com/socks5-proxies', |
|
|
'https://www.preproxy.com/socks4-proxies', |
|
|
'https://www.proxy-listen.de/en/proxy-server-list/http.html', |
|
|
'https://www.proxy-listen.de/en/proxy-server-list/https.html', |
|
|
'https://www.proxy-listen.de/en/proxy-server-list/socks4.html', |
|
|
'https://www.proxy-listen.de/en/proxy-server-list/socks5.html', |
|
|
'https://www.freeproxylists.net/en/http_proxies.html', |
|
|
'https://www.freeproxylists.net/en/https_proxies.html', |
|
|
'https://www.freeproxylists.net/en/socks4_proxies.html', |
|
|
'https://www.freeproxylists.net/en/socks5_proxies.html', |
|
|
'https://www.xroxy.com/proxylist.htm', |
|
|
'https://proxylist.me/api/v1/getProxy?anon=elite', |
|
|
'https://proxylist.me/api/v1/getProxy?anon=anonymous', |
|
|
'https://proxylist.me/api/v1/getProxy?anon=transparent', |
|
|
'https://proxylist.me/api/v1/getProxy?type=http', |
|
|
'https://proxylist.me/api/v1/getProxy?type=https', |
|
|
'https://proxylist.me/api/v1/getProxy?type=socks4', |
|
|
'https://proxylist.me/api/v1/getProxy?type=socks5', |
|
|
'https://www.proxyscan.io/api/proxy?type=http', |
|
|
'https://www.proxyscan.io/api/proxy?type=https', |
|
|
'https://www.proxyscan.io/api/proxy?type=socks4', |
|
|
'https://www.proxyscan.io/api/proxy?type=socks5', |
|
|
'https://www.proxyscan.io/api/proxy?last_check=60', |
|
|
'https://openproxies.pl/proxylist.txt', |
|
|
'https://openproxies.pl/socks4.txt', |
|
|
'https://openproxies.pl/socks5.txt', |
|
|
'https://www.gatherproxy.com/zh/proxylist/country/?c=United%20States', |
|
|
'https://www.free-proxy-cz.com/en/proxylist/country/all/http/ping/all', |
|
|
'https://www.fre-proxy.ru/en/proxy/country/ru', |
|
|
'https://www.proxydocker.com/en/proxylist/country/all', |
|
|
'https://proxydb.net/?protocol=http&anonlvl=4', |
|
|
'https://proxydb.net/?protocol=socks4&anonlvl=4', |
|
|
'https://proxydb.net/?protocol=socks5&anonlvl=4', |
|
|
'https://www.ip-tracker.org/proxies/https-proxy-list.php', |
|
|
'https://www.ip-tracker.org/proxies/socks4-proxy-list.php', |
|
|
'https://www.ip-tracker.org/proxies/socks5-proxy-list.php', |
|
|
'https://www.proxy-server-list.com/country/us', |
|
|
'https://www.7proxies.net/index.php/proxy-list/proxies.html', |
|
|
'https://www.best-proxy.net/en/country/code/', |
|
|
'https://www.haiproxy.com/en/free-proxy-list/', |
|
|
'https://www.bonanza.com/items/like/228248525/VPN-Proxy-List-100000-Fresh-Daily-Updated-Private-Proxies-HTTP-SOCKS-Proxy', |
|
|
'https://www.megaproxylist.net/anonymous-proxy-list.html', |
|
|
'https://www.megaproxylist.net/elite-proxy-list.html', |
|
|
'https://www.proxy-ipv4.net/en/proxy-list-anonymous.html', |
|
|
'https://www.proxy-ipv4.net/en/proxy-list-elite-anonymous.html', |
|
|
'https://www.proxiesforfree.com/proxies/http_proxies.html', |
|
|
'https://www.proxiesforfree.com/proxies/socks_proxies.html', |
|
|
'https://www.proxy-list.download/HTTPS', |
|
|
'https://www.proxy-list.download/HTTP', |
|
|
'https://www.proxy-list.download/SOCKS4', |
|
|
'https://www.proxy-list.download/SOCKS5', |
|
|
'https://www.freeproxyme.org/proxy-list.html', |
|
|
'https://proxy-hub.com/en/us-anonymous-proxy-list.html', |
|
|
'https://www.ipaddress.com/proxy-list/', |
|
|
'https://www.proxylists.net/http.txt', |
|
|
'https://www.proxylists.net/socks4.txt', |
|
|
'https://www.proxylists.net/socks5.txt', |
|
|
'https://www.proxy-my-ip.com/free-proxy-list.html', |
|
|
'https://www.proxy-german.de/proxy-list/', |
|
|
'https://proxylist.cc/free-proxy-list/', |
|
|
'https://www.proxy-checker.net/proxy_list.php?type=https', |
|
|
'https://www.proxy-checker.net/proxy_list.php?type=http', |
|
|
'https://www.proxydrop.com/en/us-proxy-list/', |
|
|
'https://www.proxydrop.com/en/elite-proxy-list/', |
|
|
'https://www.proxydrop.com/en/anonymous-proxy-list/', |
|
|
'https://www.proxy-store.com/proxylist/country/us', |
|
|
'https://www.proxy-israel.com/en/proxy-list/', |
|
|
'https://www.proxies-socks5.net/proxy-lists/country-us', |
|
|
'https://www.proxies-socks5.net/proxy-lists/country-de', |
|
|
'https://www.proxies-socks5.net/proxy-lists/type-socks5', |
|
|
'https://www.proxies-socks5.net/proxy-lists/type-socks4', |
|
|
'https://www.proxy-list.org/en/socks-list/country-US.html', |
|
|
'https://www.proxy-ipv6.net/index_anonymous.html', |
|
|
'https://www.proxy-ipv6.net/index_elite.html', |
|
|
'https://www.proxy-ipv6.net/index_transparent.html', |
|
|
'https://www.proxy-ipv6.net/index_ssl.html', |
|
|
'https://www.proxies-socks5.net/proxy-lists/anonymous-elite', |
|
|
'https://www.proxies-socks5.net/proxy-lists/anonymous', |
|
|
'https://www.proxies-socks5.net/proxy-lists/transparent', |
|
|
'https://www.proxynow.top/proxy-list-anonymous', |
|
|
'https://www.proxynow.top/proxy-list-elite', |
|
|
'https://www.proxy-server.net/proxies/anonymous-elite-proxy-list.aspx', |
|
|
'https://www.proxy-server.net/proxies/anonymous-proxy-list.aspx', |
|
|
'https://www.proxy-server.net/proxies/transparent-proxy-list.aspx', |
|
|
'https://www.proxydaddy.com/proxylist.php?protocol=HTTP', |
|
|
'https://www.proxydaddy.com/proxylist.php?protocol=HTTPS', |
|
|
'https://www.proxydaddy.com/proxylist.php?protocol=SOCKS4', |
|
|
'https://www.proxydaddy.com/proxylist.php?protocol=SOCKS5', |
|
|
'https://www.proxy4free.com/list/webproxy_anon.html', |
|
|
'https://www.proxy4free.com/list/webproxy_elite.html', |
|
|
'https://www.freeproxylist.co/anonymous.html', |
|
|
'https://www.freeproxylist.co/elite.html', |
|
|
'https://www.proxylisty.com/anonymous-proxies', |
|
|
'https://www.proxylisty.com/elite-proxies', |
|
|
'https://www.aliveproxy.com/anonymous-proxy-list.aspx', |
|
|
'https://www.aliveproxy.com/elite-proxy-list.aspx', |
|
|
'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/https.php', |
|
|
'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/http.php', |
|
|
'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/socks5.php', |
|
|
'https://proxyservers.net/proxy-list/sort/country/order/asc', |
|
|
'https://www.ip2location.com/free/proxy-list', |
|
|
'https://www.yougetproxy.com/zh-cn/https-proxy-list/', |
|
|
'https://www.yougetproxy.com/zh-cn/http-proxy-list/', |
|
|
'https://www.ip-finder.net/en/proxylist/anonymous-proxies', |
|
|
'https://www.ip-finder.net/en/proxylist/elite-proxies', |
|
|
'https://www.rapidunblock.com/proxy-list/', |
|
|
'https://www.privateproxyguide.com/anonymous-proxy-list/', |
|
|
'https://www.privateproxyguide.com/elite-proxy-list/', |
|
|
'https://www.proxy-german.de/', |
|
|
'https://www.proxy-italia.it/proxy-list/', |
|
|
'https://www.proxylistchecker.pro/https_proxies.txt', |
|
|
'https://www.proxylistchecker.pro/http_proxies.txt', |
|
|
'https://www.proxylistchecker.pro/socks4_proxies.txt', |
|
|
'https://www.proxylistchecker.pro/socks5_proxies.txt', |
|
|
'https://www.proxy-list.org/en/socks-list-1.html', |
|
|
'https://www.proxy-list.org/en/https-anonymous-elite-proxy-list-1.html', |
|
|
'https://www.proxy-list.org/en/https-anonymous-proxy-list-1.html', |
|
|
'https://www.proxy-list.org/en/http-anonymous-elite-proxy-list-1.html', |
|
|
'https://www.proxy-list.org/en/http-anonymous-proxy-list-1.html', |
|
|
'https://www.proxyserverlist24.top/proxylists/http', |
|
|
'https://www.proxyserverlist24.top/proxylists/https', |
|
|
'https://www.proxyserverlist24.top/proxylists/socks4', |
|
|
'https://www.proxyserverlist24.top/proxylists/socks5', |
|
|
'https://www.proxy-ipv4.net/', |
|
|
'https://www.proxy-ipv6.net/', |
|
|
'https://www.proxy-list.org/', |
|
|
'https://www.proxyserverlist24.top/', |
|
|
'https://www.freeproxy.world/', |
|
|
'https://www.free-proxy-server.net/', |
|
|
'https://www.proxylists.me/', |
|
|
'https://www.proxy-directory.com/', |
|
|
'https://www.proxy-provider.net/', |
|
|
'https://www.proxy-server.net/', |
|
|
'https://www.proxy-list.me/', |
|
|
'https://www.proxy-daily.com/', |
|
|
'https://www.proxy-store.com/', |
|
|
'https://www.proxy-israel.com/', |
|
|
'https://www.proxy-german.de/', |
|
|
'https://www.proxy-italia.it/', |
|
|
'https://www.proxynow.top/', |
|
|
'https://www.proxydaddy.com/', |
|
|
'https://www.proxy4free.com/', |
|
|
'https://www.freeproxylist.co/', |
|
|
'https://www.proxylisty.com/', |
|
|
'https://www.aliveproxy.com/', |
|
|
'https://proxyscrape.com/free-proxy-list', |
|
|
'https://www.goldproxy.jp/proxylist/anonymous.html', |
|
|
'https://www.goldproxy.jp/proxylist/country/US.html', |
|
|
'https://www.goldproxy.jp/proxylist/elite.html', |
|
|
'https://www.ip-port.net/free-proxy-anonymous-server-list', |
|
|
'https://www.ip-port.net/free-proxy-elite-server-list', |
|
|
'https://www.ip-port.net/free-proxy-transparent-server-list', |
|
|
'https://www.gatherproxy.com/', |
|
|
'https://incloak.com/proxy-list/', |
|
|
'https://proxyservers.pro/', |
|
|
'https://www.proxy-anonymous.com/en/us-free-proxy-list.shtml', |
|
|
'https://www.proxy-anonymous.com/en/country_US-01.shtml', |
|
|
'https://www.proxy-anonymous.com/en/http-ssl-anonymous-proxy-list.shtml', |
|
|
'https://www.proxy-anonymous.com/en/socks-anonymous-proxy-list.shtml', |
|
|
'https://www.checkerproxy.net/api/anon/http', |
|
|
'https://www.checkerproxy.net/api/anon/https', |
|
|
'https://www.checkerproxy.net/api/anon/socks4', |
|
|
'https://www.checkerproxy.net/api/anon/socks5', |
|
|
'https://www.proxyscout.io/en/https', |
|
|
'https://www.proxyscout.io/en/socks4', |
|
|
'https://www.proxyscout.io/en/socks5', |
|
|
'https://www.proxy-list.download/SOCKS', |
|
|
'https://www.proxy-list.org/english/index.php?p=http', |
|
|
'https://www.proxy-list.org/english/index.php?p=https', |
|
|
'https://www.proxy-list.org/english/index.php?p=socks4', |
|
|
'https://www.proxy-list.org/english/index.php?p=socks5', |
|
|
'https://proxylists.net/', |
|
|
'https://www.rapidunblock.com/', |
|
|
'https://www.privateproxyguide.com/', |
|
|
'https://www.proxy-directory.com/proxylists.php', |
|
|
'https://www.proxy-provider.net/proxylists.php', |
|
|
'https://www.proxy-server.net/proxylists.php', |
|
|
'https://www.proxy-list.me/proxylists.php', |
|
|
'https://www.proxy-daily.com/proxylists.php', |
|
|
'https://www.proxy-store.com/proxylists.php', |
|
|
'https://www.proxy-israel.com/proxylists.php', |
|
|
'https://www.proxy-german.de/proxylists.php', |
|
|
'https://www.proxy-italia.it/proxylists.php', |
|
|
'https://www.proxynow.top/proxylists.php', |
|
|
'https://www.proxydaddy.com/proxylists.php', |
|
|
'https://www.proxy4free.com/proxylists.php', |
|
|
'https://www.freeproxylist.co/proxylists.php', |
|
|
'https://www.proxylisty.com/proxylists.php', |
|
|
'https://www.aliveproxy.com/proxylists.aspx', |
|
|
'https://www.proxyscrape.com/proxylists.php', |
|
|
'https://www.goldproxy.jp/proxylists.php', |
|
|
'https://www.ip-port.net/proxylists.php', |
|
|
'https://www.gatherproxy.com/proxylists.php', |
|
|
'https://incloak.com/proxylists.php', |
|
|
'https://www.proxyservers.pro/proxylists.php', |
|
|
'https://www.proxy-anonymous.com/proxylists.php', |
|
|
'https://www.checkerproxy.net/proxylists.php', |
|
|
'https://www.proxyscout.io/proxylists.php', |
|
|
'https://www.proxy-list.download/proxylists.php', |
|
|
'https://www.proxy-list.org/proxylists.php', |
|
|
'https://www.proxyserverlist24.top/proxylists.php', |
|
|
'https://www.freeproxy.world/proxylists.php', |
|
|
'https://www.free-proxy-server.net/proxylists.php', |
|
|
'https://www.proxylists.me/proxylists.php', |
|
|
'https://www.ip2location.com/proxylists.php', |
|
|
'https://www.yougetproxy.com/proxylists.php', |
|
|
'https://www.ip-finder.net/proxylists.php', |
|
|
'https://www.freeproxylists.net/proxylists.php', |
|
|
'https://www.xroxy.com/proxylists.php', |
|
|
'https://www.sslproxies.org/proxylists.php', |
|
|
'https://www.us-proxy.org/proxylists.php', |
|
|
'https://www.socks-proxy.net/proxylists.php', |
|
|
'https://www.httptunnel.ge/proxylists.php', |
|
|
'https://open-proxy.net/proxylists.php', |
|
|
'https://premproxy.com/proxylists.php', |
|
|
'https://www.vipsocks24.net/proxylists.php', |
|
|
'https://www.cool-proxy.net/proxylists.php', |
|
|
] |
|
|
|
|
|
all_proxy_urls = list(set(raw_proxy_urls)) |
|
|
print(f"Total unique proxy URLs after deduplication: {len(all_proxy_urls)}") |
|
|
|
|
|
def create_headers(): |
|
|
user_agent = faker.user_agent() |
|
|
random_ip = faker.ipv4() |
|
|
return {"User-Agent": user_agent, "X-Forwarded-For": random_ip, "Client-IP": random_ip, "X-Real-IP": random_ip} |
|
|
|
|
|
def is_valid_proxy(proxy: str) -> bool: |
|
|
ip_port_pattern = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5}$') |
|
|
if not ip_port_pattern.match(proxy): |
|
|
return False |
|
|
try: |
|
|
ip, port_str = proxy.split(':') |
|
|
port = int(port_str) |
|
|
if not (0 < port < 65536): |
|
|
return False |
|
|
parts = ip.split('.') |
|
|
if len(parts) != 4: |
|
|
return False |
|
|
for part in parts: |
|
|
if not (0 <= int(part) <= 255): |
|
|
return False |
|
|
except ValueError: |
|
|
return False |
|
|
return True |
|
|
|
|
|
|
|
|
async def verify_proxy(proxy: str): |
|
|
if not is_valid_proxy(proxy): |
|
|
with proxies_lock: |
|
|
invalid_proxies[proxy] = True |
|
|
return proxy, False |
|
|
|
|
|
test_urls = ["https://google.com", "http://httpbin.org/ip"] |
|
|
proxy_dict = {"http": f"http://{proxy}", "https": f"http://{proxy}"} |
|
|
headers = create_headers() |
|
|
|
|
|
success = False |
|
|
try: |
|
|
for url in test_urls: |
|
|
try: |
|
|
response = await asyncio.to_thread(requests.get, url, proxies=proxy_dict, headers=headers, timeout=10) |
|
|
response.raise_for_status() |
|
|
success = True |
|
|
break |
|
|
except (requests.RequestException, ValueError): |
|
|
continue |
|
|
|
|
|
if success: |
|
|
with proxies_lock: |
|
|
valid_proxies[proxy] = True |
|
|
invalid_proxies.pop(proxy, None) |
|
|
return proxy, True |
|
|
else: |
|
|
with proxies_lock: |
|
|
valid_proxies.pop(proxy, None) |
|
|
invalid_proxies[proxy] = True |
|
|
return proxy, False |
|
|
|
|
|
except Exception: |
|
|
with proxies_lock: |
|
|
valid_proxies.pop(proxy, None) |
|
|
invalid_proxies[proxy] = True |
|
|
return proxy, False |
|
|
|
|
|
async def verify_proxies_in_background(): |
|
|
while True: |
|
|
proxies_to_verify = [] |
|
|
with proxies_lock: |
|
|
if valid_proxies: |
|
|
proxies_to_verify = list(valid_proxies.keys()) |
|
|
else: |
|
|
if invalid_proxies: |
|
|
sample_size = min(len(invalid_proxies), 50) |
|
|
proxies_to_verify = random.sample(list(invalid_proxies.keys()), sample_size) |
|
|
|
|
|
|
|
|
if proxies_to_verify: |
|
|
tasks = [verify_proxy(proxy) for proxy in proxies_to_verify] |
|
|
await asyncio.gather(*tasks) |
|
|
|
|
|
await asyncio.sleep(60) |
|
|
|
|
|
async def fetch_proxies_from_sources(): |
|
|
proxies = set() |
|
|
for url in all_proxy_urls: |
|
|
response_text = fetch_response(url) |
|
|
if response_text: |
|
|
found_in_source = set() |
|
|
try: |
|
|
data = json.loads(response_text) |
|
|
if isinstance(data, list): |
|
|
for item in data: |
|
|
if isinstance(item, str) and ':' in item: |
|
|
found_in_source.add(item.strip()) |
|
|
elif isinstance(item, dict): |
|
|
ip = item.get('ip') or item.get('Ip') or item.get('IP') or item.get('proxy') or item.get('addr') or item.get('PX') |
|
|
port = item.get('port') or item.get('Port') |
|
|
if ip and port: |
|
|
found_in_source.add(f"{ip}:{port}") |
|
|
|
|
|
elif isinstance(data, dict): |
|
|
if 'data' in data and isinstance(data['data'], list): |
|
|
for item in data['data']: |
|
|
if isinstance(item, dict): |
|
|
ip = item.get('ip') or item.get('proxy') |
|
|
port = item.get('port') |
|
|
if ip and port: |
|
|
found_in_source.add(f"{ip}:{port}") |
|
|
elif 'ipPort' in item: |
|
|
found_in_source.add(item['ipPort']) |
|
|
elif 'proxies' in data and isinstance(data['proxies'], list): |
|
|
for item in data['proxies']: |
|
|
if isinstance(item, dict): |
|
|
ip = item.get('ip') or item.get('addr') |
|
|
port = item.get('port') |
|
|
if ip and port: |
|
|
found_in_source.add(f"{ip}:{port}") |
|
|
elif isinstance(item, str) and ':' in item: |
|
|
found_in_source.add(item.strip()) |
|
|
elif 'PXs' in data and isinstance(data['PXs'], list): |
|
|
for item in data['PXs']: |
|
|
if isinstance(item, dict): |
|
|
ip = item.get('PX') |
|
|
port = item.get('Port') |
|
|
if ip and port: |
|
|
found_in_source.add(f"{ip}:{port}") |
|
|
|
|
|
|
|
|
except (json.JSONDecodeError, ValueError, KeyError, TypeError, IndexError): |
|
|
pass |
|
|
|
|
|
try: |
|
|
soup = BeautifulSoup(response_text, 'html.parser') |
|
|
tables = soup.find_all('table') |
|
|
for table in tables: |
|
|
rows = table.find_all('tr') |
|
|
for row in rows[1:]: |
|
|
cols = row.find_all(['td', 'th']) |
|
|
if len(cols) >= 2: |
|
|
ip = cols[0].get_text(strip=True) |
|
|
port = cols[1].get_text(strip=True) |
|
|
if re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', ip) and port.isdigit(): |
|
|
found_in_source.add(f"{ip}:{port}") |
|
|
|
|
|
textareas = soup.find_all('textarea') |
|
|
for textarea in textareas: |
|
|
lines = textarea.get_text().splitlines() |
|
|
for line in lines: |
|
|
line = line.strip() |
|
|
if is_valid_proxy(line): |
|
|
found_in_source.add(line) |
|
|
|
|
|
pres = soup.find_all('pre') |
|
|
for pre in pres: |
|
|
lines = pre.get_text().splitlines() |
|
|
for line in lines: |
|
|
line = line.strip() |
|
|
if is_valid_proxy(line): |
|
|
found_in_source.add(line) |
|
|
|
|
|
found_in_source.update(re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5})', response_text)) |
|
|
|
|
|
except Exception: |
|
|
pass |
|
|
|
|
|
lines = response_text.splitlines() |
|
|
for line in lines: |
|
|
line = line.strip() |
|
|
parts = re.split(r'[:\s]+', line) |
|
|
if len(parts) >= 2 and re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', parts[0]) and parts[1].isdigit(): |
|
|
found_in_source.add(f"{parts[0]}:{parts[1]}") |
|
|
elif is_valid_proxy(line): |
|
|
found_in_source.add(line) |
|
|
|
|
|
|
|
|
for proxy in found_in_source: |
|
|
if is_valid_proxy(proxy): |
|
|
proxies.add(proxy) |
|
|
|
|
|
await asyncio.sleep(1) |
|
|
|
|
|
print(f"Fetched {len(proxies)} potential proxies from sources.") |
|
|
|
|
|
initial_verification_tasks = [verify_proxy(p) for p in proxies if p not in valid_proxies and p not in invalid_proxies] |
|
|
if initial_verification_tasks: |
|
|
print(f"Performing initial verification for {len(initial_verification_tasks)} new proxies.") |
|
|
await asyncio.gather(*initial_verification_tasks) |
|
|
print("Initial verification complete.") |
|
|
|
|
|
with proxies_lock: |
|
|
print(f"Total valid proxies: {len(valid_proxies)}, Total invalid proxies: {len(invalid_proxies)}") |
|
|
return valid_proxies, invalid_proxies |
|
|
|
|
|
|
|
|
@app.on_event("startup") |
|
|
async def on_startup(): |
|
|
global valid_proxies, invalid_proxies |
|
|
print("Starting proxy fetch and initial verification...") |
|
|
await fetch_proxies_from_sources() |
|
|
print("Initial fetch and verification complete. Starting background verification task.") |
|
|
asyncio.create_task(verify_proxies_in_background()) |
|
|
|
|
|
|
|
|
def fetch_response(url): |
|
|
headers = create_headers() |
|
|
try: |
|
|
response = requests.get(url, headers=headers, timeout=20) |
|
|
response.raise_for_status() |
|
|
try: |
|
|
return response.content.decode('utf-8') |
|
|
except UnicodeDecodeError: |
|
|
try: |
|
|
return response.content.decode('iso-8859-1') |
|
|
except UnicodeDecodeError: |
|
|
return response.text |
|
|
except requests.RequestException as e: |
|
|
print(f"Failed to fetch URL: {url} - Error: {e}") |
|
|
return None |
|
|
except Exception as e: |
|
|
print(f"An unexpected error occurred fetching {url}: {e}") |
|
|
return None |
|
|
|
|
|
|
|
|
def get_current_user(credentials: HTTPBasicCredentials = Depends(basic_auth)): |
|
|
correct_username = os.environ.get("PROXY_API_USERNAME", "proxyuser") |
|
|
correct_password = os.environ.get("PROXY_API_PASSWORD", "proxypassword") |
|
|
|
|
|
is_user_valid = credentials.username == correct_username |
|
|
is_pass_valid = credentials.password == correct_password |
|
|
|
|
|
if not (is_user_valid and is_pass_valid): |
|
|
raise HTTPException( |
|
|
status_code=401, |
|
|
detail="Incorrect username or password", |
|
|
headers={"WWW-Authenticate": "Basic"}, |
|
|
) |
|
|
return credentials.username |
|
|
|
|
|
@app.get("/") |
|
|
async def root(): |
|
|
html_content = """ |
|
|
<!DOCTYPE html> |
|
|
<html> |
|
|
<head> |
|
|
<title>Proxy Service</title> |
|
|
<style> |
|
|
body { font-family: sans-serif; padding: 20px; } |
|
|
h1 { color: #333; } |
|
|
ul { list-style: none; padding: 0; } |
|
|
li { margin-bottom: 10px; } |
|
|
a { text-decoration: none; color: #007bff; } |
|
|
a:hover { text-decoration: underline; } |
|
|
.endpoint { background-color: #f8f9fa; border: 1px solid #dee2e6; border-radius: 4px; padding: 15px; margin-bottom:15px; } |
|
|
code { background-color: #e9ecef; padding: 2px 4px; border-radius: 3px; } |
|
|
</style> |
|
|
</head> |
|
|
<body> |
|
|
<h1>Proxy List Service</h1> |
|
|
<p>This service scrapes and verifies public proxies.</p> |
|
|
|
|
|
<h2>Available Endpoints:</h2> |
|
|
|
|
|
<div class="endpoint"> |
|
|
<strong>GET /valid</strong> |
|
|
<p>Returns a sample list (up to 50) of currently valid proxies in plain text format.</p> |
|
|
<p>Example: <a href="/valid" target="_blank">/valid</a></p> |
|
|
</div> |
|
|
|
|
|
<div class="endpoint"> |
|
|
<strong>GET /all-valid</strong> |
|
|
<p>Returns the complete list of currently valid proxies in plain text format.</p> |
|
|
<p>Example: <a href="/all-valid" target="_blank">/all-valid</a></p> |
|
|
</div> |
|
|
|
|
|
<div class="endpoint"> |
|
|
<strong>GET /valid-auth</strong> |
|
|
<p>Returns a sample list (up to 50) of currently valid proxies in plain text format. Requires Basic Authentication.</p> |
|
|
<p>Credentials are set via environment variables <code>PROXY_API_USERNAME</code> and <code>PROXY_API_PASSWORD</code> (defaults: proxyuser/proxypassword).</p> |
|
|
<p>Example: <a href="/valid-auth" target="_blank">/valid-auth</a> (will prompt for credentials)</p> |
|
|
</div> |
|
|
|
|
|
<div class="endpoint"> |
|
|
<strong>GET /invalid</strong> |
|
|
<p>Returns a sample list (up to 50) of proxies that recently failed verification, in plain text format.</p> |
|
|
<p>Example: <a href="/invalid" target="_blank">/invalid</a></p> |
|
|
</div> |
|
|
|
|
|
<div class="endpoint"> |
|
|
<strong>GET /stats</strong> |
|
|
<p>Returns the current count of valid and invalid proxies in JSON format.</p> |
|
|
<p>Example: <a href="/stats" target="_blank">/stats</a></p> |
|
|
</div> |
|
|
|
|
|
<div class="endpoint"> |
|
|
<strong>WebSocket /ws</strong> |
|
|
<p>Streams the current list of valid proxies. The list is sent periodically.</p> |
|
|
<p>Use a WebSocket client to connect to <code>ws://your-server-address/ws</code>.</p> |
|
|
</div> |
|
|
|
|
|
</body> |
|
|
</html> |
|
|
""" |
|
|
return HTMLResponse(content=html_content, status_code=200) |
|
|
|
|
|
@app.get("/valid") |
|
|
async def valid_proxies_endpoint(): |
|
|
with proxies_lock: |
|
|
valid_proxies_list = list(valid_proxies.keys()) |
|
|
count = len(valid_proxies_list) |
|
|
if count > 50: |
|
|
sample_list = random.sample(valid_proxies_list, 50) |
|
|
else: |
|
|
sample_list = valid_proxies_list |
|
|
response_content = "\n".join(sample_list) |
|
|
return Response(content=response_content, media_type='text/plain') |
|
|
|
|
|
@app.get("/all-valid") |
|
|
async def all_valid_proxies_endpoint(): |
|
|
with proxies_lock: |
|
|
valid_proxies_list = list(valid_proxies.keys()) |
|
|
response_content = "\n".join(valid_proxies_list) |
|
|
return Response(content=response_content, media_type='text/plain') |
|
|
|
|
|
|
|
|
@app.get("/invalid") |
|
|
async def invalid_proxies_endpoint(): |
|
|
with proxies_lock: |
|
|
invalid_proxies_list = list(invalid_proxies.keys()) |
|
|
count = len(invalid_proxies_list) |
|
|
if count > 50: |
|
|
sample_list = random.sample(invalid_proxies_list, 50) |
|
|
else: |
|
|
sample_list = invalid_proxies_list |
|
|
response_content = "\n".join(sample_list) |
|
|
return Response(content=response_content, media_type='text/plain') |
|
|
|
|
|
@app.get("/valid-auth") |
|
|
async def valid_proxies_auth_endpoint(username: str = Depends(get_current_user)): |
|
|
with proxies_lock: |
|
|
valid_proxies_list = list(valid_proxies.keys()) |
|
|
count = len(valid_proxies_list) |
|
|
if count > 50: |
|
|
sample_list = random.sample(valid_proxies_list, 50) |
|
|
else: |
|
|
sample_list = valid_proxies_list |
|
|
response_content = "\n".join(sample_list) |
|
|
return Response(content=response_content, media_type='text/plain') |
|
|
|
|
|
@app.get("/stats") |
|
|
async def get_stats(): |
|
|
with proxies_lock: |
|
|
stats = { |
|
|
"valid_count": len(valid_proxies), |
|
|
"invalid_count": len(invalid_proxies) |
|
|
} |
|
|
return stats |
|
|
|
|
|
@app.websocket("/ws") |
|
|
async def websocket_endpoint(websocket: WebSocket): |
|
|
await websocket.accept() |
|
|
try: |
|
|
while True: |
|
|
with proxies_lock: |
|
|
proxies_list = list(valid_proxies.keys()) |
|
|
await websocket.send_text("\n".join(proxies_list)) |
|
|
await asyncio.sleep(10) |
|
|
except Exception as e: |
|
|
print(f"WebSocket Error: {e}") |
|
|
finally: |
|
|
print("WebSocket connection closed") |
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
uvicorn.run(app, host="0.0.0.0", port="7860") |
