File size: 7,773 Bytes
7319d31 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 |
# tools/search_tools.py
import requests
import os
from typing import List, Dict, Any, Optional
from .utils import get_env_var, logger
class SearchTools:
"""Free and cost-effective search tools with multiple providers"""
def __init__(self):
# Primary: Free alternatives
self.duckduckgo_enabled = True
# Secondary: Tavily (cost-effective)
self.tavily_api_key = os.getenv("TAVILY_API_KEY")
# Tertiary: SerpAPI (expensive, fallback only)
self.serpapi_key = os.getenv("SERPAPI_KEY")
def search_duckduckgo(self, query: str, max_results: int = 5) -> List[Dict[str, Any]]:
"""
Free search using DuckDuckGo Instant Answer API
Args:
query: Search query
max_results: Maximum number of results
Returns:
List of search results
"""
try:
# DuckDuckGo Instant Answer API (free)
url = "https://api.duckduckgo.com/"
params = {
'q': query,
'format': 'json',
'no_html': '1',
'skip_disambig': '1'
}
response = requests.get(url, params=params, timeout=10)
response.raise_for_status()
data = response.json()
results = []
# Process abstract
if data.get('Abstract'):
results.append({
'title': data.get('Heading', 'DuckDuckGo Result'),
'url': data.get('AbstractURL', ''),
'content': data.get('Abstract', ''),
'source': 'DuckDuckGo'
})
# Process related topics
for topic in data.get('RelatedTopics', [])[:max_results-len(results)]:
if isinstance(topic, dict) and 'Text' in topic:
results.append({
'title': topic.get('Text', '')[:100],
'url': topic.get('FirstURL', ''),
'content': topic.get('Text', ''),
'source': 'DuckDuckGo'
})
return results[:max_results]
except Exception as e:
logger.error(f"DuckDuckGo search failed: {str(e)}")
return []
def search_tavily(self, query: str, max_results: int = 5) -> List[Dict[str, Any]]:
"""
Search using Tavily API (cost-effective)
Args:
query: Search query
max_results: Maximum number of results
Returns:
List of search results
"""
if not self.tavily_api_key:
logger.warning("Tavily API key not provided")
return []
try:
url = "https://api.tavily.com/search"
payload = {
"api_key": self.tavily_api_key,
"query": query,
"search_depth": "basic",
"include_answer": False,
"include_images": False,
"include_raw_content": False,
"max_results": max_results
}
response = requests.post(url, json=payload, timeout=15)
response.raise_for_status()
data = response.json()
results = []
for result in data.get('results', []):
results.append({
'title': result.get('title', ''),
'url': result.get('url', ''),
'content': result.get('content', ''),
'source': 'Tavily'
})
return results
except Exception as e:
logger.error(f"Tavily search failed: {str(e)}")
return []
def search_serpapi(self, query: str, max_results: int = 5) -> List[Dict[str, Any]]:
"""
Search using SerpAPI (expensive, fallback only)
Args:
query: Search query
max_results: Maximum number of results
Returns:
List of search results
"""
if not self.serpapi_key:
logger.warning("SerpAPI key not provided")
return []
try:
url = "https://serpapi.com/search"
params = {
'api_key': self.serpapi_key,
'engine': 'google',
'q': query,
'num': max_results,
'gl': 'us', # Geolocation
'hl': 'en' # Language
}
response = requests.get(url, params=params, timeout=15)
response.raise_for_status()
data = response.json()
results = []
for result in data.get('organic_results', []):
results.append({
'title': result.get('title', ''),
'url': result.get('link', ''),
'content': result.get('snippet', ''),
'source': 'Google (SerpAPI)'
})
return results
except Exception as e:
logger.error(f"SerpAPI search failed: {str(e)}")
return []
def search(self, query: str, max_results: int = 5) -> List[Dict[str, Any]]:
"""
Comprehensive search using multiple providers with fallback strategy
Args:
query: Search query
max_results: Maximum number of results
Returns:
List of search results from best available provider
"""
if not query.strip():
return []
# Try providers in order of preference (free -> cheap -> expensive)
providers = [
("DuckDuckGo", self.search_duckduckgo),
("Tavily", self.search_tavily),
("SerpAPI", self.search_serpapi)
]
for provider_name, search_func in providers:
try:
logger.info(f"Attempting search with {provider_name}")
results = search_func(query, max_results)
if results:
logger.info(f"Successfully retrieved {len(results)} results from {provider_name}")
return results
else:
logger.warning(f"No results from {provider_name}")
except Exception as e:
logger.error(f"Error with {provider_name}: {str(e)}")
continue
logger.error("All search providers failed")
return []
def search_news(self, query: str, max_results: int = 5) -> List[Dict[str, Any]]:
"""Search for news articles"""
news_query = f"news {query}"
return self.search(news_query, max_results)
def search_academic(self, query: str, max_results: int = 5) -> List[Dict[str, Any]]:
"""Search for academic content"""
academic_query = f"academic research {query} site:scholar.google.com OR site:arxiv.org OR site:researchgate.net"
return self.search(academic_query, max_results)
# Convenience functions
def search_web(query: str, max_results: int = 5) -> List[Dict[str, Any]]:
"""Standalone function for web search"""
tools = SearchTools()
return tools.search(query, max_results)
def search_news(query: str, max_results: int = 5) -> List[Dict[str, Any]]:
"""Standalone function for news search"""
tools = SearchTools()
return tools.search_news(query, max_results)
|