import os import sys import json import shutil import joblib import pandas as pd from pathlib import Path from datetime import datetime from sklearn.pipeline import Pipeline from sklearn.ensemble import VotingClassifier from sklearn.model_selection import cross_validate from sklearn.linear_model import LogisticRegression from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score, f1_score from sklearn.feature_extraction.text import TfidfVectorizer # Import the new path manager# Cal try: from path_config import path_manager except ImportError: # Add current directory to path sys.path.append(os.path.dirname(os.path.abspath(__file__))) from path_config import path_manager def log_step(message): """Log initialization steps""" print(f"[{datetime.now().strftime('%H:%M:%S')}] {message}") def create_directories(): """Create necessary directories""" log_step("Creating directory structure...") # Directories are already created by path_manager initialization directories = [ path_manager.get_data_path(), path_manager.get_model_path(), path_manager.get_logs_path(), path_manager.get_cache_path(), path_manager.get_temp_path() ] for dir_path in directories: if dir_path.exists(): log_step(f"✅ Directory exists: {dir_path}") else: try: dir_path.mkdir(parents=True, exist_ok=True) log_step(f"✅ Created directory: {dir_path}") except Exception as e: log_step(f"⚠️ Failed to create {dir_path}: {e}") return False # Create kaggle subdirectory kaggle_dir = path_manager.get_data_path('kaggle') kaggle_dir.mkdir(parents=True, exist_ok=True) log_step(f"✅ Created kaggle directory: {kaggle_dir}") return True def check_existing_datasets(): """Check for existing datasets in the project structure""" log_step("Checking for existing datasets...") # Check for datasets in the current project structure base_dir = path_manager.base_paths['base'] # Possible source locations source_locations = [ base_dir / "data" / "kaggle" / "Fake.csv", base_dir / "data" / "kaggle" / "True.csv", base_dir / "data" / "combined_dataset.csv" ] found_files = [] for source_file in source_locations: if source_file.exists(): found_files.append(source_file) log_step(f"✅ Found existing dataset: {source_file}") return found_files def copy_existing_datasets(): """Copy existing datasets if they're not in the target location""" log_step("Copying existing datasets to target locations...") base_dir = path_manager.base_paths['base'] target_data_dir = path_manager.get_data_path() # Define source-target pairs copy_operations = [ (base_dir / "data" / "kaggle" / "Fake.csv", target_data_dir / "kaggle" / "Fake.csv"), (base_dir / "data" / "kaggle" / "True.csv", target_data_dir / "kaggle" / "True.csv"), (base_dir / "data" / "combined_dataset.csv", target_data_dir / "combined_dataset.csv") ] copied_count = 0 for source, target in copy_operations: # Skip if source and target are the same (already in correct location) if source == target: if source.exists(): log_step(f"✅ Dataset already in correct location: {target}") copied_count += 1 continue if source.exists(): try: # Ensure target directory exists target.parent.mkdir(parents=True, exist_ok=True) # Copy file shutil.copy2(source, target) log_step(f"✅ Copied {source} → {target}") copied_count += 1 except Exception as e: log_step(f"⚠️ Failed to copy {source}: {e}") else: log_step(f"⚠️ Source file not found: {source}") return copied_count > 0 def create_minimal_dataset(): """Create a minimal dataset if no existing dataset is found""" log_step("Creating minimal dataset...") combined_path = path_manager.get_combined_dataset_path() if combined_path.exists(): log_step(f"✅ Combined dataset already exists: {combined_path}") return True try: # Create minimal training data with diverse examples minimal_data = pd.DataFrame({ 'text': [ # Real news examples 'Scientists at MIT have developed a new renewable energy technology that could revolutionize solar power generation.', 'The Federal Reserve announced interest rate decisions following their latest economic review meeting.', 'Local authorities report significant improvements in air quality following new environmental regulations.', 'Research published in Nature journal reveals new insights about climate change adaptation strategies.', 'Economic indicators show steady growth in the manufacturing sector across multiple regions.', 'Healthcare officials recommend updated vaccination schedules based on latest medical research findings.', 'Transportation department announces infrastructure improvements for major highway systems nationwide.', 'Educational institutions implement new digital learning platforms to enhance student engagement.', 'Agricultural experts develop drought-resistant crop varieties to improve food security globally.', 'Technology companies invest heavily in cybersecurity measures to protect user data privacy.', # Fake news examples 'SHOCKING: Government officials secretly planning to control population through mind control technology.', 'EXCLUSIVE: Celebrities caught in massive alien communication scandal that mainstream media won\'t report.', 'BREAKING: Scientists discover time travel but government hiding the truth from public knowledge.', 'EXPOSED: Pharmaceutical companies deliberately spreading diseases to increase their massive profits.', 'URGENT: Social media platforms using secret algorithms to brainwash users into political compliance.', 'LEAKED: Banking system about to collapse completely, insiders reveal financial catastrophe coming soon.', 'CONFIRMED: Weather modification technology being used to create artificial natural disasters worldwide.', 'REVEALED: Food companies adding dangerous chemicals that cause instant health problems and addiction.', 'CONSPIRACY: Educational system designed to suppress critical thinking and create obedient citizens.', 'TRUTH: Technology giants working with foreign powers to undermine national sovereignty completely.' ], 'label': [ # Real news labels (0) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # Fake news labels (1) 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ] }) # Save the dataset minimal_data.to_csv(combined_path, index=False) log_step(f"✅ Created minimal dataset with {len(minimal_data)} samples at {combined_path}") # Verify the file was created correctly if combined_path.exists(): df_check = pd.read_csv(combined_path) log_step(f"✅ Verified dataset: {len(df_check)} rows loaded successfully") return True else: log_step("❌ Failed to verify created dataset") return False except Exception as e: log_step(f"❌ Failed to create minimal dataset: {str(e)}") return False def run_initial_training(): """Run basic model training""" log_step("Starting initial model training...") try: # Get all the paths model_path = path_manager.get_model_file_path() vectorizer_path = path_manager.get_vectorizer_path() pipeline_path = path_manager.get_pipeline_path() log_step(f"Model path: {model_path}") log_step(f"Vectorizer path: {vectorizer_path}") log_step(f"Pipeline path: {pipeline_path}") # Check if model already exists if pipeline_path.exists() or (model_path.exists() and vectorizer_path.exists()): log_step("✅ Model files already exist, checking if pipeline needs to be created...") # If individual components exist but pipeline doesn't, create pipeline if model_path.exists() and vectorizer_path.exists() and not pipeline_path.exists(): log_step("Creating pipeline from existing components...") try: # Load existing components # model = joblib.load(model_path) vectorizer = joblib.load(vectorizer_path) # Create pipeline # pipeline = Pipeline([ # ('vectorizer', vectorizer), # ('model', model) # ]) # Create ensemble method pipeline # Initialize ensemble model ensemble_model = VotingClassifier( estimators=[ ('logistic', LogisticRegression(max_iter=1000, random_state=42, class_weight='balanced')), ('random_forest', RandomForestClassifier(n_estimators=50, random_state=42, class_weight='balanced')) ], voting='soft' ) pipeline = Pipeline([ ('vectorizer', TfidfVectorizer(...)), ('model', ensemble_model) # Use ensemble instead of single model ]) # Save pipeline joblib.dump(pipeline, pipeline_path) log_step(f"✅ Created pipeline from existing components: {pipeline_path}") except Exception as e: log_step(f"⚠️ Failed to create pipeline from existing components: {e}") return True # Load dataset dataset_path = path_manager.get_combined_dataset_path() if not dataset_path.exists(): log_step("❌ No dataset available for training") return False df = pd.read_csv(dataset_path) log_step(f"Loaded dataset with {len(df)} samples") # Validate dataset if len(df) < 10: log_step("❌ Dataset too small for training") return False # Prepare data X = df['text'].values y = df['label'].values # Check class distribution class_counts = pd.Series(y).value_counts() log_step(f"Class distribution: {class_counts.to_dict()}") # Train-test split X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=42, stratify=y if len(class_counts) > 1 else None ) # Create pipeline with preprocessing pipeline = Pipeline([ ('vectorizer', TfidfVectorizer( max_features=5000, stop_words='english', ngram_range=(1, 2), min_df=1, max_df=0.95 )), ('model', LogisticRegression( max_iter=1000, random_state=42, class_weight='balanced' )) ]) # Train model with cross-validation log_step("Training model with cross-validation...") # Perform cross-validation before final training cv_results = cross_validate( pipeline, X_train, y_train, cv=3, scoring=['accuracy', 'f1_weighted', 'precision_weighted', 'recall_weighted'], return_train_score=True ) # Train final model on all training data pipeline.fit(X_train, y_train) # Evaluate on test set y_pred = pipeline.predict(X_test) accuracy = accuracy_score(y_test, y_pred) f1 = f1_score(y_test, y_pred, average='weighted') # Calculate quality indicators directly from cv_results train_acc_mean = float(cv_results['train_accuracy'].mean()) test_acc_mean = float(cv_results['test_accuracy'].mean()) test_acc_std = float(cv_results['test_accuracy'].std()) overfitting_score = train_acc_mean - test_acc_mean stability_score = 1 - (test_acc_std / test_acc_mean) if test_acc_mean > 0 else 0 # Save CV results for API access cv_data = { "methodology": { "n_splits": 3, "cv_type": "StratifiedKFold", "random_state": 42 }, "test_scores": { "accuracy": { "mean": test_acc_mean, "std": test_acc_std, "scores": cv_results['test_accuracy'].tolist() }, "f1": { "mean": float(cv_results['test_f1_weighted'].mean()), "std": float(cv_results['test_f1_weighted'].std()), "scores": cv_results['test_f1_weighted'].tolist() } }, "train_scores": { "accuracy": { "mean": train_acc_mean, "std": float(cv_results['train_accuracy'].std()), "scores": cv_results['train_accuracy'].tolist() } }, "performance_indicators": { "overfitting_score": overfitting_score, "stability_score": stability_score } } # Save CV results cv_results_path = path_manager.get_logs_path("cv_results.json") with open(cv_results_path, 'w') as f: json.dump(cv_data, f, indent=2) log_step(f"Saved CV results to: {cv_results_path}") # Ensure model directory exists model_path.parent.mkdir(parents=True, exist_ok=True) # Save complete pipeline FIRST (this is the priority) log_step(f"Saving pipeline to: {pipeline_path}") joblib.dump(pipeline, pipeline_path) # Verify pipeline was saved if pipeline_path.exists(): log_step(f"✅ Pipeline saved successfully to {pipeline_path}") # Test loading the pipeline try: test_pipeline = joblib.load(pipeline_path) test_pred = test_pipeline.predict(["This is a test"]) log_step(f"✅ Pipeline verification successful: {test_pred}") except Exception as e: log_step(f"⚠️ Pipeline verification failed: {e}") else: log_step(f"❌ Pipeline was not saved to {pipeline_path}") # Save individual components for backward compatibility try: joblib.dump(pipeline.named_steps['model'], model_path) joblib.dump(pipeline.named_steps['vectorizer'], vectorizer_path) log_step(f"✅ Saved individual components") except Exception as e: log_step(f"⚠️ Failed to save individual components: {e}") # Save metadata metadata = { "model_version": "v1.0_init", "model_type": "ensemble_voting_pipeline", # "logistic_regression_pipeline", "test_accuracy": float(accuracy), "test_f1": float(f1), "train_size": len(X_train), "test_size": len(X_test), "timestamp": datetime.now().isoformat(), "training_method": "initialization", "environment": path_manager.environment, "data_path": str(dataset_path), "class_distribution": class_counts.to_dict(), "pipeline_created": pipeline_path.exists(), "individual_components_created": model_path.exists() and vectorizer_path.exists(), "cv_f1_mean": float(cv_results['test_f1_weighted'].mean()), "cv_f1_std": float(cv_results['test_f1_weighted'].std()), "cv_accuracy_mean": float(cv_results['test_accuracy'].mean()), "cv_accuracy_std": float(cv_results['test_accuracy'].std()), "overfitting_score": cv_data['performance_indicators']['overfitting_score'], "stability_score": cv_data['performance_indicators']['stability_score'] } metadata_path = path_manager.get_metadata_path() with open(metadata_path, 'w') as f: json.dump(metadata, f, indent=2) log_step(f"✅ Training completed successfully") log_step(f" Accuracy: {accuracy:.4f}") log_step(f" F1 Score: {f1:.4f}") log_step(f" Pipeline saved: {pipeline_path.exists()}") log_step(f" Model saved to: {model_path}") log_step(f" Vectorizer saved to: {vectorizer_path}") return True except Exception as e: log_step(f"❌ Training failed: {str(e)}") import traceback log_step(f"❌ Traceback: {traceback.format_exc()}") return False def create_initial_logs(): """Create initial log files""" log_step("Creating initial log files...") try: # Activity log activity_log = [{ "timestamp": datetime.now().strftime("%Y-%m-%d %I:%M %p"), "event": "System initialized successfully", "level": "INFO", "environment": path_manager.environment }] activity_log_path = path_manager.get_activity_log_path() with open(activity_log_path, 'w') as f: json.dump(activity_log, f, indent=2) log_step(f"✅ Created activity log: {activity_log_path}") # Create monitoring directory structure monitor_dir = path_manager.get_logs_path("monitor") monitor_dir.mkdir(parents=True, exist_ok=True) log_step(f"✅ Created monitor directory: {monitor_dir}") # Create empty monitoring logs monitoring_log_path = path_manager.get_logs_path("monitoring_log.json") with open(monitoring_log_path, 'w') as f: json.dump([], f) log_step(f"✅ Created monitoring log: {monitoring_log_path}") # Create monitor-specific log files monitor_log_files = [ "monitor/predictions.json", "monitor/metrics.json", "monitor/alerts.json" ] for log_file in monitor_log_files: log_path = path_manager.get_logs_path(log_file) log_path.parent.mkdir(parents=True, exist_ok=True) # Ensure parent dir exists if not log_path.exists(): with open(log_path, 'w') as f: json.dump([], f) log_step(f"✅ Created {log_file}") # Create other necessary log files log_files = [ "drift_history.json", "drift_alerts.json", "scheduler_execution.json", "scheduler_errors.json" ] for log_file in log_files: log_path = path_manager.get_logs_path(log_file) if not log_path.exists(): with open(log_path, 'w') as f: json.dump([], f) log_step(f"✅ Created {log_file}") return True except Exception as e: log_step(f"❌ Log creation failed: {str(e)}") return False def verify_system(): """Verify that the system is properly initialized""" log_step("Verifying system initialization...") # Check critical files critical_files = [ (path_manager.get_combined_dataset_path(), "Combined dataset"), (path_manager.get_model_file_path(), "Model file"), (path_manager.get_vectorizer_path(), "Vectorizer file"), (path_manager.get_pipeline_path(), "Pipeline file"), (path_manager.get_metadata_path(), "Metadata file"), (path_manager.get_activity_log_path(), "Activity log") ] all_good = True for file_path, description in critical_files: if file_path.exists(): log_step(f"✅ {description}: {file_path}") else: log_step(f"❌ Missing {description}: {file_path}") if description == "Pipeline file": # Pipeline is critical, mark as not all good all_good = False # Test model loading - prioritize pipeline try: import joblib pipeline_path = path_manager.get_pipeline_path() if pipeline_path.exists(): pipeline = joblib.load(pipeline_path) test_pred = pipeline.predict(["This is a test text"]) log_step(f"✅ Pipeline test prediction successful: {test_pred}") else: log_step("⚠️ Pipeline not available, testing individual components...") model_path = path_manager.get_model_file_path() vectorizer_path = path_manager.get_vectorizer_path() if model_path.exists() and vectorizer_path.exists(): model = joblib.load(model_path) vectorizer = joblib.load(vectorizer_path) test_text_vec = vectorizer.transform(["This is a test text"]) test_pred = model.predict(test_text_vec) log_step(f"✅ Individual components test prediction successful: {test_pred}") else: log_step("❌ No working model components found") all_good = False except Exception as e: log_step(f"❌ Model test failed: {e}") all_good = False return all_good def main(): """Main initialization function""" log_step("🚀 Starting system initialization...") log_step(f"🌍 Environment: {path_manager.environment}") log_step(f"📁 Base directory: {path_manager.base_paths['base']}") log_step(f"📊 Data directory: {path_manager.base_paths['data']}") log_step(f"🤖 Model directory: {path_manager.base_paths['model']}") steps = [ ("Directory Creation", create_directories), ("Existing Dataset Copy", copy_existing_datasets), ("Minimal Dataset Creation", create_minimal_dataset), ("Model Training", run_initial_training), ("Log File Creation", create_initial_logs), ("System Verification", verify_system) ] failed_steps = [] completed_steps = [] for step_name, step_function in steps: try: log_step(f"🔄 Starting: {step_name}") if step_function(): log_step(f"✅ {step_name} completed") completed_steps.append(step_name) else: log_step(f"❌ {step_name} failed") failed_steps.append(step_name) except Exception as e: log_step(f"❌ {step_name} failed: {str(e)}") failed_steps.append(step_name) # Summary log_step(f"\n📊 Initialization Summary:") log_step(f" ✅ Completed: {len(completed_steps)}/{len(steps)} steps") log_step(f" ❌ Failed: {len(failed_steps)}/{len(steps)} steps") if completed_steps: log_step(f" Completed steps: {', '.join(completed_steps)}") if failed_steps: log_step(f" Failed steps: {', '.join(failed_steps)}") log_step(f"⚠️ Initialization completed with {len(failed_steps)} failed steps") else: log_step("🎉 System initialization completed successfully!") # Environment info log_step(f"\n🔍 Environment Information:") env_info = path_manager.get_environment_info() log_step(f" Environment: {env_info['environment']}") log_step(f" Available datasets: {sum(env_info['available_datasets'].values())}") log_step(f" Available models: {sum(env_info['available_models'].values())}") # Final pipeline check pipeline_path = path_manager.get_pipeline_path() log_step(f"\n🎯 Final Pipeline Check:") log_step(f" Pipeline path: {pipeline_path}") log_step(f" Pipeline exists: {pipeline_path.exists()}") if pipeline_path.exists(): try: import joblib pipeline = joblib.load(pipeline_path) log_step(f" Pipeline loadable: ✅") log_step(f" Pipeline steps: {list(pipeline.named_steps.keys())}") except Exception as e: log_step(f" Pipeline load error: {e}") log_step("\n🎯 System ready for use!") return len(failed_steps) == 0 if __name__ == "__main__": success = main() if not success: sys.exit(1)