{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "11ab9ea6-1c5b-4f9b-a6ea-1bc75be56108", "metadata": {}, "outputs": [], "source": [ "import os\n", "import warnings\n", "\n", "import joblib\n", "import numpy as np\n", "import pandas as pd\n", "import torch\n", "from sklearn.metrics import mean_absolute_error\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.preprocessing import StandardScaler\n", "from torch import nn\n", "from torch.optim import AdamW\n", "from torch.optim.lr_scheduler import LinearLR\n", "from torch.utils.data import DataLoader, Dataset\n", "from tqdm import tqdm\n", "from transformers import (\n", " AutoConfig,\n", " AutoModel,\n", " AutoTokenizer,\n", " BertConfig,\n", " BertModel,\n", " BertTokenizerFast,\n", " PreTrainedModel,\n", ")\n", "from transformers.activations import ACT2FN\n", "\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "torch.backends.cuda.matmul.allow_tf32 = True\n", "torch.backends.cudnn.allow_tf32 = True\n", "\n", "def global_ap(x):\n", " return torch.mean(x.view(x.size(0), x.size(1), -1), dim=1)\n", "\n", "class SimSonEncoder(nn.Module):\n", " def __init__(self, config: BertConfig, max_len: int, dropout: float = 0.1):\n", " super(SimSonEncoder, self).__init__()\n", " self.config = config\n", " self.max_len = max_len\n", " \n", " self.bert = BertModel(config, add_pooling_layer=False)\n", " \n", " self.linear = nn.Linear(config.hidden_size, max_len)\n", " self.dropout = nn.Dropout(dropout)\n", " \n", " def forward(self, input_ids, attention_mask=None):\n", " if attention_mask is None:\n", " attention_mask = input_ids.ne(0)\n", " \n", " outputs = self.bert(\n", " input_ids=input_ids,\n", " attention_mask=attention_mask\n", " )\n", " \n", " hidden_states = outputs.last_hidden_state\n", " \n", " hidden_states = self.dropout(hidden_states)\n", " \n", " pooled = global_ap(hidden_states)\n", " \n", " out = self.linear(pooled)\n", " \n", " return out\n", "\n", "class SimSonClassifier(nn.Module):\n", " def __init__(self, encoder: SimSonEncoder, num_labels: int, dropout=0.1):\n", " super(SimSonClassifier, self).__init__()\n", " self.encoder = encoder\n", " self.clf = nn.Linear(encoder.max_len, num_labels)\n", " self.relu = nn.ReLU()\n", " self.dropout = nn.Dropout(dropout)\n", "\n", " def forward(self, input_ids, attention_mask=None, labels=None):\n", " x = self.encoder(input_ids, attention_mask)\n", " x = self.relu(self.dropout(x))\n", " x = self.clf(x)\n", " return x" ] }, { "cell_type": "code", "execution_count": 10, "id": "ce760993-fbef-4546-8b2c-1e7a722ad374", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import torch\n", "from torch.utils.data import Dataset\n", "\n", "\n", "class SMILESDataset(Dataset):\n", " def __init__(self, smiles_list, labels, tokenizer, max_length=256):\n", " self.smiles_list = smiles_list\n", " self.labels = labels # Shape: (num_samples, 2) - already scaled\n", " self.tokenizer = tokenizer\n", " self.max_length = max_length\n", " \n", " # Create mask for valid (non-NaN) labels\n", " self.label_masks = ~np.isnan(self.labels) # True where label is valid\n", " \n", " # Replace NaNs with 0 for safe tensor conversion (mask will handle exclusion)\n", " self.labels = np.nan_to_num(self.labels, nan=0.0)\n", " \n", " def __len__(self):\n", " return len(self.smiles_list)\n", " \n", " def __getitem__(self, idx):\n", " smiles = self.tokenizer.cls_token + self.smiles_list[idx]\n", " \n", " # Tokenize the SMILES string\n", " encoding = self.tokenizer(\n", " smiles,\n", " truncation=True,\n", " padding='max_length',\n", " max_length=self.max_length,\n", " return_tensors='pt'\n", " )\n", " \n", " return {\n", " 'input_ids': encoding['input_ids'].flatten(),\n", " 'attention_mask': encoding['attention_mask'].flatten(),\n", " 'labels': torch.tensor(self.labels[idx], dtype=torch.float32),\n", " 'label_mask': torch.tensor(self.label_masks[idx], dtype=torch.float32)\n", " }\n", " \n", " def get_label_statistics(self):\n", " \"\"\"Return statistics about label availability\"\"\"\n", " label_counts = self.label_masks.sum(axis=0)\n", " total_samples = len(self.smiles_list)\n", " \n", " stats = {\n", " 'total_samples': total_samples,\n", " 'label_0_count': label_counts[0],\n", " 'label_1_count': label_counts[1],\n", " 'label_0_ratio': label_counts[0] / total_samples,\n", " 'label_1_ratio': label_counts[1] / total_samples,\n", " 'both_labels_count': (self.label_masks.sum(axis=1) == 2).sum(),\n", " 'single_label_count': (self.label_masks.sum(axis=1) == 1).sum(),\n", " 'no_labels_count': (self.label_masks.sum(axis=1) == 0).sum()\n", " }\n", " \n", " return stats\n", "\n", "def calculate_weighted_loss(predictions, labels, label_mask, label_weights):\n", " \"\"\"\n", " Calculate weighted loss for two labels with masking\n", " \n", " Args:\n", " predictions: Model outputs (batch_size, 2)\n", " labels: Ground truth labels (batch_size, 2)\n", " label_mask: Mask for valid labels (batch_size, 2)\n", " label_weights: Weights for each label (2,)\n", " \"\"\"\n", " loss_fn = nn.MSELoss(reduction='none')\n", " \n", " # Calculate per-sample, per-label losses\n", " losses = loss_fn(predictions, labels) # Shape: (batch_size, 2)\n", " \n", " # Apply masking to exclude NaN labels\n", " valid_mask = label_mask.bool()\n", " masked_losses = losses * valid_mask.float()\n", " \n", " # Apply label-specific weights\n", " weighted_losses = masked_losses * label_weights.unsqueeze(0) # Broadcast weights\n", " \n", " # Calculate final loss (only over valid predictions)\n", " total_loss = weighted_losses.sum()\n", " total_valid = valid_mask.sum()\n", " \n", " return total_loss / total_valid if total_valid > 0 else torch.tensor(0.0, device=predictions.device, requires_grad=True)\n", "\n", "def compute_label_weights(dataset):\n", " \"\"\"\n", " Compute inverse frequency weights based on label availability\n", " \n", " Args:\n", " dataset: SMILESDataset instance\n", " \n", " Returns:\n", " torch.Tensor: Normalized weights for each label\n", " \"\"\"\n", " # Get label counts from dataset\n", " label_counts = dataset.label_masks.sum(axis=0) # Count valid samples per label\n", " total_samples = len(dataset)\n", " \n", " # Inverse frequency weighting\n", " weights = total_samples / (2 * label_counts) # 2 is the number of labels\n", " \n", " # Normalize weights so they sum to number of labels (2)\n", " weights = weights / weights.sum() * 2\n", " \n", " return torch.tensor(weights, dtype=torch.float32)\n", "\n", "def calculate_true_loss(predictions, labels, label_mask, scalers=None):\n", " \"\"\"\n", " Calculate unscaled MAE loss for monitoring using separate scalers for each label\n", " \n", " Args:\n", " predictions (torch.Tensor): Model outputs of shape (batch_size, 2).\n", " labels (torch.Tensor): Ground truth labels of shape (batch_size, 2).\n", " label_mask (torch.Tensor): Boolean mask for valid labels of shape (batch_size, 2).\n", " scalers: List of scaler objects, one for each label\n", " \"\"\"\n", " # Detach tensors from the computation graph and move to CPU\n", " predictions_np = predictions.cpu().detach().numpy()\n", " labels_np = labels.cpu().numpy()\n", " label_mask_np = label_mask.cpu().numpy().astype(bool)\n", " \n", " total_mae = 0\n", " total_samples = 0\n", " \n", " for label_idx in range(2):\n", " # Get valid samples for this label\n", " valid_mask = label_mask_np[:, label_idx]\n", " \n", " if valid_mask.any():\n", " valid_preds = predictions_np[valid_mask, label_idx].reshape(-1, 1)\n", " valid_labels = labels_np[valid_mask, label_idx].reshape(-1, 1)\n", " \n", " if scalers is not None:\n", " # Unscale using the corresponding scaler for this label\n", " unscaled_preds = scalers[label_idx].inverse_transform(valid_preds).flatten()\n", " unscaled_labels = scalers[label_idx].inverse_transform(valid_labels).flatten()\n", " else:\n", " unscaled_preds = valid_preds.flatten()\n", " unscaled_labels = valid_labels.flatten()\n", " \n", " # Calculate MAE for this label\n", " mae = np.mean(np.abs(unscaled_preds - unscaled_labels))\n", " total_mae += mae * len(unscaled_preds)\n", " total_samples += len(unscaled_preds)\n", " \n", " return total_mae / total_samples if total_samples > 0 else 0.0\n", "\n", "\n", "def train_model(model, train_dataloader, val_dataloader, label_weights, \n", " scalers=None, num_epochs=10, learning_rate=2e-5, device='cuda', \n", " patience=3, validation_steps=500):\n", " \"\"\"\n", " Train model with weighted loss for two labels with step-based validation\n", " \n", " Args:\n", " model: CustomModel instance (should output 2 labels)\n", " train_dataloader: Training data loader\n", " val_dataloader: Validation data loader \n", " label_weights: Tensor with weights for each label\n", " scalers: List of scalers for unscaled loss monitoring\n", " num_epochs: Number of training epochs\n", " learning_rate: Learning rate\n", " device: Training device\n", " patience: Early stopping patience (in validation steps)\n", " validation_steps: Perform validation every N training steps\n", " \"\"\"\n", " model.to(device)\n", " label_weights = label_weights.to(device)\n", " \n", " optimizer = AdamW(model.parameters(), lr=learning_rate, weight_decay=0.01)\n", " total_steps = len(train_dataloader) * num_epochs\n", " scheduler = LinearLR(optimizer, start_factor=1.0, end_factor=0.1, total_iters=total_steps)\n", " \n", " train_losses = []\n", " val_losses = []\n", " \n", " # Early stopping initialization\n", " best_val_loss = float('inf')\n", " steps_no_improve = 0\n", " best_model_state = None\n", " \n", " # Training tracking\n", " global_step = 0\n", " running_train_loss = 0\n", " running_true_train_loss = 0\n", " train_steps_count = 0\n", " \n", " print(f\"Label weights: {label_weights.cpu().numpy()}\")\n", " print(f\"Validation will be performed every {validation_steps} steps\")\n", " \n", " model.train()\n", " \n", " for epoch in range(num_epochs):\n", " print(f\"\\nEpoch {epoch + 1}/{num_epochs}\")\n", " \n", " train_progress = tqdm(train_dataloader, desc=\"Training\", leave=False)\n", " \n", " for batch_idx, batch in enumerate(train_progress):\n", " with torch.autocast(dtype=torch.float16, device_type=\"cuda\"):\n", " input_ids = batch['input_ids'].to(device)\n", " attention_mask = batch['attention_mask'].to(device)\n", " labels = batch['labels'].to(device)\n", " label_mask = batch['label_mask'].to(device)\n", " \n", " optimizer.zero_grad()\n", " \n", " # Model forward pass\n", " outputs = model(\n", " input_ids=input_ids,\n", " attention_mask=attention_mask,\n", " )\n", " \n", " # Calculate weighted loss\n", " loss = calculate_weighted_loss(outputs, labels, label_mask, label_weights)\n", " \n", " # Calculate true loss for monitoring\n", " true_loss = calculate_true_loss(outputs, labels, label_mask, scalers)\n", " \n", " # Accumulate losses for averaging\n", " running_train_loss += loss.item()\n", " running_true_train_loss += true_loss\n", " train_steps_count += 1\n", " \n", " loss.backward()\n", " \n", " torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n", " \n", " optimizer.step()\n", " scheduler.step()\n", " \n", " global_step += 1\n", " \n", " train_progress.set_postfix({\n", " 'step': global_step,\n", " 'loss': f'{loss.item():.4f}',\n", " 'true_loss': f'{true_loss:.4f}',\n", " 'lr': f'{scheduler.get_last_lr()[0]:.2e}'\n", " })\n", " \n", " # Perform validation every validation_steps\n", " if global_step % validation_steps == 0:\n", " # Calculate average training losses since last validation\n", " avg_train_loss = running_train_loss / train_steps_count\n", " avg_true_train_loss = running_true_train_loss / train_steps_count\n", " \n", " train_losses.append(avg_train_loss)\n", " \n", " # Reset running averages\n", " running_train_loss = 0\n", " running_true_train_loss = 0\n", " train_steps_count = 0\n", " \n", " # Validation\n", " model.eval()\n", " total_val_loss = 0\n", " total_true_val_loss = 0\n", " val_batches = 0\n", "\n", " with torch.no_grad():\n", " for val_batch in val_dataloader:\n", " with torch.autocast(dtype=torch.float16, device_type=\"cuda\"):\n", " input_ids = val_batch['input_ids'].to(device)\n", " attention_mask = val_batch['attention_mask'].to(device)\n", " labels = val_batch['labels'].to(device)\n", " label_mask = val_batch['label_mask'].to(device)\n", " \n", " outputs = model(\n", " input_ids=input_ids,\n", " attention_mask=attention_mask,\n", " )\n", " \n", " val_loss = calculate_weighted_loss(outputs, labels, label_mask, label_weights)\n", " val_true_loss = calculate_true_loss(outputs, labels, label_mask, scalers)\n", "\n", " total_val_loss += val_loss.item()\n", " total_true_val_loss += val_true_loss\n", " val_batches += 1\n", " \n", " avg_val_loss = total_val_loss / val_batches\n", " avg_val_true_loss = total_true_val_loss / val_batches\n", " val_losses.append(avg_val_loss)\n", " \n", " print(f\"\\nStep {global_step} | Train Loss: {avg_train_loss:.4f} | Val Loss: {avg_val_loss:.4f} | True train loss: {avg_true_train_loss:.4f} | True val loss: {avg_val_true_loss:.4f}\")\n", " \n", " # Early stopping check and best model saving\n", " if avg_val_loss < best_val_loss:\n", " best_val_loss = avg_val_loss\n", " steps_no_improve = 0\n", " best_model_state = model.state_dict().copy()\n", " print(f\"New best validation loss: {best_val_loss:.4f}\")\n", " else:\n", " steps_no_improve += 1\n", " if steps_no_improve >= patience:\n", " print(f\"Early stopping triggered after {global_step} steps ({steps_no_improve} validation steps without improvement).\")\n", " # Load best model and return\n", " if best_model_state is not None:\n", " model.load_state_dict(best_model_state)\n", " print(f\"Loaded best model with validation loss: {best_val_loss:.4f}\")\n", " return train_losses, val_losses, best_val_loss\n", " \n", " model.train()\n", " \n", " # Handle any remaining training loss that hasn't been validated\n", " if train_steps_count > 0:\n", " avg_train_loss = running_train_loss / train_steps_count\n", " train_losses.append(avg_train_loss)\n", " \n", " # Load the best model state before returning\n", " if best_model_state is not None:\n", " model.load_state_dict(best_model_state)\n", " print(f\"Loaded best model with validation loss: {best_val_loss:.4f}\")\n", " \n", " return train_losses, val_losses, best_val_loss\n", "\n", "def run_training(smiles_train, smiles_test, labels_train, labels_test, \n", " model, tokenizer, scalers, num_epochs=5, learning_rate=1e-5, \n", " batch_size=256, validation_steps=500):\n", " \"\"\"\n", " Complete training pipeline for two labels with step-based validation\n", " \n", " Args:\n", " smiles_train, smiles_test: Lists of SMILES strings\n", " labels_train, labels_test: numpy arrays of shape (num_samples, 2) - ALREADY SCALED\n", " model: CustomModel instance (configured for 2 outputs)\n", " tokenizer: Tokenizer instance\n", " scalers: List of 2 scalers, one for each label (for inverse transform only)\n", " num_epochs: Number of training epochs\n", " learning_rate: Learning rate\n", " batch_size: Batch size for training\n", " validation_steps: Perform validation every N training steps\n", " \"\"\"\n", " \n", " print(\"Setting up datasets for two-label training (labels assumed pre-scaled)\")\n", " \n", " # Create datasets - no scaling performed here\n", " train_dataset = SMILESDataset(smiles_train, labels_train, tokenizer)\n", " val_dataset = SMILESDataset(smiles_test, labels_test, tokenizer)\n", " \n", " # Print dataset statistics\n", " train_stats = train_dataset.get_label_statistics()\n", " val_stats = val_dataset.get_label_statistics()\n", " \n", " print(\"Training dataset statistics:\")\n", " for key, value in train_stats.items():\n", " print(f\" {key}: {value}\")\n", " \n", " print(\"Validation dataset statistics:\")\n", " for key, value in val_stats.items():\n", " print(f\" {key}: {value}\")\n", " \n", " # Compute label weights based on training data\n", " label_weights = compute_label_weights(train_dataset)\n", " print(f\"Computed label weights: {label_weights.numpy()}\")\n", " \n", " # Create data loaders\n", " train_dataloader = DataLoader(\n", " train_dataset,\n", " batch_size=batch_size,\n", " shuffle=True,\n", " num_workers=4,\n", " pin_memory=True\n", " )\n", " \n", " val_dataloader = DataLoader(\n", " val_dataset,\n", " batch_size=batch_size,\n", " shuffle=False,\n", " num_workers=4,\n", " pin_memory=True\n", " )\n", " \n", " # Set device\n", " device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", " print(f\"Using device: {device}\")\n", " print(f\"Training steps per epoch: {len(train_dataloader)}\")\n", " print(f\"Total training steps: {len(train_dataloader) * num_epochs}\")\n", " \n", " # Train the model\n", " train_losses, val_losses, best_val_loss = train_model(\n", " model=model,\n", " train_dataloader=train_dataloader,\n", " val_dataloader=val_dataloader,\n", " label_weights=label_weights,\n", " scalers=scalers, # Still pass scalers for true loss calculation\n", " num_epochs=num_epochs,\n", " learning_rate=learning_rate,\n", " device=device,\n", " patience=10,\n", " validation_steps=validation_steps,\n", " )\n", " \n", " print('Training completed.')\n", " print(f'Number of validation checkpoints: {len(val_losses)}')\n", " print(f'Final training losses: {train_losses[-5:] if len(train_losses) >= 5 else train_losses}')\n", " print(f'Best validation loss: {best_val_loss:.4f}')\n", " \n", " # Save model\n", " torch.save(model.state_dict(), '/home/jovyan/simson_training_bolgov/regression/regression_simson.pth')\n", " print(\"Model saved successfully!\")\n", " \n", " return train_losses, val_losses, best_val_loss" ] }, { "cell_type": "code", "execution_count": 3, "id": "12a2b8c3-2c4d-4b1b-8cc7-930c9fe68fd7", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "df = pd.read_csv('/home/jovyan/simson_training_bolgov/regression/PI_Tg_P308K_synth_db_chem.csv')\n", "targets = ['Tg', 'He', 'N2', 'O2', 'CH4', 'CO2']" ] }, { "cell_type": "code", "execution_count": 6, "id": "8e1296a2-551c-48ab-aab3-fcf4b6110d75", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGvCAYAAAC3lbrBAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAYQxJREFUeJzt3XtcFdXeP/APSICZYOhPkCc1T8fjJW+pSZR1tHhEM0+ezKNl5THSUw+UynOs7DEyrSjzgheSzLyd5HjpJJUainjBC6KgJKLiDYXEDSrCBpTr3r8/jJENm32d2TN7z+f9eu3XC/asPbPmtuY7a9as5abX6/UgIiIiUiF3uTNAREREJBcGQkRERKRaDISIiIhItRgIERERkWoxECIiIiLVYiBEREREqsVAiIiIiFSLgRARERGplofcGVAynU6HgoICtG7dGm5ubnJnh4iIiCyg1+tRVlaGwMBAuLubrvNhIGRCQUEBOnbsKHc2iIiIyAb5+fl44IEHTKZhIGRC69atAdzZkD4+PjLnhoiIiCyh1WrRsWNH4TpuCgMhE+ofh/n4+DAQIiIicjKWNGthY2kiIiJSLQZCREREpFoMhIiIiEi1GAgRERGRajEQIiIiItViIERERESqxUCIiIiIVIuBEBEREakWAyEiIiJSLQZCREREpFoMhIiIiEi1GAgRERGRalkdCKWkpGDUqFEIDAyEm5sbEhISmqQ5ffo0/vKXv8DX1xetWrXCo48+iry8PGF6ZWUlwsPD0bZtW9x3330YM2YMCgsLDeaRl5eHkSNH4t5770X79u0xY8YM1NbWGqTZu3cv+vfvDy8vL/zxj3/EmjVrmuQlNjYWDz74ILy9vREUFIQjR45Yu8ouY29OEd77/gRuVdeaT2yHklvVmLH5Vxy+eEPS5UjhX4cv48sdZ5qdrtfrEf3Lafz7SJ7B96evahG5KRP5xbesXubS5HP4JuWi1b8jdbp8owKRmzJxtrBM7qwo0tf7LiB2z3mHLvNaWRX+d9OvyLh8U9T5nr6qxZ+/3IPxK1Jxo7xK1HnLqVBbif/d9Ct+zS+ROysAbAiEKioq0LdvX8TGxhqdfuHCBQwePBjdu3fH3r17ceLECXz44Yfw9vYW0kyfPh0///wzNm/ejH379qGgoAAvvPCCML2urg4jR45EdXU1Dh06hLVr12LNmjWIiooS0uTm5mLkyJEYOnQoMjMzMW3aNLzxxhvYsWOHkGbjxo2IjIzERx99hGPHjqFv374IDQ1FUVGRtavtEv6++ig2pufjqz0XJF1O9PYz2JzxG8avOCzpcqTwYcJJxO65gDMardHpv/5Wiq/3XcTMH7IMvh+19AB+OHYF//hXhlXLK9RWYkHSWXy6/TSqautszjepx6TVR/HDsSsY89UhubOiOLeqaxH9yxl8uSMH1x0YOMxKyMJ/jv2GMcvF3Scjl+zH5Ru3cPhiMWb/fErUecvpn5t/xX+O/YbnYw/KnRUAgIe1PxgxYgRGjBjR7PT/+7//w7PPPot58+YJ3z300EPC36Wlpfj2228RHx+Pp59+GgCwevVq9OjRA4cPH8Zjjz2GnTt34tSpU9i1axf8/f3Rr18/zJ07F++99x5mz54NT09PxMXFoUuXLliwYAEAoEePHjhw4AAWLVqE0NBQAMDChQsxefJkTJo0CQAQFxeHbdu2YdWqVXj//fetXXWXUVB6W9L5Xy6ukHT+jlBRZbzWTHu7xuj3tTo9AOBckXV36ZU1d4Mfvd6qn5JKXbx+5/wqa+YYVbP68xAAqmt1Dltu7nVpyrwGq4PLN5y/XK138Zqy1kXUNkI6nQ7btm3Dn/70J4SGhqJ9+/YICgoyeHyWkZGBmpoahISECN91794dnTp1QmpqKgAgNTUVvXv3hr+/v5AmNDQUWq0W2dnZQpqG86hPUz+P6upqZGRkGKRxd3dHSEiIkKaxqqoqaLVagw8RERG5LlEDoaKiIpSXl+Pzzz/H8OHDsXPnTvz1r3/FCy+8gH379gEANBoNPD090aZNG4Pf+vv7Q6PRCGkaBkH10+unmUqj1Wpx+/ZtXL9+HXV1dUbT1M+jsejoaPj6+gqfjh072rYhiIiIyCmIXiMEAM8//zymT5+Ofv364f3338dzzz2HuLg4MRcliZkzZ6K0tFT45Ofny50lIiIikpCogVC7du3g4eGBnj17Gnzfo0cP4a2xgIAAVFdXo6SkxCBNYWEhAgIChDSN3yKr/99cGh8fH7Rs2RLt2rVDixYtjKapn0djXl5e8PHxMfgQERGR6xI1EPL09MSjjz6KnJwcg+/Pnj2Lzp07AwAGDBiAe+65B8nJycL0nJwc5OXlITg4GAAQHByMrKwsg7e7kpKS4OPjIwRZwcHBBvOoT1M/D09PTwwYMMAgjU6nQ3JyspCGiIiI1M3qt8bKy8tx/vzdPhpyc3ORmZkJPz8/dOrUCTNmzMC4cePw1FNPYejQoUhMTMTPP/+MvXv3AgB8fX0RFhaGyMhI+Pn5wcfHB2+//TaCg4Px2GOPAQCGDRuGnj174tVXX8W8efOg0Wgwa9YshIeHw8vLCwDw5ptvYtmyZXj33Xfx+uuvY/fu3di0aRO2bdsm5C0yMhITJ07EwIEDMWjQIMTExKCiokJ4i4yIiIjUzepAKD09HUOHDhX+j4yMBABMnDgRa9aswV//+lfExcUhOjoa77zzDrp164b//Oc/GDx4sPCbRYsWwd3dHWPGjEFVVRVCQ0Px1VdfCdNbtGiBrVu34q233kJwcDBatWqFiRMnYs6cOUKaLl26YNu2bZg+fToWL16MBx54ACtXrhRenQeAcePG4dq1a4iKioJGo0G/fv2QmJjYpAE1ERERqZPVgdCQIUOgN9Phyeuvv47XX3+92ene3t6IjY1ttlNGAOjcuTO2b99uNi/Hjx83mSYiIgIREREm0xAREZE6cawxIiIiUi0GQkRERKRaDISIiIhItRgIERnlZuOvbPsdERHJg4EQERERqRYDIRVirYWEuGmJiJwKAyEiInIJDXt2ceNNCVmIgRARERGpFgMhIiN4N0lEpA4MhIiIiOzgiHaXvDeTDgMhIiIiUi0GQkRERKRaDISIrMC2Q0REroWBEBERkR14g+TcGAgRERGRajEQIjLC1hs8a3/HXr6JiOTFQIhIIRr2iktERI7BQIjICqzBISJyLQyEiIiISLUYCKkQ33AgIiK6g4EQkYgYZBIRORcGQkRE5HLYno8sxUCIiIhcA9+8JBswECIiIiLVYiBEREREqsVAiIiIiFSLgRCREW58/YuISBUYCBEZYftYYwygiIicCQMhIoVgJRQRNYsFhGQYCBEREZFqMRAiIiIi1bI6EEpJScGoUaMQGBgINzc3JCQkNJv2zTffhJubG2JiYgy+Ly4uxoQJE+Dj44M2bdogLCwM5eXlBmlOnDiBJ598Et7e3ujYsSPmzZvXZP6bN29G9+7d4e3tjd69e2P79u0G0/V6PaKiotChQwe0bNkSISEhOHfunLWrTCQZPXuAIyKSldWBUEVFBfr27YvY2FiT6bZs2YLDhw8jMDCwybQJEyYgOzsbSUlJ2Lp1K1JSUjBlyhRhularxbBhw9C5c2dkZGTgyy+/xOzZs7FixQohzaFDh/DSSy8hLCwMx48fx+jRozF69GicPHlSSDNv3jwsWbIEcXFxSEtLQ6tWrRAaGorKykprV5uIiEg+et40ScXD2h+MGDECI0aMMJnmypUrePvtt7Fjxw6MHDnSYNrp06eRmJiIo0ePYuDAgQCApUuX4tlnn8X8+fMRGBiI9evXo7q6GqtWrYKnpycefvhhZGZmYuHChULAtHjxYgwfPhwzZswAAMydOxdJSUlYtmwZ4uLioNfrERMTg1mzZuH5558HAKxbtw7+/v5ISEjA+PHjrV11l8Emd7Zje0UiItciehshnU6HV199FTNmzMDDDz/cZHpqairatGkjBEEAEBISAnd3d6SlpQlpnnrqKXh6egppQkNDkZOTg5s3bwppQkJCDOYdGhqK1NRUAEBubi40Go1BGl9fXwQFBQlpGquqqoJWqzX4EBERkesSPRD64osv4OHhgXfeecfodI1Gg/bt2xt85+HhAT8/P2g0GiGNv7+/QZr6/82laTi94e+MpWksOjoavr6+wqdjx45m15eoIdYYERE5F1EDoYyMDCxevBhr1qxxyp55Z86cidLSUuGTn58vd5aIiEjhnPF6R3eJGgjt378fRUVF6NSpEzw8PODh4YHLly/jf//3f/Hggw8CAAICAlBUVGTwu9raWhQXFyMgIEBIU1hYaJCm/n9zaRpOb/g7Y2ka8/Lygo+Pj8GHiIicD2MTspSogdCrr76KEydOIDMzU/gEBgZixowZ2LFjBwAgODgYJSUlyMjIEH63e/du6HQ6BAUFCWlSUlJQU1MjpElKSkK3bt1w//33C2mSk5MNlp+UlITg4GAAQJcuXRAQEGCQRqvVIi0tTUhDRERE6mb1W2Pl5eU4f/688H9ubi4yMzPh5+eHTp06oW3btgbp77nnHgQEBKBbt24AgB49emD48OGYPHky4uLiUFNTg4iICIwfP1541f7ll1/Gxx9/jLCwMLz33ns4efIkFi9ejEWLFgnznTp1Kv785z9jwYIFGDlyJDZs2ID09HThFXs3NzdMmzYNn3zyCbp27YouXbrgww8/RGBgIEaPHm31hiJ1sfVu0tqfcWwyIiJ5WR0IpaenY+jQocL/kZGRAICJEydizZo1Fs1j/fr1iIiIwDPPPAN3d3eMGTMGS5YsEab7+vpi586dCA8Px4ABA9CuXTtERUUZ9DX0+OOPIz4+HrNmzcIHH3yArl27IiEhAb169RLSvPvuu6ioqMCUKVNQUlKCwYMHIzExEd7e3tauNhEREbkgqwOhIUOGQG9Fx06XLl1q8p2fnx/i4+NN/q5Pnz7Yv3+/yTRjx47F2LFjm53u5uaGOXPmYM6cORbllYiIiNSFY40RKQQ7jiUicjwGQkRWYIseIiLXwkCIyAg2YiYiUgcGQkRERKRaDISIiMjlsM0dWYqBkAqxx1UiIvE4pEhlwS0ZBkJERESkWgyEiETEwReJiJwLAyEiIiJSLQZCREY4aqwxIiKSFwMhIiJyOXxKTZZiIERERESqxUCIyBq8yyRSLD3YeRBZj4EQERERqRYDISIiIlItBkJERESkWgyEiIiI7MA31JwbAyEiIiJSLQZCREREpFoMhFTIje+AExERAWAgRERERCrGQIjICJsbP1r5OzayJCKSFwMhIhExriEici4MhIgUgrVDRM7JEecuiwfpMBAiIiIi1WIgRKQQeo4XSSQa1qCQpRgIEVmBXQ8QEbkWBkJERESkWgyEiIjIJfDxsnNQ2oshDISIiIhItawOhFJSUjBq1CgEBgbCzc0NCQkJwrSamhq899576N27N1q1aoXAwEC89tprKCgoMJhHcXExJkyYAB8fH7Rp0wZhYWEoLy83SHPixAk8+eST8Pb2RseOHTFv3rwmedm8eTO6d+8Ob29v9O7dG9u3bzeYrtfrERUVhQ4dOqBly5YICQnBuXPnrF1lIiIiEonSau6sDoQqKirQt29fxMbGNpl269YtHDt2DB9++CGOHTuGH374ATk5OfjLX/5ikG7ChAnIzs5GUlIStm7dipSUFEyZMkWYrtVqMWzYMHTu3BkZGRn48ssvMXv2bKxYsUJIc+jQIbz00ksICwvD8ePHMXr0aIwePRonT54U0sybNw9LlixBXFwc0tLS0KpVK4SGhqKystLa1SaShNIKBCIitfGw9gcjRozAiBEjjE7z9fVFUlKSwXfLli3DoEGDkJeXh06dOuH06dNITEzE0aNHMXDgQADA0qVL8eyzz2L+/PkIDAzE+vXrUV1djVWrVsHT0xMPP/wwMjMzsXDhQiFgWrx4MYYPH44ZM2YAAObOnYukpCQsW7YMcXFx0Ov1iImJwaxZs/D8888DANatWwd/f38kJCRg/Pjx1q46ERGRLHjPJB3J2wiVlpbCzc0Nbdq0AQCkpqaiTZs2QhAEACEhIXB3d0daWpqQ5qmnnoKnp6eQJjQ0FDk5Obh586aQJiQkxGBZoaGhSE1NBQDk5uZCo9EYpPH19UVQUJCQprGqqipotVqDjytSWkM1JWruNXlz286NG5eIyKlIGghVVlbivffew0svvQQfHx8AgEajQfv27Q3SeXh4wM/PDxqNRkjj7+9vkKb+f3NpGk5v+DtjaRqLjo6Gr6+v8OnYsaPV60xERETOQ7JAqKamBn/729+g1+uxfPlyqRYjqpkzZ6K0tFT45Ofny50lIiIikpDVbYQsUR8EXb58Gbt37xZqgwAgICAARUVFBulra2tRXFyMgIAAIU1hYaFBmvr/zaVpOL3+uw4dOhik6devn9F8e3l5wcvLy9rVJRLwyRgRkXMRvUaoPgg6d+4cdu3ahbZt2xpMDw4ORklJCTIyMoTvdu/eDZ1Oh6CgICFNSkoKampqhDRJSUno1q0b7r//fiFNcnKywbyTkpIQHBwMAOjSpQsCAgIM0mi1WqSlpQlpiJrDgIaISB2sDoTKy8uRmZmJzMxMAHcaJWdmZiIvLw81NTV48cUXkZ6ejvXr16Ourg4ajQYajQbV1dUAgB49emD48OGYPHkyjhw5goMHDyIiIgLjx49HYGAgAODll1+Gp6cnwsLCkJ2djY0bN2Lx4sWIjIwU8jF16lQkJiZiwYIFOHPmDGbPno309HREREQAuNNoddq0afjkk0/w008/ISsrC6+99hoCAwMxevRoOzcbERHRHRyD0LlZ/WgsPT0dQ4cOFf6vD04mTpyI2bNn46effgKAJo+f9uzZgyFDhgAA1q9fj4iICDzzzDNwd3fHmDFjsGTJEiGtr68vdu7cifDwcAwYMADt2rVDVFSUQV9Djz/+OOLj4zFr1ix88MEH6Nq1KxISEtCrVy8hzbvvvouKigpMmTIFJSUlGDx4MBITE+Ht7W3tahMREZELsjoQGjJkCPQmeoEzNa2en58f4uPjTabp06cP9u/fbzLN2LFjMXbs2Ganu7m5Yc6cOZgzZ47ZPBERkQthJQ1ZiGONERERkWoxECIiIrKDI16uYAWXdBgIERERkWoxECIiIiLVYiBEZIStVd3W/oz9FRERyYuBEBEREakWAyEVYi0EERHRHQyEiIiIyGGUdjPOQIiIiIhUi4EQERERqRYDISIRudlR56uH+eFpiIhIXAyEiKygsEfbRERkJwZCREREpFoMhIisYM+jLyIiUh4GQkRGuPEhGBGRKjAQIvqdXs/GykSugjczZCkGQkRGOGqsMSIitVHaPScDISIiIlItBkJERESkWgyEiBSCbRqI7KM3+Ntxz1945jo3BkKqxNOWiEg0DuhWgz13SIeBEBEREakWAyEiIiJSLQZCREREpFoMhIiIiEi1GAgRiYgNGomInAsDIRKd0noNJSIi5VDaDSMDIaLfMYAjIlIfBkIkOqVF+7awfRVcYOWJiFSEgZAC7ckpwrcHco1OW30wF7vPFDo4R9JIv1SMZbvPoU7Hqhg10+v1WLn/Ivadveaw+VTX6rAk+RyO5920a5nO5vRVLRYlnUVFVa3Dl12krcSCnTm4UnJb+O52dR0WJZ1FdkGpw/Pj7EpuVWPBzhxcvFaOiqpaLEw6izMardzZckoecmeAmpq0+igAoO8Dvhj4oJ/wfWZ+CT7++RQA4NLnI2XJm5hejEsFALS9zwsvDeokc27Ecb28Su4sOJ3UizfwybbTAOw7rq2Zz7rUS1iYdBYLk866xLlkqRGL9wMAyiprETWqp0OX/eZ3GTiWV4KtJ65izz+HAAAWJ59D3L4LWJx8TvT94OpD1sz8IQu/nNTgm/0XMf7RTlhz6BKWSLAd1cDqGqGUlBSMGjUKgYGBcHNzQ0JCgsF0vV6PqKgodOjQAS1btkRISAjOnTtnkKa4uBgTJkyAj48P2rRpg7CwMJSXlxukOXHiBJ588kl4e3ujY8eOmDdvXpO8bN68Gd27d4e3tzd69+6N7du3W50XJdNoKw3/L61sJqVzu3it3HwicllXbt42n8gCBSWWnx9nC8tEWaazOnnF8TUwx/JKAAC51yuE71gTZLujl+7UZlbW6JAlw/50JVYHQhUVFejbty9iY2ONTp83bx6WLFmCuLg4pKWloVWrVggNDUVl5d1CasKECcjOzkZSUhK2bt2KlJQUTJkyRZiu1WoxbNgwdO7cGRkZGfjyyy8xe/ZsrFixQkhz6NAhvPTSSwgLC8Px48cxevRojB49GidPnrQqL0TGuEI7JyIiMs/qR2MjRozAiBEjjE7T6/WIiYnBrFmz8PzzzwMA1q1bB39/fyQkJGD8+PE4ffo0EhMTcfToUQwcOBAAsHTpUjz77LOYP38+AgMDsX79elRXV2PVqlXw9PTEww8/jMzMTCxcuFAImBYvXozhw4djxowZAIC5c+ciKSkJy5YtQ1xcnEV5ISIicgZ8q1U6ojaWzs3NhUajQUhIiPCdr68vgoKCkJp6pz1Iamoq2rRpIwRBABASEgJ3d3ekpaUJaZ566il4enoKaUJDQ5GTk4ObN28KaRoupz5N/XIsyUtjVVVV0Gq1Bh9XxNoO23HbERG5FlEDIY1GAwDw9/c3+N7f31+YptFo0L59e4PpHh4e8PPzM0hjbB4Nl9FcmobTzeWlsejoaPj6+gqfjh07WrDWRESkBHpWm5AN+Pp8AzNnzkRpaanwyc/PlztLpCJ6sBAnInI0UQOhgIAAAEBhoWE/N4WFhcK0gIAAFBUVGUyvra1FcXGxQRpj82i4jObSNJxuLi+NeXl5wcfHx+BD1BCfjBERuRZRA6EuXbogICAAycnJwndarRZpaWkIDg4GAAQHB6OkpAQZGRlCmt27d0On0yEoKEhIk5KSgpqaGiFNUlISunXrhvvvv19I03A59Wnql2NJXpSuaT8YrDEg18OjmpydI26Q2D5ROlYHQuXl5cjMzERmZiaAO42SMzMzkZeXBzc3N0ybNg2ffPIJfvrpJ2RlZeG1115DYGAgRo8eDQDo0aMHhg8fjsmTJ+PIkSM4ePAgIiIiMH78eAQGBgIAXn75ZXh6eiIsLAzZ2dnYuHEjFi9ejMjISCEfU6dORWJiIhYsWIAzZ85g9uzZSE9PR0REBABYlBel44HvWLwgy0SkDc/2IURkC6tfn09PT8fQoUOF/+uDk4kTJ2LNmjV49913UVFRgSlTpqCkpASDBw9GYmIivL29hd+sX78eEREReOaZZ+Du7o4xY8ZgyZIlwnRfX1/s3LkT4eHhGDBgANq1a4eoqCiDvoYef/xxxMfHY9asWfjggw/QtWtXJCQkoFevXkIaS/JCZByjUCJSMgb+YrE6EBoyZIjJOy83NzfMmTMHc+bMaTaNn58f4uPjTS6nT58+2L9/v8k0Y8eOxdixY+3KCxEREakX3xpzKqylICL78Q1FccndjIFXBvswECIi1WFzIiL5KO38YyBEREREqsVAiIhkxcc0JAW5H1dJTWm1Ks6MgZCCufh5TEREKqS0IJWBkFPhLQAREZGYGAgpmFRRs8KCcVI5sar4eZtARLZgIERkhNKqbomISBoMhEh0bMRHRCQtFrPiYSBE9DuOVUVEpD4MhEh0rvBYyQVWgYiILMBAyKnw8kyuh/VwRPZxhZtPOTEQUjQe3UQkPj4FJrqLgZCCnLxSitzrFSZSNF96FZTcRsblYouWk3LuGm5V11qZO3U6W1iGHE2Z8L81d16F2kocybVsn5D1tJU12H/uGup0vKorRfqlYlwtvS3Z/LMLSnHhWrnd89Hr9Th0/jpulFeJkKvmndFoca6wrNnpR3KLUaitlDQPYtLr9Th4/jqKK6rlzoqoGAjJJPXCDfSfm4Rfsq4CAIq0lXhu6QEMnb9XSGPNRffxz3djzPJUnLxSajZtfvFtvB1/3Nosq05lTR2GLUpBaEwKKmvqrP590GfJ+NvXqTh6ybJgyI01gFYZ9/VhvPrtEXyz/6LcWSEAx/Nu4sW4VARH77b4N9bUTN2sqMbIJQfwzIJ9NuTO0PYsDV5emYYhDcpbsZVX1WJ4zH7896IU1NTpmkxPu3gDf/s6FUGfJUuWB7ElZF7BhJVp+O+F9u8DJWEgJJOXvjmM4opqvLX+GPKLb+Fy8S1R5ns8v8SidMlnikRZnhiUWk1fXlVr9G9rpV28IUZ2XJat+//0VS0AIOH4ld9nZMUybVskmZB+6aak879aKl7NSfLpQgBAWaU4NePGbmFuNqg1MRYIHb5oXW2xEm6Tdmbf2W43WCNEYisoka4qmSzHiyORY7GRLykBAyEiIiJSLQZCRCQrPeviSAKuXtnEDmDFw0BIwVz9RCYiItswDBIPAyEihWDNCBGR4zEQIvqdZTXNrKcjcna85aCGGAgpBB/3KosbX2chcllsX0MNMRBSMF6MSQ3Euibx0aLlnHFLKXn/yl1WO6ozVleNHxkIuShL73isuTPiXZTlGm4ruQvJenLuP6UcO0rJB5GaNSwSlXBOMhBSCDEPhkJtJR6LTsbCnTkm021Kz8eAT3Yh04LeqG9X1+HpBfvw3vcnRMqlstkTupwvKsOjn+4SLS9iiNyYiZCF+2waKsRen/9yBo9/vluycZ0sPXXyi29h0GfJiN1zXpJ8yMGSIXXURP5LKllr3IrDsgdDDIRc0NLd51CorcKS3aYL/He/P4HiimqErz9mdp7bsq4i93oFNqbni5VNxWt4bloTGM1KOInr5crqgv6H41dw4VoF9sgwtErcvgu4WlqJlQdyjU53VBH45Y4cXCurwpc7TN8gOJN/bv5V7iyQjRRSUWwVKR5PHsktxjWJB781h4GQgtl0nuj1kGIwbrkjdjnYetKrcFM5BVfcLUo61mw5X+TKv4I2G0H+AacZCClcbZ0OeTfEGZC1OVdcdKwznU6PS9crzAZx1bU65JsZ9NZR7Xyqausk3x83K6pRcqsal29UQGciaq6tM79dnJUjL8D5xbdQa2TQTXMqqmpRpBVvoFFXotdbdm67soarruSG5MYobbcxEFK4Kf/KwFNf7sG2E1clXc7eHOWMRi+Wj37KxpD5e7Ei5aLJdGOWH8KT8/YgLVf+UeJHLjmAJz7fjV8taLdli+o6HR6Zm4R+c5Lw5y/3YoaJNl+T1hzFk/P2YEe2RpK8GHMsT9oRzB0t6VQhnpy3BxNXH7H6t/3m7MSgz5IZDBkx+/dz+2sz53ZzlHYhJnkxEFKI5s7L3b+36Vh10Hj7CrHEp+VJOn85/OvwZQAw2yYk6/cGp//J+E3yPJlzvqgcAPDzrwWSzP9Go7ZL/znW/DrvP3cdALDm4CVJ8mLMj8evOGxZjrAu9RIA4OB564Psmro7pYKrBYdiWJt659yel3hG5pyQKxA9EKqrq8OHH36ILl26oGXLlnjooYcwd+5cgypMvV6PqKgodOjQAS1btkRISAjOnTtnMJ/i4mJMmDABPj4+aNOmDcLCwlBeXm6Q5sSJE3jyySfh7e2Njh07Yt68eU3ys3nzZnTv3h3e3t7o3bs3tm/fLvYqOwzvYoiIiMQleiD0xRdfYPny5Vi2bBlOnz6NL774AvPmzcPSpUuFNPPmzcOSJUsQFxeHtLQ0tGrVCqGhoaisvFsFPGHCBGRnZyMpKQlbt25FSkoKpkyZIkzXarUYNmwYOnfujIyMDHz55ZeYPXs2VqxYIaQ5dOgQXnrpJYSFheH48eMYPXo0Ro8ejZMnT4q92pJwxrcKAAZsZCWRDhged6RUcjcGFournmOiB0KHDh3C888/j5EjR+LBBx/Eiy++iGHDhuHIkTvPyPV6PWJiYjBr1iw8//zz6NOnD9atW4eCggIkJCQAAE6fPo3ExESsXLkSQUFBGDx4MJYuXYoNGzagoODOI4P169ejuroaq1atwsMPP4zx48fjnXfewcKFC4W8LF68GMOHD8eMGTPQo0cPzJ07F/3798eyZcvEXm3lcNboSQFc9BwnF+ZsjWTt4WwXYWfLr5qJHgg9/vjjSE5OxtmzZwEAv/76Kw4cOIARI0YAAHJzc6HRaBASEiL8xtfXF0FBQUhNTQUApKamok2bNhg4cKCQJiQkBO7u7khLSxPSPPXUU/D09BTShIaGIicnBzdv3hTSNFxOfZr65TRWVVUFrVZr8HGE5s6XxjGNnDGONee0s8ZiBv0GNbMOzrpu9ZSYf14vyNHEPubkOK0Meq93kRonuYgeCL3//vsYP348unfvjnvuuQePPPIIpk2bhgkTJgAANJo7b6D4+/sb/M7f31+YptFo0L59e4PpHh4e8PPzM0hjbB4Nl9FcmvrpjUVHR8PX11f4dOzY0er1t5Wxu4dC7d1OpnI0ZYq5w8jRlOF43s1mX12V8nV/vV6PMxotamx4HZlcky21IheulTf57krJbRRXVDv9MVZWWYNL1yuE/6+VVUFTavjmmZpfOydqTPRAaNOmTVi/fj3i4+Nx7NgxrF27FvPnz8fatWvFXpToZs6cidLSUuGTny9vL8ozf8gS/i6vqkX8Ecve7JK6jAuNScFfvzqEuH1NX129dL0CBaXmX/e9XX13qAdrOoD8Li0Pw2P2463vMiz/kYWUWFsiNl7/7mg8rExxRTWe+Hw3+s9NwppDlzA8Zj/ejj8uT+bs9OinuzBk/l6cLyqHTqfHo5/uwmPRybhVXSt31qgBFRQ3TkP0QGjGjBlCrVDv3r3x6quvYvr06YiOjgYABAQEAAAKCwsNfldYWChMCwgIQFGRYb82tbW1KC4uNkhjbB4Nl9FcmvrpjXl5ecHHx8fgoyT1rzNLwZYgYPnepkN4HL5o2WvCRWV3g6VaneV33t/uvxN87TrtgH6PbBxig5zPucIy4e/6fqcSHdh/kpgqa+6cT4cuXEddg8i3YQ2z2jm6RsxV2nK5xlo0JXogdOvWLbi7G862RYsW0P1+sevSpQsCAgKQnJwsTNdqtUhLS0NwcDAAIDg4GCUlJcjIuHvXv3v3buh0OgQFBQlpUlJSUFNTI6RJSkpCt27dcP/99wtpGi6nPk39cpTEVU4UJeKWvcuWbcFjkywhdTsVZ6uxdbb8yknuMkb0QGjUqFH49NNPsW3bNly6dAlbtmzBwoUL8de//hXAnaEKpk2bhk8++QQ//fQTsrKy8NprryEwMBCjR48GAPTo0QPDhw/H5MmTceTIERw8eBAREREYP348AgMDAQAvv/wyPD09ERYWhuzsbGzcuBGLFy9GZGSkkJepU6ciMTERCxYswJkzZzB79mykp6cjIiJC7NUmF6CkcouhBzkbuS9masOtLR4PsWe4dOlSfPjhh/if//kfFBUVITAwEP/4xz8QFRUlpHn33XdRUVGBKVOmoKSkBIMHD0ZiYiK8vb2FNOvXr0dERASeeeYZuLu7Y8yYMViyZIkw3dfXFzt37kR4eDgGDBiAdu3aISoqyqCvoccffxzx8fGYNWsWPvjgA3Tt2hUJCQno1auX2KutMDxFiNTA2dt8MXgiJRA9EGrdujViYmIQExPTbBo3NzfMmTMHc+bMaTaNn58f4uPjTS6rT58+2L9/v8k0Y8eOxdixY02mcSlOWDIqNcsKzZbTaq7GTaz976r7S4rzQ+3bylXXX2pKLavtxbHGSHZKaZRqaoR5Pu+XjhJe5ZY/B+pkTbsiBRwm5KIYCBEACwokCQuha2XKe5tFzIafDKKIiJSLgRAREakLa5dk1WTUBJlfVWEgpBQKPzHV0KhRCY9oyHrcbSQ32Wt95V6+k2MgpBBiluW8MNhPjoKN+43E5AYeU0ome/BEAgZC5PIsrekx1VhaSGNvZiwk1XJsqfVS28XUmdbXVFZNTnOidZSCGmq4yXIMhIgs4Ohn2CymHUsNgQErIIiMYyBE9LvGFwo1XByVQAmbmTUErseSGl5HkeRGSpZD1jXPEwZCBIDPqwFXPcWVT7QOFbkDSUXkONxd9RxjIORqZI5oXPQ8IRXhTYFtnOki6Yi8OtP2kJvcNbIMhBTAGU4Yi7uuV+C6yF1FLncfGeTcFHhKUSOOOMflLsdcGQMhIiIVYEBFZBwDIVejxCoZJyV3dS2RvZReiSDXOcZikhpiIKQQYp6YPMltY/Ka8ftEVk9LS7bt66TnDHtDJ2ektMOWgZBCiHlnZMu8XPn6bvHFwoW3QT2dLR0qirTs5o4xe+evsDJVMdiztHqooOiSFAMhonq8aBBcJ3hwkdVwWaxdVg4GQkQyaraWRKKrGN9gUy/ueSLjGAgRWYA3b9JhOxfHU/sWd4UXIeQ4b5x/qxnHQIjICAY+6uKqBTwRmcdAiCyiigtFo+DHFSsqXOFOWAquuK+peY7e365y3klVCyX3I3sGQuRUXKM4IRKHks4H1qIa4vZwHgyEFELuO1JHR+Ryry8REcmjcZAod40ZAyGyCAMXInk4y7knRz5NvYLu6hUyTnJYOAUGQmQzZ+kHw9ICQ+7n1A0patOqrMQVc9vLeY64Qf47bSJnwEBIIcQsruS8g2TBS47G1+/JWo4+YpR0k0VNMRByMXJfElzpmiTnqki1HV1p/4ipYQDvKtvIGVZDrm0t+nJtiHPErXkUb16mOMMxZQsGQkQW4P2cdMS6KJmrjVR7zZGiHrcSKQgDIRdzp12AjT80QQ2PvNSwjuRirDhklRgHMjizXcP9qcR960wYCBEZ0biRq7M0DFcytpNwPFfa5s52sXe2/KoZAyEF0EOv+mp7V8YYihSH5Y2oHHGKsxiRjiSB0JUrV/DKK6+gbdu2aNmyJXr37o309HRhul6vR1RUFDp06ICWLVsiJCQE586dM5hHcXExJkyYAB8fH7Rp0wZhYWEoLy83SHPixAk8+eST8Pb2RseOHTFv3rwmedm8eTO6d+8Ob29v9O7dG9u3b5dilQnA2cIyzEo4Kdvy16VewoKdOTb/vvHds63BacmtGpt+p/RLU3lVLd77/gRSzl4Tdb5KeCTZ3K5elHS2yXdnNFpEbspEfvEtiXNFDS3bc858IpnU6Rx7DCeevIrbNXUOXSbguvGz6IHQzZs38cQTT+Cee+7BL7/8glOnTmHBggW4//77hTTz5s3DkiVLEBcXh7S0NLRq1QqhoaGorKwU0kyYMAHZ2dlISkrC1q1bkZKSgilTpgjTtVothg0bhs6dOyMjIwNffvklZs+ejRUrVghpDh06hJdeeglhYWE4fvw4Ro8ejdGjR+PkSfku1q7sha8Oybr8qB+zsXT3eZwtLDP43tI7KUtqbiyZV06j5ZviTAXL0t3nsDE9H6+tOiJ3VkTX3G5YnHwOeTcMA56/LD2IH45dwRtr05v51V28ixdHUVkldmQXmk0nV1CdfvmmQ5f35nfHHLo8Vyd6IPTFF1+gY8eOWL16NQYNGoQuXbpg2LBheOihhwDcucuOiYnBrFmz8Pzzz6NPnz5Yt24dCgoKkJCQAAA4ffo0EhMTsXLlSgQFBWHw4MFYunQpNmzYgIKCAgDA+vXrUV1djVWrVuHhhx/G+PHj8c4772DhwoVCXhYvXozhw4djxowZ6NGjB+bOnYv+/ftj2bJlYq+2y8su0JpNU15VK8qyDp2/Dk1ppfmEFuajYdF46Px1XC29bfO81ejIpWIAwG83Ld9upbdrsOdMEWrrdFJly2Fu1dRi/7lrKCq7c0xW/75ODQPe0ts12JMjzvqevqrFKQvON3OcLQjbfaYQlc3UclTV2LddzxeV49f8kgbfONEdCElO9EDop59+wsCBAzF27Fi0b98ejzzyCL755hthem5uLjQaDUJCQoTvfH19ERQUhNTUVABAamoq2rRpg4EDBwppQkJC4O7ujrS0NCHNU089BU9PTyFNaGgocnJycPPmTSFNw+XUp6lfTmNVVVXQarUGHwK0lTWIT8tzyLL2n7uGl1em4bHoZNHnffD8dby8Mg3B0btFn7ezsLX4N7yImDfu61RMWnMUX6dctHGJyrEzuxCvfnsET3ze/HEz7utUTFpt//pW1tRhxOL9eHbJfuE7e2o5lPDY0VKvr0nH+/85Icm8Qxbuw/OxB4Vg1plqYus5YZadhuiB0MWLF7F8+XJ07doVO3bswFtvvYV33nkHa9euBQBoNBoAgL+/v8Hv/P39hWkajQbt27c3mO7h4QE/Pz+DNMbm0XAZzaWpn95YdHQ0fH19hU/Hjh2tXn9byX2Qm7p7vFFe7bB8pF64Idm8D1+0f95qbficaWUgdEZzp7bkx8wrEuTGkNTnzr7f20TV1DW/JLHWt0KkWlU5iLEfEjILRJhL865YUatJTUl2rsl8ARQ9ENLpdOjfvz8+++wzPPLII5gyZQomT56MuLg4sRclupkzZ6K0tFT45Ofny50lmzjqbodvuknHme7k7WX3YaQXaT5OyNQqq3BzOBVX6trA2YkeCHXo0AE9e/Y0+K5Hjx7Iy7vzaCUgIAAAUFho2PCtsLBQmBYQEICioiKD6bW1tSguLjZIY2weDZfRXJr66Y15eXnBx8fH4KMWW09cRcjCfYq9Iz1bWIbFu87hVrX5NyXEuCCyiCJXI2dt5rrUS/ItXAXScovlzoJ9ZC5wRQ+EnnjiCeTkGL7CfPbsWXTu3BkA0KVLFwQEBCA5+W4bEK1Wi7S0NAQHBwMAgoODUVJSgoyMDCHN7t27odPpEBQUJKRJSUlBTc3dV5WTkpLQrVs34Q214OBgg+XUp6lfjqIo4PbtfFE5Pv45W+5sGDVsUQoW7TqLa2VVki3D1nOx8VtqLkuq8c+kma1VWLspnV/zSxD1ozLLFSJAgkBo+vTpOHz4MD777DOcP38e8fHxWLFiBcLDwwHc6aF32rRp+OSTT/DTTz8hKysLr732GgIDAzF69GgAd2qQhg8fjsmTJ+PIkSM4ePAgIiIiMH78eAQGBgIAXn75ZXh6eiIsLAzZ2dnYuHEjFi9ejMjISCEvU6dORWJiIhYsWIAzZ85g9uzZSE9PR0REhNir7TIc/RqoKzAXnCmpdonXe3K0qybeAJXyeFRrez6ynofYM3z00UexZcsWzJw5E3PmzEGXLl0QExODCRMmCGneffddVFRUYMqUKSgpKcHgwYORmJgIb29vIc369esRERGBZ555Bu7u7hgzZgyWLFkiTPf19cXOnTsRHh6OAQMGoF27doiKijLoa+jxxx9HfHw8Zs2ahQ8++ABdu3ZFQkICevXqJfZqKwavc+JpWEizTCVH4PnrGNzO1JDogRAAPPfcc3juueeane7m5oY5c+Zgzpw5zabx8/NDfHy8yeX06dMH+/fvN5lm7NixGDt2rOkMk0mOfGyglLu4xvmQKl/NzVeqhpSO3r5sEEpKUj9mIB+FUkMca4wURZwLp2EhZ2mZp4ai0dHlv1ICW3PE2ixKCvxM5UQNx7qSOMt5oFYMhFyQPa9e83x1bWp6LV8sSj4nWLPhHFxlN7nq8cZAiAy45mFuGVsveEq+UIpFjDva5uYhVtlq12xM/Nia+Srtzt+R1y2FrbrslHYsUPMYCLkYnnuOo9frMWn1EbyzIdPq33654wyeW7q/2bGVpKq5sefC6Kq1SZvTld1xqql9VqfXY/yKVMz8wfzQFC56M+9wqw7k4r8X7kOR1vbxEElZGAgphKgXGRZ4DpFXfAt7cq7hern1fRvF7rmAk1e0+D5D+iEolM7eY9/ew33G96aDCEfeXBgLVkytX37xbRy+WIx/H1F2MGcvsWtX7Dlm5mw9hXNF5Viw86xo+SF5MRBSCCXfrTkia9W1d0aXdqbqZJ0IG6ZOd3dU7YIS5Y6DVF2rk7Qzy3rXbAgqxWZNYKYx0UeOmApKbqPOygPOVGpNaSWKKxwzhqCSzmkxs1JTpzOfyAQlbRfZudpYY2QDBQdBjjLy99G2RXlnzEm357BFKXJnoVmfbDuNo5ek72xz24mryLtxS/LliOWx6GT8auWAtNZKPl2Ixz/fjX/8K8N8YhMaXniLyqrQf24SSm/VNP8DKynitGuQCVPlgK1lhDVDhThrOaRGDIRIFo3LiHNF5bLkoyFH9xUkB4f3I2TDAvfkFJlP5EDmrmdbjkv7ePPH30dk33W60ExK62UXlIo+T1f2+S9nZFu2gooRl8NAiOh3SuoDhhzLVRuCE5F5DISILGAsSHLGsMnhHSo6dnGicJWA2DXWgkh6DITIgOyFp4zPkVgrQMY49K0xlR2DbEdDSsBAyMU4slyxpf2HI4ixDdzgxkLaQUTbzhLtMGsO88bnhEJPEacjVllTPxtbjxS1705XLRMZCCmEkg8wJedNSg3vznlBcxw5trUUbxg5A7lXzdnOK6Xe/Dk9mTcrAyEyqbii2qH920h1Puj1epzRlFmeDwsyImWZaMvFN/d6BcqrasXPjINZu1nv7FstaurkvqybzvudPN7pe6b0Vg3yi52nmwB7GNsm1oxZlXfjFrS3xXvN39rlOxspjy1XfXTrIXcG6A4xDy8x59V/bpKIc5PPmkOXkHTK9OvHljSSVWpD2lMFWjy7ZD98vD1wYnao3NkRWBIs2ttR46UbtzA8Zr9d83DEdXF4zH483b09Vv39UfSdsxMAkDrzaXTwbSn9whXm2wO5eOPJP5hNl198C099uafZ6bKejcosClR/bNmCNUJkwDXjfeCblIuizEepNeP1fe9oK5VVI2TJ9jpVoJU+I1aQ8q539xnDPpJ+zZeuHx8ln8srLDwfj+QWS5wT+UhdlEh5bLkaBkIuyJmrfcUINGxdfck6VLShyFNqwKVG5vaFqelW70YJT125ajOdtzRqiqela2IgRPQ7J44fnYLFFxGFRYHmjgulPi61hNqOebv3lVvDPy2fl8IOaWqEgRCpQoFIg2N+sCVL+NvSgjD+SJ7Vy1lz6JLR7xNPajA27hB+u2lbY8gNR63Pi6PxmiEOS7fj2kOXMGHlYdyqtu+xqphB1b+P5GH8ilRoK8VtJG1MRPwxs7Xoer0ekZsyUdbg0fN/jv2Gbw/kYszyQ1Yvs7KmDtGNhus4W1iGMcsP4eD561bPz5wPE07i3e9/FX2+9WrrdAhbcxTLdp+zbQYcdJWUhBeh5t2sqMaJ36x/7p57vcLq3zRXLr/5XQaOXrqJD7actHqeAJBfrLwR7hu3yamsqZMpJ/ZT8p2/sbZPeujx0U/ZOHj+BtalXnZYXsxtp5k/ZOHwxWJ8tfeC5HnZeuIqss20U8svvo0fjjUdU27u1lPIuGz9YMTG5jVlXToyLt/EhJVpVs/PlFvVtfjX4cvYlP4bCrXi3BA2tvNUIZLPFGH+zrOSzF9qDIQUwpnb9YhJzscM5grnOhH2UZ1OL8q+LhX5dWKpWLI/63TyH/umcmBdh4p2Z0U2txTY9UKFg/JUa+YYrNHpRF2esdq3G+XVoi6jXsNVq9PpUVuns7kMau5nt6utu3lR2uNkBkIuxpq+csixaup0GDp/L15oUJX+HyN3ho5Wcsv6AtjSgtRcYLA5PR/H8kqsXr7rkDYI5P2VcXJvl2N51tciieHmrWo8MicJUzdkyrL85qTJ/HYgAyEXE59mXxsQuctNW++o6zuqA6yrXQtff6zB76xbprV5zdGUIa/4Fo43uPAXV4hzF2hPTUTCceuDsdGxB0WpyZnx/Ykm3zlb7732ZPfN747hbOHdmxdzW/TDBNseiQLA2kNNH31JFRDUn1elt2rwj39lNJluaY2A3AGLVLZnaWRZ7sglB1BWVYuffi3A9xm/yZIHY/65Wbr2S5ZgIEQuIf3S3Tssa8rObVlXjX7vXJdix/v1t1LkuGjto6Or7d//T9NgsDn/Omx7O55VB3NNThcz5qg/r75LM51fc4GO2H06NRe0qvF8FzP4cPZ4lYGQQiilZ2lnLRDEKDAbF5KKvhtVdOacj5xt9BTQRAqANIdUc9tV7ko/a8sLe7eNnKermMd2c7Ny9jauDIQUwJ5DSK/X2z1EgRpZ8lpuw/3i5ua8QaKlrpVVOX2BJgY5x1PSiXnRsuO3tXU63LTysa3cwY2rc/btq+TyhYGQk/siMQePfrpL7myIxuZz3crzq8/snU3eSDHfg7BySyJ7H+fsOlWIRz/dhchN4j6rV+4WMyRWLaq9++GNtel2/V4sf1l2EI/MTbKp6wcxKe3tIkm58KpuPJqHRz/d1aTvJKVgIOTk4vaZ7mcj7eINq+Yn98XekUNsnC8qt39hDnCzohoHz1+HTsJnKEt+7whtixUNp51tJOrfbtrWh5Ijb2LN9WdjDXtOpVNX7+RjezNt6BzF2mMs34J9XF2rw8HzhuWi2MXesbyb+K1EmhHglUav1yP1gunrzNytpwHcHWOu8X6V+x6To88rhFSF7bgVhxE/OQiPP9TOwnw418XNmdi6aUcs3g+NthLzXuzTbBq5C5JmKShjmfklksxXQatoNb3B344796XaZMZ6eW5cpn25w/paCWv28fmicrzwlfW9TTurn09cxQ82vHmqJKwRUgFz0Topm+b33mATT9595daVw1WlxRVq6VDRkfdAltY8S5Gn7w5LO8xMdoGyRn0X90WcpnPbdarQ7vnK/QiUgZALUmOljuir7Nb4X7lPVZIL97zrM7eP1VimqonkgdDnn38ONzc3TJs2TfiusrIS4eHhaNu2Le677z6MGTMGhYWGUWVeXh5GjhyJe++9F+3bt8eMGTNQW2vYuHXv3r3o378/vLy88Mc//hFr1qxpsvzY2Fg8+OCD8Pb2RlBQEI4cOSLFaroMKc/31As38Ozi/TiSW4yXvzks4ZIcw9q7fykfPdhzqbY1V5ZcHH5t9Dhq39lreHbxfpy8oqy7ZkfadDTf4H8pzzlzx5wjXq12BlLX5BWU3saopQfwwzHHdWJo6zq9+/2vCLdgIFqr8iLanKQhaSB09OhRfP311+jTx7Btw/Tp0/Hzzz9j8+bN2LdvHwoKCvDCCy8I0+vq6jBy5EhUV1fj0KFDWLt2LdasWYOoqCghTW5uLkaOHImhQ4ciMzMT06ZNwxtvvIEdO3YIaTZu3IjIyEh89NFHOHbsGPr27YvQ0FAUFRVJudqKo5QC6qVvDuPUVS3+9nUqrjYzGrwYjbVtDziUfro6v4mrjuDUVS0mrTkqd1YMmDpH7AlgjR3P7zbuQFEpJ6gTcpYt9/HPp5B1pVT0tzLFVllTh03pv2Hbias2v1zgjCQLhMrLyzFhwgR88803uP/++4XvS0tL8e2332LhwoV4+umnMWDAAKxevRqHDh3C4cN3agl27tyJU6dO4bvvvkO/fv0wYsQIzJ07F7GxsaiuvtO3RVxcHLp06YIFCxagR48eiIiIwIsvvohFixYJy1q4cCEmT56MSZMmoWfPnoiLi8O9996LVatWSbXa5MTMvz7vmHyogdZJBo21RMNgx9kOEX0zfzubHE2Z0YFM5dBwyJR6DbvqKL1l/NhX2rEjZp9W5shdtkoWCIWHh2PkyJEICQkx+D4jIwM1NTUG33fv3h2dOnVCamoqACA1NRW9e/eGv7+/kCY0NBRarRbZ2dlCmsbzDg0NFeZRXV2NjIwMgzTu7u4ICQkR0jRWVVUFrVZr8FEdJy0NbTlnHd2TtJjzFzuvSqqUkLtQVCtjx4Az7IuUs9cQGpOC4TH7jU539LEdu8d0lyZ95+w0+r25bNq6HlKvv0XHiMKPI0len9+wYQOOHTuGo0ebVn9rNBp4enqiTZs2Bt/7+/tDo9EIaRoGQfXT66eZSqPVanH79m3cvHkTdXV1RtOcOWP89cno6Gh8/PHHlq+oqJTRo6y63d1yxhpPNv5O7j6XGlJQVpxe4wuHuYa03PTWs/R4taQs+/nXAgBAXrH5fnvkOk/YLYmyiV4jlJ+fj6lTp2L9+vXw9vYWe/aSmjlzJkpLS4VPfn6++R+JxBnPE1c/uV19/RpSUiAlz2Y3sVAFbRtHk+qNufrjzWz7K/Wcgk5BqnNT7lNM9EAoIyMDRUVF6N+/Pzw8PODh4YF9+/ZhyZIl8PDwgL+/P6qrq1FSUmLwu8LCQgQEBAAAAgICmrxFVv+/uTQ+Pj5o2bIl2rVrhxYtWhhNUz+Pxry8vODj42PwcQXO1gMwGWcqMHO2V7yVfEQ2CQqVnFkiiYgZ9Ci9dBI9EHrmmWeQlZWFzMxM4TNw4EBMmDBB+Puee+5BcnKy8JucnBzk5eUhODgYABAcHIysrCyDt7uSkpLg4+ODnj17CmkazqM+Tf08PD09MWDAAIM0Op0OycnJQhoyQuYj1taaCYNgz+YT2HDhYl//lHo9VVoh5a60DJljMNiYk2Xe4LRR4BHqZJvT1Rk7QlxhF4neRqh169bo1auXwXetWrVC27Zthe/DwsIQGRkJPz8/+Pj44O2330ZwcDAee+wxAMCwYcPQs2dPvPrqq5g3bx40Gg1mzZqF8PBweHl5AQDefPNNLFu2DO+++y5ef/117N69G5s2bcK2bduE5UZGRmLixIkYOHAgBg0ahJiYGFRUVGDSpElir7brsKIslKKNjFJrNtzcoKgz3tmut9aQo+2VWHe/Tr1bJGgsbW67mj3fHRSbSX3IWbIaznrsKKmtpK1k6Vl60aJFeO655zBmzBg89dRTCAgIwA8//CBMb9GiBbZu3YoWLVogODgYr7zyCl577TXMmTNHSNOlSxds27YNSUlJ6Nu3LxYsWICVK1ciNDRUSDNu3DjMnz8fUVFR6NevHzIzM5GYmNikATXddaOi2qbfaSud73XoTenNtwH7PuO3JoOcNj7fHX36p+UWC38rqflSobYSG47kobKmTpT56fV65GjK8GPmFWW00zKzo7Ov3H27NDO/xOrOIsVcw8bD6Viz+Q5euI4D564bfCfVMe7IG56Gm2Dj0XzcFuk4dRoWHAPlVbX495E8m8t/ayntxROHDLq6d+9eg/+9vb0RGxuL2NjYZn/TuXNnbN++3eR8hwwZguPHj5tMExERgYiICIvz6oqsKQxLbezfZfqGTJt+JydTYw59kXgGhdq7nT6KcZrae1G/Vd18AW5POaKHfYHV88sOQqOtxBlNGWb/5WHbZ9RAaEwKAMCn5T0Y2q29KPOUyoHzhsHDc0sP4NLnIx2ej9o6HV6yo8f2k1e0eOXbNGR/HGo+sYWUVlmwLvWy0e+VWhPtKFEJJ/HD8Svo0q6VJPOXO9Axh2ONkSiSz8jbW7cYFQeNz9X0y8XGEzqBg+evI0fTtGM3KdQPCrvm0CXU6SzcEXpgR7bGbLJTBa7fl5defyeI+SXrql3zqbV025thKuB2hAwrzzsx1tr8ddq+pSihYtOUX34f0Dn3eoXwXWK2BrV1OrO/3Xf2mmT5chQGQiQpOR5tKLzMkdyFa+WYsDJNqFUxx+LgpRFju9bU48aGqut0+Me/MoxOU/rdoxTWHLqEt9Yfc/hyxWwgbc28TO3iMcuNd3hLjvX5L2earUGrl3H5Jood9DhNSgyEFECvd92L95RmLnauwPpBVx3jfFG5Vek/2XZatGXvP2f/3aEK4yDslqlG1WxjZon2haWzVc6bbPIflDbvCzuyvtdMbU/WbyUWzcdckwu5ty4DIYUQd/gFw5nJWZQknSo0n6gB21+ft1/jRTdp0CfCMsSinAuE+OR4jOC6W9N6YgY/Sn8kpArcB2YxEHJBznzcyxlsmNtuzvLIRoxx1yThwtvPGZepFpY8nlfE6/NOcn5IwlUHXSUiQ1Je7AwLUcdcVY3VSon99o2j2pi5QiAixTrYuz+l74eImmPNtmluPzX5Wm/yX6fFQIhcjitc1KzlSheE5gplvV6PqlrX7ANGqZUBkrUREnHGjjnf7VyIkxZKjXeTGI/ka+t0Nr+gIRUGQuQSpKg5MFdWu1LfI85QTk/bmIlusxJxpeS23FkRlR562ba/uONJWX8+iHFhdZX2cmKXYb/dtP88kaKEGzJ/b5NzWO6SlIGQCjjDRa6enHfG5hYt98nakKl9Kuv+lnAj/ZhZAABYf9j0K732ErMHcUceM8YCAqWe+pZuF0eVB1Lf1MixH/71+3mitEBRjABNbAyEiCwgTlW+sgokZ+CooE7Oi4XSblTEDAnsXTelbRtyTQyEFEJpUTs1GlRctlxYz1FHkrGLlDNtJ0s5y5kpVtDgkPW18EDRibBSzrL/xCbGozZzN4CuEqgyECJFsbWK2hHnY5NHJgq66jfMi6sUTiQ/e2tClXSOmGI+n/atiCXnpNivzztTMSB31wEMhFTAWWub5iWeQVGDgU8tJdb6/vqbdaOImyPp6/PNfP+dhe1psqwcMR1wzEXOOY/cuxxZvtuyrYz9xpIsH8ktxuJd5xT39o+tzK+za6xnY5pS0+Wrk8SxdmMg5Ipc5Jz9au8F/I+F4y8Z9KIj0fo3rq2Ss+bF0mXPSjiJvBu3pM1MA2Lc2Smt8FVafiwmchRW8/sAnHq9Hn/7OhWLdp01GFvOWW+4XFV9GWGqrHhj3VGT82jy+ryL7mIGQgrhqgeYtZqOAH9Tnow4scYXpMO5N6RZjkzHrOS9AIuwXl/tPW//TKzUpE2IBStirh1J/aZOPl2Irv/3C747fBlj4+4OinrpRoXxH5rJgtMGl07EksD05BWtnctwDQyEyCWIMtaYlaWzUgddbXzxmZeYY9Xvj+QWi5gbGzTYsMYbZIt/GTXZHYEN87Nmm0vVyPnDH7NFaTD7S9ZVhK1NB3CnhlGMmxNR24SIsP2kH2LDdCbHfZ1qdmBSKTW/+o4JWeVuS8ZASA1cJWynZol5Yfnb16nmE5GopHqsdLawzO55vGXh42mlcoba9jQJbj6cYb2VgoGQC1L7s3qp1l7uu5aGTOXFebrVUw5X2ELGLny1dabXTOr1tnusMXGyYTdnDCrEyHPTITZcEwMhhXDVA4zuErMwVULBbG0WLO5N2GAZ8q+oI+JfqXo2tmXrNaxdlDv4V8Jx7uzs2YQKuveTFAMhcg0qLzAN+xGSb2O4YsEpffsRkQI+SUafl4byjhPTOZI7IFQqOcsaMTEQUgCx73q3Z2kazV8apbdrcKO8StR5yt2xljWcJacuUlbJxtbt90vWVew6XSRuZpyMuW0nxrHZ3Cx42DsPuctSBkIKoNcDV510RO1pGzPlzoJopBx4sayyBnO2Zks2f6UQY2T4L345I/y96kAu/n0kz2C6JbGyXq/HwqSz+CXrKraeKEDMrrNm04vN0Y2Mjd1QyX2BsZdc90U5mjLM/ikb18rEvdGzh5TlU0V1nfFlmljkqQKtaDeuct8Ae8i6dAIAbEzPx7YTV+XOhk2OXrLsbQd7LjSW/LbhRUCJ1bXzd+TY3WeHKY7oULIxY9s5Q4RXq8uqaoW/tZW1mPlDFv42sKPwnSVF5v5z17Ek+ZxNy3eiSkkDCjzsm1W/jc1lWZwaI/MzafxoOTQmBQCQe70Ca18fZH8eZNg3er3e7rKwaSeyd+d3TeSnAXJijZACOGsQVC/pVKHJ6Xq9Hq98m2bz/J/4fLfNvxVLZW2dXRfIHBteY/7n5l9tWlbjWjopy+CqWuN3kmKzdiiHNYcuWZW+xsTbVVK82iwFY2tgb6Bh7k7dXC1Fcz93gxtWHci1u8+qWp0eW45fsWsed/Jzx9pDlzDos2Th+31nr2HxrnMOD2TMDX1hiYLSSoxccgC1v/cIbgtzZV5zgVbK2Wt49ds0/HbTsl7t5b73YCBEdnGDGyavSzeZRnu7FgfPW9a7sbETokCEQsFe61IvNy30rTh7zxeVW73M7zN+a3aaUm7+Nx3NN59IBOaC7cZ2n7Gubc7mhkNFOHjjKrEG0xHmbD0ldxaa+MjI47BFZh6rSsHUo9ziimqL53Pqqlay3vlNFX+vrTqC/eeuW3wzJ3ctLAMhFXCmglaME6KyRocxyw/ZP6MGjLUV0Nl+oyUqvV4vW0lyrbxpodzcHag9R6HU7avMDRXhrGw5Khr+xu5+gBRQ9FTVyn+iZhdY91i8ue2m1+tR0eDRsdIdvliMUUsPyJ0Ns9hGiOwidyRfr2HBMS/xDC5eF/fC9u2BXAT4eBt8l2bVGF7SbKjpGzNxPO8mSkx0z29LILz6YK7ZNGm5xUbb4ezNuWY0va1tdgCgUCttewSp52/O4Yv2P36z7YZHvO43lRD0NDYrIQvfHc4zn1BhmmvX9I9/ZWCnlbWj9hCjfM+6Umo2jdzHDgMhsotF54mDgyVbgyBzJ/2n208b/G+u115HEKN9hDEf/2z+scXnDd7uaqjGjjYJFlFK9O0E7D1CJetHyEG70NIgyGxbKAcfcs0FBrYEQTxbzOOjMWrW7WZeqWxI7Ncetbctr/bdm1OE1Avijaxu7V2J3NfjkxbcacnBEeHhKSsfNYjlh2PNt9uyVbVIgaNN9UFGB7UVR2WNYxrSN2dTumPar0klR2P/OHGAfeVUfbvIOp0e18urRDtWlYY1QmTU7J+yLXrzRuxYwJqGiX9ffRQAcOGzZ0VZ9r8OX7YqvTUvMokdNOn1ejxn4bN3+eutxOUG4Nkl+2VZduQm297kM+XiNXEe40oZ1Nii+4eJeHHAA7Itv7kaS1s4vAE9ILzCb/e8RBhj45WVaUi9KN5NZ2Nyt+NijRAZZe3rx6ZIXXOi0+tlecZ8q9ry2it31k+TM7Li5DWW1NSbj0qitNNT7jYzjRkLgsQs18tlbgAueiAUHR2NRx99FK1bt0b79u0xevRo5OTkGKSprKxEeHg42rZti/vuuw9jxoxBYaHhs8+8vDyMHDkS9957L9q3b48ZM2agttZwY+3duxf9+/eHl5cX/vjHP2LNmjVN8hMbG4sHH3wQ3t7eCAoKwpEjR8ReZTIjv9iyviRsVVunt/r1ajF8e8B8g+J6YvcKq7SCsiEl501tbBt09e7fi3eZbuCedaXEhiXI35Owmtj3aEwdRA+E9u3bh/DwcBw+fBhJSUmoqanBsGHDUFFxt/p3+vTp+Pnnn7F582bs27cPBQUFeOGFF4TpdXV1GDlyJKqrq3Ho0CGsXbsWa9asQVRUlJAmNzcXI0eOxNChQ5GZmYlp06bhjTfewI4dO4Q0GzduRGRkJD766CMcO3YMffv2RWhoKIqKnH/8n5k/nBB9nC+bWHCmjP/6sKRZGPTpLmyUoT3A5RuWB3garbh9IVnTQWPJrebfKHNGi+14+8wSRQoaVsEatoxZaKyzyIaB7HUzZYyl/YM1XQajZVPEHH/SkhcfbOUqu1H0NkKJiYkG/69Zswbt27dHRkYGnnrqKZSWluLbb79FfHw8nn76aQDA6tWr0aNHDxw+fBiPPfYYdu7ciVOnTmHXrl3w9/dHv379MHfuXLz33nuYPXs2PD09ERcXhy5dumDBggUAgB49euDAgQNYtGgRQkNDAQALFy7E5MmTMWnSJABAXFwctm3bhlWrVuH9998Xe9Ud6t9H8i1uWCzlwVpWaT4PZRJXe0o9f7KO2IMIk4VsaCNk7BF4jZUdZOn1eqtreM6I1BBYLG5uwJbjzvEYz1pX7eiQVi01d5K3ESotvfNmi5+fHwAgIyMDNTU1CAkJEdJ0794dnTp1QmpqKgAgNTUVvXv3hr+/v5AmNDQUWq0W2dnZQpqG86hPUz+P6upqZGRkGKRxd3dHSEiIkKaxqqoqaLVag4+SXbhmeW/Fp68qe12IyD7H8sTpQdjafncGfZaML3eI1zBZDjV1OkzfKH5DeJsZCWqlusGIbtQtSEOmwiApB4F1NEkDIZ1Oh2nTpuGJJ55Ar169AAAajQaenp5o06aNQVp/f39oNBohTcMgqH56/TRTabRaLW7fvo3r16+jrq7OaJr6eTQWHR0NX19f4dOxY0ej6ZxNZW0d3v/PCbmzQUQSevM7cUa7t+YGC7jT63rsnguiLFsupsaak4Mjc/N1ykUHLk2ZJA2EwsPDcfLkSWzYsEHKxYhm5syZKC0tFT75+cruh8LS6uXvDufhshUNlmP3nLc1S3a5acUYOqRcrtJuQLVUuP+c4Zg9ekmaMcNMMTvoqoscLJL1IxQREYGtW7ciJSUFDzxwty+JgIAAVFdXo6SkxKBWqLCwEAEBAUKaxm931b9V1jBN4zfNCgsL4ePjg5YtW6JFixZo0aKF0TT182jMy8sLXl5etq2wwlnTYPbLHTnmE0ngf9aLc0dL8pJjkEoybtwK619UcJWLmzNz5sbkzph30WuE9Ho9IiIisGXLFuzevRtdunQxmD5gwADcc889SE5OFr7LyclBXl4egoODAQDBwcHIysoyeLsrKSkJPj4+6Nmzp5Cm4Tzq09TPw9PTEwMGDDBIo9PpkJycLKQhZZGywy5yHLE6CCR5bM8y3nTAHCe8/gmUFvwpJTcm2wi5AT//erXJ9wM+2SVdhiQieo1QeHg44uPj8eOPP6J169ZCexxfX1+0bNkSvr6+CAsLQ2RkJPz8/ODj44O3334bwcHBeOyxxwAAw4YNQ8+ePfHqq69i3rx50Gg0mDVrFsLDw4UamzfffBPLli3Du+++i9dffx27d+/Gpk2bsG3bNiEvkZGRmDhxIgYOHIhBgwYhJiYGFRUVwltkRERE5oK4fWeNDyTs6sy9NWZsQNViJ2ziIHogtHz5cgDAkCFDDL5fvXo1/v73vwMAFi1aBHd3d4wZMwZVVVUIDQ3FV199JaRt0aIFtm7dirfeegvBwcFo1aoVJk6ciDlz5ghpunTpgm3btmH69OlYvHgxHnjgAaxcuVJ4dR4Axo0bh2vXriEqKgoajQb9+vVDYmJikwbUREREzRFzTENSHtEDIUueD3p7eyM2NhaxsbHNpuncuTO2b99ucj5DhgzB8ePHTaaJiIhARESE2TwREREZ4+ghIJz5MaMz4lhjREREJhjrgVtKSomDTPcj5DoYCBERkd3kettUDEqrgalgb/kOxUCIiIhU7adfr8idBQO7zyhkPExXqvYxgYEQkYJlXHZslTyRGn2zP1fuLCiSG9xUUTvFQIhIwawd94mISCzH8m7i4Y92GJ328so0B+dGOgyEiBRsy3FlVdkTkXrkXldH56gMhIiIiEi1GAgRERGRajEQIiIiItViIERERESqxUCIiIiIVIuBEBEREakWAyEiIiJSLQZCREREpFoMhIiIiEi1GAgRERGRajEQIiIiItViIERERESqxUCIiIiIVIuBEBEREakWAyEiIiJSLQZCREREpFoMhIiIiEi1GAgRERGRajEQIiIiItViIERERESqxUCIiIiIVIuBEBEREakWAyEiIiJSLQZCREREpFqqCIRiY2Px4IMPwtvbG0FBQThy5IjcWSIiIiIFcPlAaOPGjYiMjMRHH32EY8eOoW/fvggNDUVRUZHcWSMiIiKZuXwgtHDhQkyePBmTJk1Cz549ERcXh3vvvRerVq2SO2tEREQkM5cOhKqrq5GRkYGQkBDhO3d3d4SEhCA1NbVJ+qqqKmi1WoOPFC5dr5BkvkRERM6o5Fa1bMt26UDo+vXrqKurg7+/v8H3/v7+0Gg0TdJHR0fD19dX+HTs2FGSfHX0u1eS+RIRETkjN7jJtmyXDoSsNXPmTJSWlgqf/Px8SZbTwt0NU576gyTzJiIiciavP9EFvvfeI9vyPWRbsgO0a9cOLVq0QGFhocH3hYWFCAgIaJLey8sLXl5eDsnbB8/2wAfP9nDIsoiIiMg4l64R8vT0xIABA5CcnCx8p9PpkJycjODgYBlzRkRERErg0jVCABAZGYmJEydi4MCBGDRoEGJiYlBRUYFJkybJnTUiIiKSmcsHQuPGjcO1a9cQFRUFjUaDfv36ITExsUkDaiIiIlIfN71er5c7E0ql1Wrh6+uL0tJS+Pj4yJ0dIiIisoA112+XbiNEREREZAoDISIiIlItBkJERESkWgyEiIiISLUYCBEREZFqMRAiIiIi1WIgRERERKrFQIiIiIhUi4EQERERqZbLD7Fhj/pOt7Varcw5ISIiIkvVX7ctGTyDgZAJZWVlAICOHTvKnBMiIiKyVllZGXx9fU2m4VhjJuh0OhQUFKB169Zwc3MTdd5arRYdO3ZEfn6+6scx47a4i9viLm6Lu7gt7uK2uIvb4q7G20Kv16OsrAyBgYFwdzfdCog1Qia4u7vjgQcekHQZPj4+qj+A63Fb3MVtcRe3xV3cFndxW9zFbXFXw21hriaoHhtLExERkWoxECIiIiLVYiAkEy8vL3z00Ufw8vKSOyuy47a4i9viLm6Lu7gt7uK2uIvb4i57tgUbSxMREZFqsUaIiIiIVIuBEBEREakWAyEiIiJSLQZCREREpFoMhGQSGxuLBx98EN7e3ggKCsKRI0fkzpLDpaSkYNSoUQgMDISbmxsSEhLkzpJsoqOj8eijj6J169Zo3749Ro8ejZycHLmzJYvly5ejT58+QsdowcHB+OWXX+TOluw+//xzuLm5Ydq0aXJnRRazZ8+Gm5ubwad79+5yZ0s2V65cwSuvvIK2bduiZcuW6N27N9LT0+XOlsM9+OCDTY4LNzc3hIeHWzwPBkIy2LhxIyIjI/HRRx/h2LFj6Nu3L0JDQ1FUVCR31hyqoqICffv2RWxsrNxZkd2+ffsQHh6Ow4cPIykpCTU1NRg2bBgqKirkzprDPfDAA/j888+RkZGB9PR0PP3003j++eeRnZ0td9Zkc/ToUXz99dfo06eP3FmR1cMPP4yrV68KnwMHDsidJVncvHkTTzzxBO655x788ssvOHXqFBYsWID7779f7qw53NGjRw2OiaSkJADA2LFjLZ+Jnhxu0KBB+vDwcOH/uro6fWBgoD46OlrGXMkLgH7Lli1yZ0MxioqK9AD0+/btkzsrinD//ffrV65cKXc2ZFFWVqbv2rWrPikpSf/nP/9ZP3XqVLmzJIuPPvpI37dvX7mzoQjvvfeefvDgwXJnQ5GmTp2qf+ihh/Q6nc7i37BGyMGqq6uRkZGBkJAQ4Tt3d3eEhIQgNTVVxpyRkpSWlgIA/Pz8ZM6JvOrq6rBhwwZUVFQgODhY7uzIIjw8HCNHjjQoM9Tq3LlzCAwMxB/+8AdMmDABeXl5cmdJFj/99BMGDhyIsWPHon379njkkUfwzTffyJ0t2VVXV+O7777D66+/btVA6QyEHOz69euoq6uDv7+/wff+/v7QaDQy5YqURKfTYdq0aXjiiSfQq1cvubMji6ysLNx3333w8vLCm2++iS1btqBnz55yZ8vhNmzYgGPHjiE6OlrurMguKCgIa9asQWJiIpYvX47c3Fw8+eSTKCsrkztrDnfx4kUsX74cXbt2xY4dO/DWW2/hnXfewdq1a+XOmqwSEhJQUlKCv//971b9jqPPEylMeHg4Tp48qdr2DwDQrVs3ZGZmorS0FN9//z0mTpyIffv2qSoYys/Px9SpU5GUlARvb2+5syO7ESNGCH/36dMHQUFB6Ny5MzZt2oSwsDAZc+Z4Op0OAwcOxGeffQYAeOSRR3Dy5EnExcVh4sSJMudOPt9++y1GjBiBwMBAq37HGiEHa9euHVq0aIHCwkKD7wsLCxEQECBTrkgpIiIisHXrVuzZswcPPPCA3NmRjaenJ/74xz9iwIABiI6ORt++fbF48WK5s+VQGRkZKCoqQv/+/eHh4QEPDw/s27cPS5YsgYeHB+rq6uTOoqzatGmDP/3pTzh//rzcWXG4Dh06NLkp6NGjh2ofFQLA5cuXsWvXLrzxxhtW/5aBkIN5enpiwIABSE5OFr7T6XRITk5WbRsIAvR6PSIiIrBlyxbs3r0bXbp0kTtLiqLT6VBVVSV3NhzqmWeeQVZWFjIzM4XPwIEDMWHCBGRmZqJFixZyZ1FW5eXluHDhAjp06CB3VhzuiSeeaNK9xtmzZ9G5c2eZciS/1atXo3379hg5cqTVv+WjMRlERkZi4sSJGDhwIAYNGoSYmBhUVFRg0qRJcmfNocrLyw3u5nJzc5GZmQk/Pz906tRJxpw5Xnh4OOLj4/Hjjz+idevWQnsxX19ftGzZUubcOdbMmTMxYsQIdOrUCWVlZYiPj8fevXuxY8cOubPmUK1bt27SRqxVq1Zo27atKtuO/fOf/8SoUaPQuXNnFBQU4KOPPkKLFi3w0ksvyZ01h5s+fToef/xxfPbZZ/jb3/6GI0eOYMWKFVixYoXcWZOFTqfD6tWrMXHiRHh42BDWSPcSG5mydOlSfadOnfSenp76QYMG6Q8fPix3lhxuz549egBNPhMnTpQ7aw5nbDsA0K9evVrurDnc66+/ru/cubPe09NT///+3//TP/PMM/qdO3fKnS1FUPPr8+PGjdN36NBB7+npqf+v//ov/bhx4/Tnz5+XO1uy+fnnn/W9evXSe3l56bt3765fsWKF3FmSzY4dO/QA9Dk5OTb93k2v1+vFicmIiIiInAvbCBEREZFqMRAiIiIi1WIgRERERKrFQIiIiIhUi4EQERERqRYDISIiIlItBkJERESkWgyEiIiIyOFSUlIwatQoBAYGws3NDQkJCVbPQ6/XY/78+fjTn/4ELy8v/Nd//Rc+/fRTq+bBITaIiIjI4SoqKtC3b1+8/vrreOGFF2yax9SpU7Fz507Mnz8fvXv3RnFxMYqLi62aB3uWJiIiIlm5ublhy5YtGD16tPBdVVUV/u///g///ve/UVJSgl69euGLL77AkCFDAACnT59Gnz59cPLkSXTr1s3mZfPRGBERESlOREQEUlNTsWHDBpw4cQJjx47F8OHDce7cOQDAzz//jD/84Q/YunUrunTpggcffBBvvPGG1TVCDISIiIhIUfLy8rB69Wps3rwZTz75JB566CH885//xODBg7F69WoAwMWLF3H58mVs3rwZ69atw5o1a5CRkYEXX3zRqmWxjRAREREpSlZWFurq6vCnP/3J4Puqqiq0bdsWAKDT6VBVVYV169YJ6b799lsMGDAAOTk5Fj8uYyBEREREilJeXo4WLVogIyMDLVq0MJh23333AQA6dOgADw8Pg2CpR48eAO7UKDEQIiIiIqf0yCOPoK6uDkVFRXjyySeNpnniiSdQW1uLCxcu4KGHHgIAnD17FgDQuXNni5fFt8aIiIjI4crLy3H+/HkAdwKfhQsXYujQofDz80OnTp3wyiuv4ODBg1iwYAEeeeQRXLt2DcnJyejTpw9GjhwJnU6HRx99FPfddx9iYmKg0+kQHh4OHx8f7Ny50+J8MBAiIiIih9u7dy+GDh3a5PuJEydizZo1qKmpwSeffIJ169bhypUraNeuHR577DF8/PHH6N27NwCgoKAAb7/9Nnbu3IlWrVphxIgRWLBgAfz8/CzOBwMhIiIiUi2+Pk9ERESqxUCIiIiIVIuBEBEREakWAyEiIiJSLQZCREREpFoMhIiIiEi1GAgRERGRajEQIiIiItViIERERESqxUCIiIiIVIuBEBEREakWAyEiIiJSrf8P3vCF+WY4zeQAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from matplotlib import pyplot as plt\n", "\n", "plt.plot(df['CO2'].to_list())" ] }, { "cell_type": "code", "execution_count": 4, "id": "9946f5cd-3683-49db-8535-393cb04140ce", "metadata": {}, "outputs": [], "source": [ "tokenizer_path = 'DeepChem/ChemBERTa-77M-MTR'\n", "tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)\n", "\n", "# Only the hidden size is slightly larger, everything else is the same\n", "config = BertConfig(\n", " vocab_size=tokenizer.vocab_size,\n", " hidden_size=768,\n", " num_hidden_layers=4,\n", " num_attention_heads=12,\n", " intermediate_size=2048,\n", " max_position_embeddings=512\n", " )\n", "\n", "simson_params = torch.load('/home/jovyan/simson_training_bolgov/simson_checkpoints_1M/checkpoint_best_model.bin')\n", "\n", "backbone = SimSonEncoder(config=config, max_len=512)\n", "backbone = torch.compile(backbone)\n", "backbone.load_state_dict(simson_params)\n", "\n", "\n", "model = SimSonClassifier(encoder=backbone, num_labels=len(targets))\n", "model = torch.compile(model, fullgraph=True)" ] }, { "cell_type": "code", "execution_count": 5, "id": "903489f0-9039-4504-894e-6739b4a15371", "metadata": {}, "outputs": [], "source": [ "def create_splits(df):\n", " length = len(df)\n", " train_length = int(0.99 * length)\n", " train = df.loc[:train_length]\n", " test = df.loc[train_length:]\n", " return train, test\n", "\n", "train, test = create_splits(df)\n", "\n", "train = train.reset_index(drop=True)\n", "test = test.reset_index(drop=True)" ] }, { "cell_type": "code", "execution_count": 6, "id": "00c271f1-bd44-457d-9a0e-7b221871ab78", "metadata": {}, "outputs": [], "source": [ "scalers = []\n", "\n", "for target in targets:\n", " target_scaler = StandardScaler()\n", " train[target] = target_scaler.fit_transform(train[target].to_numpy().reshape(-1, 1))\n", " test[target] = target_scaler.transform(test[target].to_numpy().reshape(-1, 1))\n", " \n", " scalers.append(target_scaler)\n", "\n", "smiles_train = train['Smiles']\n", "smiles_test = test['Smiles']\n", "\n", "labels_train = train[targets].values\n", "labels_test = test[targets].values" ] }, { "cell_type": "code", "execution_count": 7, "id": "01ebce4a-9ac0-4527-a9bd-8d13913f15e3", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['/home/jovyan/simson_training_bolgov/regression/scalers']" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import joblib\n", "\n", "joblib.dump(scalers, '/home/jovyan/simson_training_bolgov/regression/scalers')" ] }, { "cell_type": "code", "execution_count": 8, "id": "4405c601-f006-4eeb-989e-fb35dd5349ba", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Setting up datasets for two-label training (labels assumed pre-scaled)\n", "Training dataset statistics:\n", " total_samples: 6659681\n", " label_0_count: 6659681\n", " label_1_count: 6659681\n", " label_0_ratio: 1.0\n", " label_1_ratio: 1.0\n", " both_labels_count: 0\n", " single_label_count: 0\n", " no_labels_count: 0\n", "Validation dataset statistics:\n", " total_samples: 67270\n", " label_0_count: 67270\n", " label_1_count: 67270\n", " label_0_ratio: 1.0\n", " label_1_ratio: 1.0\n", " both_labels_count: 0\n", " single_label_count: 0\n", " no_labels_count: 0\n", "Computed label weights: [0.33333334 0.33333334 0.33333334 0.33333334 0.33333334 0.33333334]\n", "Using device: cuda\n", "Training steps per epoch: 26015\n", "Total training steps: 78045\n", "Label weights: [0.33333334 0.33333334 0.33333334 0.33333334 0.33333334 0.33333334]\n", "Validation will be performed every 7000 steps\n", "\n", "Epoch 1/3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 27%|██▍ | 7001/26015 [10:27<10:43:20, 2.03s/it, step=7002, loss=0.0372, true_loss=16.0679, lr=1.84e-05]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 7000 | Train Loss: 0.0618 | Val Loss: 0.1191 | True train loss: 17.4244 | True val loss: 18.3473\n", "New best validation loss: 0.1191\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 54%|████▎ | 14001/26015 [20:40<3:46:01, 1.13s/it, step=14002, loss=0.0315, true_loss=15.1059, lr=1.68e-05]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 14000 | Train Loss: 0.0357 | Val Loss: 0.0652 | True train loss: 16.0534 | True val loss: 17.3651\n", "New best validation loss: 0.0652\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 81%|██████▍ | 21001/26015 [30:53<1:34:27, 1.13s/it, step=21002, loss=0.0348, true_loss=15.9539, lr=1.52e-05]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 21000 | Train Loss: 0.0319 | Val Loss: 0.0438 | True train loss: 15.7137 | True val loss: 16.3045\n", "New best validation loss: 0.0438\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " \r" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Epoch 2/3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 8%|▋ | 1987/26015 [02:59<5:37:18, 1.19it/s, step=28002, loss=0.0224, true_loss=14.3285, lr=1.35e-05]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 28000 | Train Loss: 0.0284 | Val Loss: 0.0393 | True train loss: 15.0774 | True val loss: 15.2044\n", "New best validation loss: 0.0393\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 35%|███ | 8987/26015 [13:13<3:55:46, 1.20it/s, step=35002, loss=0.0302, true_loss=13.3737, lr=1.19e-05]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 35000 | Train Loss: 0.0257 | Val Loss: 0.0279 | True train loss: 14.4303 | True val loss: 14.4498\n", "New best validation loss: 0.0279\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 61%|████▉ | 15987/26015 [23:29<2:17:59, 1.21it/s, step=42002, loss=0.0264, true_loss=14.5345, lr=1.03e-05]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 42000 | Train Loss: 0.0245 | Val Loss: 0.0351 | True train loss: 14.1197 | True val loss: 14.2312\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 88%|████████▊ | 22987/26015 [33:46<41:56, 1.20it/s, step=49002, loss=0.0216, true_loss=14.1316, lr=8.70e-06]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 49000 | Train Loss: 0.0233 | Val Loss: 0.0290 | True train loss: 13.9434 | True val loss: 14.4628\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " \r" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Epoch 3/3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 15%|█▎ | 3971/26015 [05:52<7:13:46, 1.18s/it, step=56002, loss=0.0254, true_loss=14.4344, lr=7.08e-06]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 56000 | Train Loss: 0.0229 | Val Loss: 0.0479 | True train loss: 13.9115 | True val loss: 14.0929\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 42%|███▎ | 10971/26015 [16:06<4:48:34, 1.15s/it, step=63002, loss=0.0201, true_loss=12.8691, lr=5.47e-06]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 63000 | Train Loss: 0.0219 | Val Loss: 0.0239 | True train loss: 13.6746 | True val loss: 13.5177\n", "New best validation loss: 0.0239\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 69%|██████▏ | 17971/26015 [26:24<2:31:18, 1.13s/it, step=7e+4, loss=0.0248, true_loss=14.9835, lr=3.86e-06]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 70000 | Train Loss: 0.0212 | Val Loss: 0.0259 | True train loss: 13.5072 | True val loss: 13.7410\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Training: 96%|█████████▌| 24971/26015 [36:41<19:36, 1.13s/it, step=77002, loss=0.0228, true_loss=13.9553, lr=2.24e-06]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Step 77000 | Train Loss: 0.0207 | Val Loss: 0.0267 | True train loss: 13.4052 | True val loss: 13.8021\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " " ] }, { "name": "stdout", "output_type": "stream", "text": [ "Loaded best model with validation loss: 0.0239\n", "Training completed.\n", "Number of validation checkpoints: 11\n", "Final training losses: [0.022863016219410514, 0.02186289042873042, 0.021151691354678145, 0.020719855580878046, 0.020669196563010864]\n", "Best validation loss: 0.0239\n", "Model saved successfully!\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r" ] } ], "source": [ "import numpy as np\n", "import torch\n", "from torch.optim import AdamW\n", "from torch.optim.lr_scheduler import LinearLR\n", "from torch.utils.data import DataLoader\n", "from tqdm import tqdm\n", "\n", "train_losses, val_losses, best_loss = run_training(\n", " smiles_train, smiles_test, labels_train, labels_test, \n", " model, tokenizer, scalers, num_epochs=3, learning_rate=2e-5, batch_size=256, validation_steps=7000,\n", ")" ] } ], "metadata": { "kernelspec": { "display_name": "Python [conda env:.mlspace-bolgov_simson_training]", "language": "python", "name": "conda-env-.mlspace-bolgov_simson_training-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.11" } }, "nbformat": 4, "nbformat_minor": 5 }