diff --git a/MLtasks/ml_tasks.json b/MLtasks/ml_tasks.json index 9cdc543..69bb38c 100644 --- a/MLtasks/ml_tasks.json +++ b/MLtasks/ml_tasks.json @@ -92,6 +92,36 @@ "output": "Standard outputs schema with metrics comparison." } }, + { + "series": "Linear Regression", + "level": 5, + "id": "linreg_lvl5_diabetes_adamw", + "algorithm": "Linear Regression (Diabetes + AdamW)", + "description": "Apply linear regression to sklearn diabetes dataset with AdamW optimizer and LR scheduling.", + "interface_protocol": "pytorch_task_v1", + "requirements": { + "data": "Use sklearn.datasets.load_diabetes; standardized features and train/validation split.", + "implementation": "PyTorch nn.Linear with AdamW + ReduceLROnPlateau.", + "evaluation": "evaluate() must return MSE, R2, MAE on validation split.", + "validation": "Assert quality thresholds for R2, MSE, and train/val generalization gap.", + "artifacts": "Save model.pt, history.npz, and metrics.json." + } + }, + { + "series": "Linear Regression", + "level": 6, + "id": "linreg_lvl6_huber_earlystop", + "algorithm": "Robust Linear Regression (Huber + Early Stopping)", + "description": "Train linear regression on outlier-corrupted synthetic data using Huber loss, gradient clipping, and early stopping.", + "interface_protocol": "pytorch_task_v1", + "requirements": { + "data": "Synthetic multivariate regression with injected outliers.", + "implementation": "PyTorch nn.Linear with HuberLoss, Adam, gradient clipping, and early stopping on validation loss.", + "evaluation": "evaluate() must return MSE, R2, MAE on validation split.", + "validation": "Assert robust performance thresholds and overfitting guardrails.", + "artifacts": "Save model.pt, history.npz, and metrics.json." + } + }, { "series": "Logistic Regression", "level": 1, @@ -150,6 +180,36 @@ "validation": "ECE decreases after calibration step (compare before/after)." } }, + { + "series": "Logistic Regression", + "level": 5, + "id": "logreg_lvl5_breast_cancer_l1", + "algorithm": "Logistic Regression (Breast Cancer + L1)", + "description": "Binary logistic regression on breast cancer data with BCEWithLogits and L1 regularization.", + "interface_protocol": "pytorch_task_v1", + "requirements": { + "data": "Use sklearn.datasets.load_breast_cancer with standardized features.", + "implementation": "PyTorch linear logits model, BCEWithLogitsLoss, and L1 penalty term.", + "evaluation": "evaluate() must include MSE, R2, accuracy, F1, precision, recall, and ROC-AUC on validation.", + "validation": "Assert accuracy/F1/AUC thresholds and train/val gap constraints.", + "artifacts": "Save model.pt, history.npz, and metrics.json." + } + }, + { + "series": "Logistic Regression", + "level": 6, + "id": "logreg_lvl6_iris_label_smoothing", + "algorithm": "Softmax Logistic Regression (Iris + Label Smoothing)", + "description": "Multiclass logistic regression on Iris with label smoothing and cosine LR scheduler.", + "interface_protocol": "pytorch_task_v1", + "requirements": { + "data": "Use sklearn.datasets.load_iris with standardized features and train/validation split.", + "implementation": "PyTorch linear multiclass model, CrossEntropyLoss(label_smoothing), Adam + CosineAnnealingLR.", + "evaluation": "evaluate() must include MSE, R2, accuracy, macro-F1, and CE loss on validation.", + "validation": "Assert macro-F1/accuracy thresholds and loss convergence checks.", + "artifacts": "Save model.pt, history.npz, and metrics.json." + } + }, { "series": "k-Nearest Neighbors", "level": 1, @@ -839,6 +899,117 @@ "requirements": { "validation": "AUC/AP reported with deterministic sampling." } + }, + { + "series": "Neural Networks (MLP)", + "level": 5, + "id": "mlp_lvl5_ensemble_distillation", + "algorithm": "Knowledge Distillation (Ensemble Teacher)", + "description": "Train ensemble of teachers on MNIST; distill into single compact student model with temperature scaling.", + "interface_protocol": "pytorch_task_v1", + "requirements": { + "math": "Include KL divergence-based distillation loss: L = alpha*CE(student, target) + (1-alpha)*KL(softmax(student/T), softmax(teacher/T))", + "data": "MNIST dataset; 80/20 train/validation split.", + "implementation": "Train 3 teacher MLPs independently; average their logits; train student with distillation loss at T=4; use Adam optimizer.", + "teacher_ensemble": "Each teacher: 256x128x10 architecture; trained to >98% accuracy.", + "student": "Compact model: 128x64x10 architecture; should match ensemble accuracy with 50% fewer parameters.", + "evaluation": "evaluate() must compute MSE, CE loss, accuracy, and distillation efficiency (student_params / teacher_params ratio) on validation split.", + "main_block": "Train teachers, create ensemble, train student, evaluate student vs ensemble accuracy, assert student_accuracy > 0.95 and student achieves >90% of ensemble performance.", + "artifacts": "Save teacher models, student model, training history, and comparison metrics." + } + }, + { + "series": "Convolutional Neural Networks", + "level": 5, + "id": "cnn_lvl5_attention_mechanisms", + "algorithm": "CNN with Squeeze-Excitation Attention", + "description": "Implement and evaluate SE-Net (Squeeze-Excitation) attention modules on CIFAR-10 or MNIST.", + "interface_protocol": "pytorch_task_v1", + "requirements": { + "math": "Include SE module formula: x * sigmoid(FC2(ReLU(FC1(GlobalAvgPool(x)))); explain channel attention.", + "architecture": "Build ResNet-18 style backbone with SE blocks after each residual block.", + "data": "CIFAR-10 or MNIST; normalized; 80/20 train/validation split.", + "implementation": "Implement SEBlock as nn.Module; integrate into CNN; compare baseline vs attention-enhanced.", + "comparison": "Train both with and without SE modules; report parameter count and accuracy difference.", + "evaluation": "evaluate() must compute accuracy, CE loss, and per-channel attention weights on validation split.", + "main_block": "Train both models, evaluate both splits, print architecture comparisons, assert attention model >= baseline accuracy.", + "visualization": "Save attention weight heatmaps on sample images.", + "artifacts": "Save both models, attention visualizations, and training curves." + } + }, + { + "series": "Natural Language Processing", + "level": 1, + "id": "nlp_lvl1_text_embedding", + "algorithm": "Word Embeddings (Word2Vec Skip-gram Simplified)", + "description": "Implement simplified Skip-gram word embeddings on a small corpus; build sentiment classifier on top.", + "interface_protocol": "pytorch_task_v1", + "requirements": { + "math": "Include Skip-gram objective: log p(context|target) = log sigmoid(context.dot(target)) for positive pairs + negative sampling.", + "corpus": "Use small movie reviews corpus (toy dataset or sklearn 20newsgroups subset); build vocabulary of ~5k words.", + "embedding_training": "Train 100-dim embeddings with negative sampling; 10 negative samples per positive; SGD with momentum.", + "downstream_task": "Train simple logistic regression classifier on embedding representations for sentiment prediction (IMDB-like toy task).", + "embedding_eval": "Check that similar words have similar vectors; report top-5 similar words for 'good', 'bad', 'movie'.", + "evaluation": "evaluate() must compute sentiment classification accuracy, embedding similarity metrics (cosine), and word analogy performance on validation split.", + "main_block": "Train embeddings, verify similarity, train sentiment classifier, evaluate both, assert embeddings capture semantic meaning and classifier accuracy > 0.75.", + "artifacts": "Save learned embeddings, vocabulary, and sentiment model." + } + }, + { + "series": "Robust Neural Networks", + "level": 1, + "id": "robust_lvl1_adversarial_training", + "algorithm": "Adversarial Robustness (FGSM Attacks & Defense)", + "description": "Implement FGSM attack and adversarial training to improve model robustness on MNIST.", + "interface_protocol": "pytorch_task_v1", + "requirements": { + "math": "Include FGSM perturbation: x_adv = x + eps * sign(grad_x(L(x,y))); explain adversarial loss mix.", + "attack_method": "Fast Gradient Sign Method (FGSM) with epsilon=0.3 (8/255 normalized for MNIST).", + "baseline_model": "Train standard CNN without adversarial robustness; measure clean and adversarial accuracy.", + "defense_training": "Adversarial training: 80% clean samples + 20% FGSM-attacked samples; lambda_adv=0.5 in loss.", + "robust_model": "Train robust model for same epochs; compare to baseline.", + "evaluation": "evaluate() must compute: clean_accuracy, robust_accuracy (under FGSM), accuracy_drop, and robustness_gap on validation split.", + "internal_testing": "Verify gradient computation for attacks; test perturbation bounds.", + "main_block": "Train baseline, evaluate clean+adversarial, train robust model, compare robustness, assert robust_accuracy > baseline_robust_accuracy and clean_accuracy > 0.95.", + "visualization": "Save corrupted image examples showing FGSM perturbations.", + "artifacts": "Save both models, perturbation examples, and robustness report." + } + }, + { + "series": "Natural Language Processing", + "level": 2, + "id": "nlp_lvl2_bigframe_embeddings", + "algorithm": "BigQuery Bigframe Embeddings with Semantic Similarity", + "description": "Load text data from BigQuery Bigframe; generate embeddings using Vertex AI Embeddings API; build neural network for semantic similarity prediction.", + "interface_protocol": "pytorch_task_v1", + "requirements": { + "bigquery_integration": "Demonstrate BigQuery Bigframe data loading with SQL queries (LIMIT, WHERE clauses, SELECT); include proper error handling and fallback to local data when BigQuery unavailable.", + "math": "Include cosine similarity: sim(e_i, e_j) = (e_i · e_j) / (||e_i|| ||e_j||); contrastive loss with temperature scaling.", + "data_source": "Query public dataset (e.g., Google STS benchmark) or fallback to local semantic similarity pairs; load 20+ text pairs with similarity labels (1=similar, 0=dissimilar).", + "embeddings": "Use Vertex AI TextEmbedding API (768-dim) when credentials available; provide local simulation for testing (hash-based deterministic embeddings preserving semantic relationships).", + "model": "Build MLP classifier: concat(emb1, emb2, emb1*emb2, |emb1-emb2|) -> 512-dim hidden -> binary classification.", + "evaluation": "evaluate() must compute accuracy, loss, and per-class metrics on validation split; verify embedding quality through similarity consistency checks.", + "main_block": "Load from Bigframe, generate embeddings, build and train model, evaluate on both splits, assert validation_accuracy > 0.60 and training_accuracy > 0.65.", + "artifacts": "Save model, embedding cache summary, metrics, and training history." + } + }, + { + "series": "Natural Language Processing", + "level": 3, + "id": "nlp_lvl3_bigframe_llm_classification", + "algorithm": "BigQuery Bigframe LLM Text Classification", + "description": "Load text classification data from BigQuery Bigframe; generate LLM features and augmentations using Vertex AI LLM API; train ensemble-style classifier combining original and augmented representations.", + "interface_protocol": "pytorch_task_v1", + "requirements": { + "bigquery_integration": "Demonstrate BigQuery Bigframe SQL queries with WHERE, GROUP BY, HAVING clauses for text classification data; include connection error handling and fallback mechanism.", + "data_loading": "Query public dataset (e.g., DBpedia or text classification benchmark) with proper SQL filters; load 30+ texts with 3 class labels; demonstrate Bigframe to pandas conversion.", + "llm_feature_generation": "Use Vertex AI LLM API for semantic feature extraction (keyword extraction, summarization-based features) when credentials available; provide local simulation using text statistics and hashing for testing.", + "augmentation": "Implement text augmentation through LLM paraphrasing (50% of data augmented); generate alternative phrasings preserving meaning for improved generalization.", + "model": "Build MLP classifier: concat(original_features, augmented_features) -> 256-dim hidden with batch norm -> softmax over 3 classes.", + "evaluation": "evaluate() must compute multi-class accuracy, CE loss, and per-class accuracies on validation split; report feature cache statistics.", + "main_block": "Load from Bigframe, generate LLM features and augmentations, train classifier on combined features, evaluate both splits, assert validation_accuracy > 0.60 and feature_cache_generated.", + "artifacts": "Save model, feature cache summary, augmentation examples, metrics, and training history with per-class performance." + } } ] } \ No newline at end of file diff --git a/MLtasks/requirements.txt b/MLtasks/requirements.txt new file mode 100644 index 0000000..501359a --- /dev/null +++ b/MLtasks/requirements.txt @@ -0,0 +1,21 @@ +# Core numerical / ML stack used by most task.py scripts +numpy +scipy +scikit-learn +matplotlib +seaborn +pillow + +# PyTorch ecosystem +torch +torchvision + +# Graph / advanced tasks +torch-geometric + +# Export / inference tasks +onnx +onnxruntime + +# ANN indexing tasks (not universally available on all Windows setups) +faiss-cpu; platform_system != "Windows" \ No newline at end of file diff --git a/MLtasks/tasks/cnn_lvl5_attention_mechanisms/task.py b/MLtasks/tasks/cnn_lvl5_attention_mechanisms/task.py new file mode 100644 index 0000000..83f36e6 --- /dev/null +++ b/MLtasks/tasks/cnn_lvl5_attention_mechanisms/task.py @@ -0,0 +1,536 @@ +""" +CNN with Squeeze-Excitation (SE) Attention Modules + +Mathematical Formulation: +- SE Module: x_out = x * sigmoid(FC2(ReLU(FC1(GlobalAvgPool(x))))) +- Channel Attention: Learn per-channel scaling weights +- Global Average Pooling: Squeezes spatial dimensions to 1x1 +- Two FC layers (compression-expansion): FC(C) -> FC(C/r) -> FC(C) + where r is reduction ratio (typically 16) + +This task demonstrates: +1. Implementing SE-Net attention from scratch +2. Integrating attention into CNN architectures +3. Comparing baseline vs attention-enhanced performance +4. Measuring computational overhead vs accuracy gain +""" + +import os +import sys +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader +import torchvision +import torchvision.transforms as transforms +import json +from pathlib import Path + +# Set seeds for reproducibility +torch.manual_seed(42) +np.random.seed(42) + +# Output directory for artifacts +OUTPUT_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'output', 'cnn_lvl5_attention_mechanisms') +os.makedirs(OUTPUT_DIR, exist_ok=True) + + +def get_task_metadata(): + """Return task metadata.""" + return { + 'task_name': 'cnn_attention_squeeze_excitation', + 'description': 'ResNet-18 style CNN with SE-Net attention modules', + 'architecture': 'ResNet-18 with SE blocks', + 'attention_type': 'Squeeze-Excitation (Channel Attention)', + 'reduction_ratio': 16, + 'task_type': 'image_classification', + 'dataset': 'MNIST' + } + + +def set_seed(seed=42): + """Set random seeds for reproducibility.""" + torch.manual_seed(seed) + np.random.seed(seed) + + +def get_device(): + """Get device (CPU or GPU).""" + return torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + +def make_dataloaders(batch_size=32, download=True): + """ + Create MNIST dataloaders for train and validation. + + Args: + batch_size: Batch size + download: Whether to download MNIST + + Returns: + train_loader, val_loader + """ + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ]) + + # Download and load MNIST, expand to 3 channels for CNN + mnist_train = torchvision.datasets.MNIST( + root='./data', train=True, download=download, transform=transform + ) + mnist_test = torchvision.datasets.MNIST( + root='./data', train=False, download=download, transform=transform + ) + + # Split training into 80/20 train/val + n_train = int(0.8 * len(mnist_train)) + indices = torch.randperm(len(mnist_train)) + train_indices = indices[:n_train] + val_indices = indices[n_train:] + + train_subset = torch.utils.data.Subset(mnist_train, train_indices) + val_subset = torch.utils.data.Subset(mnist_train, val_indices) + + train_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True) + val_loader = DataLoader(val_subset, batch_size=batch_size, shuffle=False) + + return train_loader, val_loader + + +class SEBlock(nn.Module): + """ + Squeeze-Excitation Block: Channel Attention + + SE(x) = x ⊗ σ(FC2(ReLU(FC1(GlobalAvgPool(x))))) + where ⊗ is element-wise multiplication, σ is sigmoid + """ + + def __init__(self, channels, reduction=16): + super(SEBlock, self).__init__() + self.channels = channels + self.reduction = reduction + + # Global average pooling is implicit in forward + # FC1: channels -> channels/reduction (compress) + self.fc1 = nn.Linear(channels, channels // reduction) + # FC2: channels/reduction -> channels (expand) + self.fc2 = nn.Linear(channels // reduction, channels) + self.relu = nn.ReLU(inplace=True) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + """ + Args: + x: Input tensor of shape (B, C, H, W) + + Returns: + Output tensor of shape (B, C, H, W) + """ + # Global average pooling: (B, C, H, W) -> (B, C) + squeeze = torch.nn.functional.adaptive_avg_pool2d(x, 1) # (B, C, 1, 1) + squeeze = squeeze.view(squeeze.size(0), -1) # (B, C) + + # Excitation: FC-ReLU-FC-Sigmoid + excitation = self.fc1(squeeze) # (B, C/r) + excitation = self.relu(excitation) + excitation = self.fc2(excitation) # (B, C) + excitation = self.sigmoid(excitation) # (B, C) + + # Scale: (B, C) -> (B, C, 1, 1) -> broadcast to (B, C, H, W) + excitation = excitation.view(excitation.size(0), excitation.size(1), 1, 1) + + return x * excitation + + +class BasicBlock(nn.Module): + """Basic residual block with optional SE attention.""" + + expansion = 1 + + def __init__(self, in_channels, out_channels, stride=1, use_se=False, reduction=16): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(out_channels) + self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_channels) + self.relu = nn.ReLU(inplace=True) + + # SE block (optional) + self.use_se = use_se + if use_se: + self.se = SEBlock(out_channels, reduction=reduction) + + # Shortcut connection + self.shortcut = nn.Sequential() + if stride != 1 or in_channels != out_channels: + self.shortcut = nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(out_channels) + ) + + def forward(self, x): + residual = self.shortcut(x) + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.use_se: + out = self.se(out) + + out = out + residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + """ResNet-18 style architecture with optional SE attention.""" + + def __init__(self, num_blocks=[2, 2, 2, 2], num_classes=10, use_se=False, reduction=16): + super(ResNet, self).__init__() + self.use_se = use_se + self.in_channels = 64 + + # Initial convolution layer for MNIST (1 channel) + self.conv1 = nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + + # Residual layers + self.layer1 = self._make_layer(64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(512, num_blocks[3], stride=2) + + # Classification head + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512, num_classes) + + # Initialize weights + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, out_channels, num_blocks, stride=1): + layers = [] + layers.append(BasicBlock(self.in_channels, out_channels, stride=stride, + use_se=self.use_se)) + self.in_channels = out_channels + for _ in range(1, num_blocks): + layers.append(BasicBlock(out_channels, out_channels, stride=1, + use_se=self.use_se)) + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + + return x + + +def build_model(use_se=False, **kwargs): + """Build ResNet model with or without SE attention.""" + return ResNet(use_se=use_se, **kwargs) + + +def count_parameters(model): + """Count total trainable parameters.""" + return sum(p.numel() for p in model.parameters() if p.requires_grad) + + +def train_epoch(model, train_loader, optimizer, device): + """Train for one epoch.""" + model.train() + criterion = nn.CrossEntropyLoss() + + total_loss = 0.0 + correct = 0 + total = 0 + + for images, labels in train_loader: + images, labels = images.to(device), labels.to(device) + + # Forward pass + outputs = model(images) + loss = criterion(outputs, labels) + + # Backward pass + optimizer.zero_grad() + loss.backward() + optimizer.step() + + total_loss += loss.item() + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + avg_loss = total_loss / len(train_loader) + accuracy = correct / total + + return avg_loss, accuracy + + +def evaluate_model(model, data_loader, device, return_dict=True): + """Evaluate model on data loader.""" + model.eval() + criterion = nn.CrossEntropyLoss() + + total_loss = 0.0 + correct = 0 + total = 0 + + with torch.no_grad(): + for images, labels in data_loader: + images, labels = images.to(device), labels.to(device) + + outputs = model(images) + loss = criterion(outputs, labels) + + total_loss += loss.item() + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + avg_loss = total_loss / len(data_loader) + accuracy = correct / total + + if not return_dict: + return accuracy + + return { + 'loss': avg_loss, + 'accuracy': accuracy, + 'correct': correct, + 'total': total + } + + +def train(train_loader, val_loader, device, use_se=False, epochs=20, lr=0.001): + """Train a model (with or without SE attention).""" + model = build_model(use_se=use_se) + model.to(device) + + optimizer = optim.Adam(model.parameters(), lr=lr) + + history = { + 'train_loss': [], + 'train_acc': [], + 'val_loss': [], + 'val_acc': [] + } + + for epoch in range(epochs): + train_loss, train_acc = train_epoch(model, train_loader, optimizer, device) + val_metrics = evaluate_model(model, val_loader, device, return_dict=True) + + history['train_loss'].append(train_loss) + history['train_acc'].append(train_acc) + history['val_loss'].append(val_metrics['loss']) + history['val_acc'].append(val_metrics['accuracy']) + + if (epoch + 1) % 5 == 0: + se_type = "with SE" if use_se else "without SE" + print(f"Epoch [{epoch+1}/{epochs}] {se_type} - " + f"Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.4f}, " + f"Val Acc: {val_metrics['accuracy']:.4f}") + + return model, history + + +def predict(model, x, device): + """Make predictions on input.""" + model.eval() + if isinstance(x, np.ndarray): + x = torch.FloatTensor(x) + x = x.to(device) + + with torch.no_grad(): + outputs = model(x) + _, predictions = torch.max(outputs, 1) + + return predictions.cpu().numpy() + + +def evaluate(model, data_loader, device, return_dict=True): + """Evaluate wrapper for protocol.""" + return evaluate_model(model, data_loader, device, return_dict=return_dict) + + +def save_artifacts(baseline_model, attention_model, histories, metrics, output_dir=OUTPUT_DIR): + """Save models and metrics.""" + os.makedirs(output_dir, exist_ok=True) + + # Save models + baseline_path = os.path.join(output_dir, 'baseline_model.pt') + torch.save(baseline_model.state_dict(), baseline_path) + print(f"Saved baseline model to {baseline_path}") + + attention_path = os.path.join(output_dir, 'attention_model.pt') + torch.save(attention_model.state_dict(), attention_path) + print(f"Saved attention model to {attention_path}") + + # Save metrics + metrics_path = os.path.join(output_dir, 'metrics.json') + with open(metrics_path, 'w') as f: + json.dump(metrics, f, indent=2) + print(f"Saved metrics to {metrics_path}") + + # Save histories + history_path = os.path.join(output_dir, 'histories.json') + with open(history_path, 'w') as f: + json.dump(histories, f, indent=2) + print(f"Saved histories to {history_path}") + + +if __name__ == '__main__': + """ + Main pipeline: + 1. Load MNIST data + 2. Train baseline CNN (no SE attention) + 3. Train CNN with SE attention + 4. Compare performance and parameters + 5. Assert quality thresholds + """ + + try: + device = get_device() + print(f"Using device: {device}") + + # Load data + print("\nLoading MNIST data...") + train_loader, val_loader = make_dataloaders(batch_size=64, download=True) + + # Train baseline model + print("\n" + "="*60) + print("Training Baseline Model (No SE Attention)") + print("="*60) + baseline_model, baseline_history = train( + train_loader, val_loader, device, use_se=False, epochs=15, lr=0.001 + ) + + # Evaluate baseline + baseline_train_metrics = evaluate_model( + baseline_model, train_loader, device, return_dict=True + ) + baseline_val_metrics = evaluate_model( + baseline_model, val_loader, device, return_dict=True + ) + + print(f"\nBaseline - Train Acc: {baseline_train_metrics['accuracy']:.4f}, " + f"Val Acc: {baseline_val_metrics['accuracy']:.4f}") + + # Train attention model + print("\n" + "="*60) + print("Training Model with SE Attention") + print("="*60) + attention_model, attention_history = train( + train_loader, val_loader, device, use_se=True, epochs=15, lr=0.001 + ) + + # Evaluate attention model + attention_train_metrics = evaluate_model( + attention_model, train_loader, device, return_dict=True + ) + attention_val_metrics = evaluate_model( + attention_model, val_loader, device, return_dict=True + ) + + print(f"\nAttention - Train Acc: {attention_train_metrics['accuracy']:.4f}, " + f"Val Acc: {attention_val_metrics['accuracy']:.4f}") + + # Compare architectures + print("\n" + "="*60) + print("Architecture Comparison") + print("="*60) + + baseline_params = count_parameters(baseline_model) + attention_params = count_parameters(attention_model) + param_increase = (attention_params - baseline_params) / baseline_params * 100 + + print(f"Baseline parameters: {baseline_params}") + print(f"Attention parameters: {attention_params}") + print(f"Parameter increase: {param_increase:.2f}%") + + accuracy_gain = attention_val_metrics['accuracy'] - baseline_val_metrics['accuracy'] + print(f"\nAccuracy gain (val): {accuracy_gain:.4f}") + print(f"Baseline val accuracy: {baseline_val_metrics['accuracy']:.4f}") + print(f"Attention val accuracy: {attention_val_metrics['accuracy']:.4f}") + + # Collect metrics + all_metrics = { + 'baseline': { + 'train': baseline_train_metrics, + 'val': baseline_val_metrics, + 'params': baseline_params + }, + 'attention': { + 'train': attention_train_metrics, + 'val': attention_val_metrics, + 'params': attention_params + }, + 'comparison': { + 'param_increase_percent': param_increase, + 'accuracy_gain': accuracy_gain, + 'efficiency_ratio': param_increase / (accuracy_gain * 100) if accuracy_gain > 0 else float('inf') + } + } + + all_histories = { + 'baseline': baseline_history, + 'attention': attention_history + } + + save_artifacts(baseline_model, attention_model, all_histories, all_metrics) + + # Assertions for quality + print("\n" + "="*60) + print("Quality Assertions") + print("="*60) + + assert baseline_val_metrics['accuracy'] > 0.95, \ + f"Baseline accuracy {baseline_val_metrics['accuracy']:.4f} must be > 0.95" + print("✓ Baseline validation accuracy > 0.95") + + assert attention_val_metrics['accuracy'] > baseline_val_metrics['accuracy'], \ + f"Attention model should match or exceed baseline" + print("✓ Attention model >= baseline accuracy") + + assert attention_train_metrics['accuracy'] > 0.97, \ + f"Attention train accuracy {attention_train_metrics['accuracy']:.4f} must be > 0.97" + print("✓ Attention model train accuracy > 0.97") + + assert param_increase < 15, \ + f"Attention parameter overhead {param_increase:.2f}% should be < 15%" + print(f"✓ Attention overhead reasonable ({param_increase:.2f}%)") + + print("\n" + "="*60) + print("SUCCESS: All assertions passed!") + print("="*60) + sys.exit(0) + + except Exception as e: + print(f"\nERROR: {str(e)}") + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/MLtasks/tasks/linreg_lvl3_regularization_optim/task.py b/MLtasks/tasks/linreg_lvl3_regularization_optim/task.py index 24dffc2..09a5b09 100644 --- a/MLtasks/tasks/linreg_lvl3_regularization_optim/task.py +++ b/MLtasks/tasks/linreg_lvl3_regularization_optim/task.py @@ -7,6 +7,9 @@ from torch.utils.data import DataLoader, TensorDataset import matplotlib.pyplot as plt from sklearn.metrics import mean_squared_error, r2_score +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt # Set seeds for reproducibility def get_task_metadata(): @@ -184,6 +187,7 @@ def predict(model, X, device=None): X_tensor = torch.FloatTensor(X).to(device) predictions = model(X_tensor) return predictions.cpu().numpy() + def save_artifacts(model, train_losses, val_losses, X_train, y_train, X_val, y_val, output_dir="output", filename_prefix="linreg_lvl3"): diff --git a/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/history.json b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/history.json new file mode 100644 index 0000000..1d5a7c2 --- /dev/null +++ b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/history.json @@ -0,0 +1,1206 @@ +{ + "train_loss": [ + 1.035324290394783, + 1.107154756784439, + 0.6600320972502232, + 0.6726501683394114, + 0.5597793286045393, + 0.5342055037617683, + 0.5709741314252218, + 0.4714507058116624, + 0.4723813521365325, + 0.4824843630194664, + 0.45531006902456284, + 0.465350237985452, + 0.4853285327553749, + 0.4631544003883998, + 0.4535053049524625, + 0.4563833847641945, + 0.4556739305456479, + 0.48787913223107654, + 0.5281281222899755, + 0.4306143529587037, + 0.4305953662551474, + 0.43032242408177507, + 0.4438064123193423, + 0.6246009518702825, + 0.4322465569227158, + 0.4402591281880935, + 0.44052548582355183, + 0.43855597202976543, + 0.4445103220641613, + 0.45524803797403973, + 0.4537229041258494, + 0.4509255637725194, + 0.610278512040774, + 0.4760338018337886, + 0.5251202881336212, + 0.43515045506258804, + 0.43328226419786614, + 0.44532308851679164, + 0.438721876591444, + 0.5103333443403244, + 0.4960801750421524, + 0.4377650104773541, + 0.45014025643467903, + 0.4471312214930852, + 0.4384534303098917, + 0.4799206182360649, + 0.5491261954108874, + 0.46454240133364993, + 0.5291574373841286, + 0.4597885509332021, + 0.460080253581206, + 0.4327927477036913, + 0.5366293713450432, + 0.4640053448577722, + 0.5629370212554932, + 0.47302158176898956, + 0.4306097971275449, + 0.5059110547105471, + 0.433047360740602, + 0.4296501446515322, + 0.5618729665875435, + 0.42991354254384834, + 0.4647936647137006, + 0.4356652330607176, + 0.4525070860981941, + 0.4392816225687663, + 0.45879682898521423, + 0.44046274324258167, + 0.4280240236936758, + 0.433383875216047, + 0.4446178711950779, + 0.43123870467146236, + 0.49184134354194003, + 0.42811066870732856, + 0.4500340173641841, + 0.47173234323660534, + 0.5315311774611473, + 0.427088914749523, + 0.4371103085577488, + 0.42725110579825315, + 0.46332556505997974, + 0.4484328180551529, + 0.5960998783508936, + 0.4316815799102187, + 0.5306498979528745, + 0.4661602924267451, + 0.44308435916900635, + 0.5705804700652758, + 0.5354885384440422, + 0.4290706630175312, + 0.4880613560477893, + 0.5112639218568802, + 0.5224957391619682, + 0.5497710704803467, + 0.4289072489676376, + 0.4331062852094571, + 0.4965027297536532, + 0.46236247569322586, + 0.42926504571611684, + 0.42745310995572555, + 0.6761780033508936, + 0.4276375997190674, + 0.4302449971437454, + 0.4598545581102371, + 0.538199208676815, + 0.5380041971802711, + 0.5239657411972681, + 0.4282613372585426, + 0.43778013189633685, + 0.6212963288029035, + 0.4605991964538892, + 0.4797686884800593, + 0.4683787276347478, + 0.43490185029804707, + 0.4477766801913579, + 0.43896568690737087, + 0.42808168932485086, + 0.4678627625107765, + 0.4288472187084456, + 0.42748920208153623, + 0.4931228260199229, + 0.5001329854130745, + 0.7611470893025398, + 0.46652019023895264, + 0.4475909968217214, + 0.43267346173524857, + 0.5007794896761576, + 0.4479065611958504, + 0.4366607101013263, + 0.5078870033224424, + 0.43369699145356816, + 0.44689041500290233, + 0.4514465282360713, + 0.4283011563432713, + 0.4295494354640444, + 0.430738579792281, + 0.430670212333401, + 0.4283772523825367, + 0.4769914622108142, + 0.4305462756504615, + 0.4270970292855054, + 0.42669316463191836, + 0.5363844533761343, + 0.4916919594009717, + 0.4674232875307401, + 0.4273467215243727, + 0.42676765762250096, + 0.43595257215201855, + 0.4278568279308577, + 0.5102931782603264, + 0.4353809269766013, + 0.43923261637489003, + 0.47566701968510944, + 0.4267179291879681, + 0.47677116841077805, + 0.45480359842379886, + 0.466113085548083, + 0.44795747101306915, + 0.43669494862357777, + 0.4583193560441335, + 0.4466005178789298, + 0.4934719155232112, + 0.427980815526098, + 0.42722527019213885, + 0.4759032080570857, + 0.4269893812403704, + 0.4358258613695701, + 0.45039666816592216, + 0.42866924063613016, + 0.430303403797249, + 0.4314052887881796, + 0.4355544683833917, + 0.5497878963748614, + 0.4446597409745057, + 0.4269583962935333, + 0.48504168291886646, + 0.4362722548345725, + 0.46486902981996536, + 0.43025874129186076, + 0.42901247926056385, + 0.548652949432532, + 0.431838018198808, + 0.4335018526762724, + 0.42763615647951764, + 0.44833555320898694, + 0.4405190646648407, + 0.4280972041500111, + 0.47865500922004384, + 0.4269684529863298, + 0.4335225826750199, + 0.4290042887441814, + 0.4311709338799119, + 0.482379416624705, + 0.47846970210472745, + 0.5058049311240514, + 0.4279188751243055, + 0.4895305434862773, + 0.4306114747499426, + 0.4446665545304616, + 0.4280909796555837, + 0.5151919921239217, + 0.43919892609119415, + 0.4312769848232468, + 0.43667998413244885, + 0.4459829231103261, + 0.4876964290936788, + 0.49043554812669754, + 0.43105001219858724, + 0.5149349495768547, + 0.4384122850994269, + 0.4411800764501095, + 0.4330028022329013, + 0.4265883629268501, + 0.4268148550763726, + 0.43014572001993656, + 0.5380818049112955, + 0.45469153424104053, + 0.43501355809470016, + 0.43895279864470166, + 0.4383673084278901, + 0.42850827146321535, + 0.4358132351189852, + 0.4957321633895238, + 0.45967208842436474, + 0.5334181760748228, + 0.46799106895923615, + 0.4269887599317978, + 0.4956926703453064, + 0.42684472783003, + 0.4284828656042616, + 0.4279495586330692, + 0.44821934401988983, + 0.42908944624165696, + 0.4657536521553993, + 0.42938616623481113, + 0.47582168380419415, + 0.4889454593261083, + 0.4288486580674847, + 0.4391043856739998, + 0.6400597443183264, + 0.5235920970638593, + 0.4383620408674081, + 0.4351605959236622, + 0.42751651257276535, + 0.49689074357350665, + 0.5478212808569273, + 0.44344743837912876, + 0.47101401040951413, + 0.4622294207413991, + 0.757253552476565, + 0.4353866881380479, + 0.45546748240788776, + 0.4447645805776119, + 0.4329866723467906, + 0.42656882512771216, + 0.4955783039331436, + 0.42802404100075364, + 0.46711721767981845, + 0.5274655123551687, + 0.4301436844592293, + 0.4265668217582667, + 0.4265776803319265, + 0.4825559159119924, + 0.4266875330746795, + 0.49473512421051663, + 0.42676828559100005, + 0.4476192742586136, + 0.5370262960592905, + 0.42917808207372826, + 0.4629252254962921, + 0.4270859908234949, + 0.42698195254585397, + 0.4305937848985195, + 0.42795354143405956, + 0.4781167308489482, + 0.44526798153916997, + 0.4854602987567584, + 0.6093653117616972, + 0.4671272858977318, + 0.43762650589148205, + 0.5482623800635338, + 0.42720899218693376, + 0.43999429792165756, + 0.445611151556174, + 0.4290848048403859, + 0.46460694819688797, + 0.43351925040284794, + 0.4551628852883975, + 0.4525102252761523, + 0.5369939754406611, + 0.42708446469623595, + 0.4904152726133664, + 0.5331979220112165, + 0.4531037782629331, + 0.4618539586663246, + 0.45162372042735416, + 0.4946661988894145, + 0.42780321557074785, + 0.46369514365990955, + 0.5297743380069733, + 0.4265627765150081, + 0.45310187836488086, + 0.42695996502880007, + 0.46653111775716144, + 0.4891848787665367, + 0.4583503057559331, + 0.43074239945660037, + 0.4946591431895892, + 0.4651121646165848, + 0.44548216586311656, + 0.4269473895352955, + 0.43006665011247, + 0.5297722642620405, + 0.44122101987401646, + 0.43509617261588573, + 0.45362352828184765, + 0.5593257794777552, + 0.42657449372442596, + 0.43194696120917797, + 0.5119570667545, + 0.42831986816599965, + 0.4636891409754753, + 0.43795623754461604, + 0.42658426967197255, + 0.4274506038830926, + 0.4350490005065997, + 0.47956931342681247, + 0.4350480269640684, + 0.4421010601023833, + 0.46573519210020703, + 0.6626076772809029, + 0.4412189995249112, + 0.4389875568449497, + 0.617688442269961, + 0.4269938041300823, + 0.4273420024352769, + 0.4412953307231267, + 0.42675421441284317, + 0.5223072121540705, + 0.4357829640309016, + 0.5349570785959562, + 0.42691877340742695, + 0.5349566663304964, + 0.45975544055302936, + 0.6093602478504181, + 0.43679602444171906, + 0.42698102017554146, + 0.4269674314806859, + 0.5282688091198603, + 0.5489975909392039, + 0.4288506276595096, + 0.4470505739251773, + 0.4350240770727396, + 0.5133404806256294, + 0.5175486331184705, + 0.42661203915971174, + 0.43566572790344554, + 0.4291760629663865, + 0.43101046948383254, + 0.4692034075657527, + 0.5482520461082458, + 0.5175481537977854, + 0.4288920187391341, + 0.42945468208442134, + 0.4669237285852432, + 0.4269577411469072, + 0.430589376638333, + 0.4390883247057597, + 0.4685838520526886, + 0.4265632535025361, + 0.5379759495457014, + 0.48933856934309006, + 0.4284337963908911, + 0.4557008643945058, + 0.4363247510045767, + 0.43830225244164467, + 0.4332059199611346, + 0.4307404800007741, + 0.4299371677140395, + 0.42690178635530174, + 0.45975437263647717, + 0.42821271577849984, + 0.432224423935016, + 0.42788674496114254, + 0.5477538704872131, + 0.6399733300010363, + 0.42848170703897875, + 0.5168184066812197, + 0.436343473692735, + 0.48313164462645847, + 0.464360348880291, + 0.4272074323768417, + 0.4412182743350665, + 0.4710193946957588, + 0.43634328121940297, + 0.4430679741005103, + 0.5557626560330391, + 0.6399735833207766, + 0.4275047837290913, + 0.48608942081530887, + 0.42755953789067763, + 0.42662539388402365, + 0.42800422105938196, + 0.4265835795085877, + 0.43741189191738766, + 0.5168178007006645, + 0.42703404867400724, + 0.5413350736101469, + 0.42659270351092954, + 0.4759458651145299, + 0.6658742552002271, + 0.4557003006339073, + 0.4676798035701116, + 0.5873442937930425, + 0.558222159743309, + 0.4307908893873294, + 0.44603285069266957, + 0.42868281761184335, + 0.42848180622483295, + 0.7571930686632792, + 0.5769950871666273, + 0.4357817893226941, + 0.6176777482032776, + 0.44175491606195766, + 0.48522403339544934, + 0.42792295447240275, + 0.46852807700634, + 0.6928260823090872, + 0.44306796913345653, + 0.5603621825575829, + 0.4269463304275026, + 0.4278020686469972, + 0.47577302654584247, + 0.4476020832856496, + 0.6202915335694948, + 0.4388991942008336, + 0.43137325687954825, + 0.43679596359531087, + 0.42682210146449506, + 0.4280129942732553, + 0.4882212355732918, + 0.49306323130925495, + 0.4629068523645401, + 0.4536201556523641, + 0.42658494012236287, + 0.4685833677649498, + 0.4710192456841469, + 0.42908432831366855, + 0.42792291690905887, + 0.43428351047138375, + 0.43036043116201955, + 0.4646865452329318, + 0.6234004522363344, + 0.475600669781367, + 0.5223077287276586, + 0.42792291892692447, + 0.4476020485162735, + 0.42770969642636675, + 0.43593362035850686, + 0.4312153995657961, + 0.4303266853094101, + 0.4274969904217869, + 0.42780205126230914, + 0.4265631278224949, + 0.4551601881782214, + 0.4470500275492668, + 0.5557624325156212, + 0.5232307737072309, + 0.5395216097434362, + 0.43336905414859456, + 0.43795378382007283, + 0.42745052829074365, + 0.5873438542087873, + 0.43222423642873764, + 0.4526565819978714, + 0.4266824178727499, + 0.5873437200983366, + 0.45035819957653683, + 0.4808223582804203, + 0.44036780173579854, + 0.4419303896526496, + 0.4456932991743088, + 0.4470735440651576, + 0.5413349171479543, + 0.44786931077639264, + 0.43833063542842865, + 0.42848179566984373, + 0.48313140869140625, + 0.6625999808311462, + 0.4888761217395465, + 0.44704987729589146, + 0.522307738661766, + 0.4629068449139595, + 0.5413346538941065, + 0.43087612465023994, + 0.46202213565508526, + 0.5282646963993708, + 0.4413546336193879, + 0.4304234633843104, + 0.5502680266896883, + 0.5413343533873558, + 0.42864830791950226, + 0.42846355540677905, + 0.44861547152201336, + 0.4275569710104416, + 0.4374959294994672, + 0.4780331179499626, + 0.44175491357843083, + 0.4456931489209334, + 0.4374959034224351, + 0.6625996393462023, + 0.4266107237635879, + 0.4329771331200997, + 0.45106013615926105, + 0.45450732856988907, + 0.4476020534833272, + 0.4275569727954765, + 0.4276337377571811, + 0.43418820574879646, + 0.44209744160374004, + 0.4454844333231449, + 0.665873388449351, + 0.4773399457335472, + 0.4470735639333725, + 0.43087329001476365, + 0.4854516138633092, + 0.42653930090637004, + 0.47550543894370395, + 0.431506101662914, + 0.44603309283653897, + 0.4392868752280871, + 0.4374114287396272, + 0.44838641087214154, + 0.44732216000556946, + 0.4383024573326111, + 0.4583464562892914, + 0.45035820206006366, + 0.4265511476005486, + 0.43074024841189384, + 0.43554925297697383, + 0.4470735415816307, + 0.47602783888578415, + 0.447073499361674, + 0.46678049365679425, + 0.43036032530168694, + 0.4651025980710983, + 0.427450506715104, + 0.4275593933804582, + 0.4831311032176018, + 0.4756004437804222, + 0.5104372575879097, + 0.45572711278994876, + 0.4930632909138997, + 0.5482515394687653, + 0.4363246864328782, + 0.5395219027996063, + 0.4594046051303546, + 0.4983728999892871, + 0.45798252274592716, + 0.43481330138941604, + 0.42745048420814175, + 0.42657890342040145, + 0.4604746177792549, + 0.42892235207060975, + 0.5482513283689817, + 0.4685831442475319, + 0.43222397565841675, + 0.44603322197993595, + 0.4685283750295639, + 0.4420975608130296, + 0.5924079070488611, + 0.4333743279178937, + 0.4265848830885564, + 0.4293927239874999, + 0.5055855363607407, + 0.42657890386666014, + 0.6658720299601555, + 0.43928683921694756, + 0.43679607411225635, + 0.42755691419976455, + 0.44643469403187436, + 0.5593102325995764, + 0.43058567369977635, + 0.43974484379092854, + 0.433081600194176, + 0.5063242092728615, + 0.4266252550781549, + 0.5506113544106483, + 0.4319465005149444, + 0.42659255781715427, + 0.43830259144306183, + 0.46510230253140133, + 0.4327113814651966, + 0.47602779418230057, + 0.426561248243767, + 0.43630694846312207, + 0.4365672462930282, + 0.4460332704087098, + 0.4454844544331233, + 0.6658714835842451 + ], + "val_loss": [ + 0.9293766021728516, + 0.8400806784629822, + 0.7606217265129089, + 0.7106595635414124, + 0.6789438128471375, + 0.6525706648826599, + 0.6297838091850281, + 0.620349645614624, + 0.6066277623176575, + 0.5968912839889526, + 0.587565541267395, + 0.5806708931922913, + 0.5777190327644348, + 0.5767351388931274, + 0.5733930468559265, + 0.5732976794242859, + 0.5710626840591431, + 0.5656630992889404, + 0.5610532164573669, + 0.5552923679351807, + 0.5523164868354797, + 0.5533589720726013, + 0.5522278547286987, + 0.5511414408683777, + 0.554003894329071, + 0.5541664958000183, + 0.5551878809928894, + 0.5569944977760315, + 0.5565626621246338, + 0.5577270984649658, + 0.5552704930305481, + 0.5574724674224854, + 0.5566246509552002, + 0.5631648898124695, + 0.5596378445625305, + 0.5597408413887024, + 0.5565659403800964, + 0.5553036332130432, + 0.5546241998672485, + 0.5592823028564453, + 0.571992814540863, + 0.5833666324615479, + 0.5802960991859436, + 0.5790232419967651, + 0.5839936137199402, + 0.5801540017127991, + 0.5778616666793823, + 0.573805570602417, + 0.5696023106575012, + 0.5734301209449768, + 0.573445200920105, + 0.5738602876663208, + 0.5713258981704712, + 0.5702441334724426, + 0.5683383941650391, + 0.5615531802177429, + 0.5595998167991638, + 0.5590642690658569, + 0.5628986358642578, + 0.563922643661499, + 0.5631725788116455, + 0.5597935318946838, + 0.5591951012611389, + 0.5597640872001648, + 0.5603768229484558, + 0.5620993971824646, + 0.562282383441925, + 0.5607417821884155, + 0.5592714548110962, + 0.558448851108551, + 0.5584564805030823, + 0.5611964464187622, + 0.5617803335189819, + 0.5624975562095642, + 0.562020480632782, + 0.5612857937812805, + 0.5612106919288635, + 0.5623995661735535, + 0.5619585514068604, + 0.5608959197998047, + 0.5602303147315979, + 0.5588372349739075, + 0.5587987899780273, + 0.5571296811103821, + 0.5569753050804138, + 0.557715654373169, + 0.5583426356315613, + 0.5578076839447021, + 0.5578439235687256, + 0.5568171143531799, + 0.5564279556274414, + 0.5558187365531921, + 0.5568453073501587, + 0.5572828054428101, + 0.5576978325843811, + 0.5577327013015747, + 0.5578310489654541, + 0.5582382082939148, + 0.5578768849372864, + 0.5575799345970154, + 0.5573912858963013, + 0.5585786700248718, + 0.5585137009620667, + 0.5584135055541992, + 0.5584436655044556, + 0.5569170713424683, + 0.5569329261779785, + 0.5575743317604065, + 0.5576494336128235, + 0.5577654242515564, + 0.5575302839279175, + 0.5572336316108704, + 0.5567386746406555, + 0.5562914609909058, + 0.5560296177864075, + 0.5559079051017761, + 0.5561579465866089, + 0.5563152432441711, + 0.5561518669128418, + 0.556119441986084, + 0.5561613440513611, + 0.5557867288589478, + 0.5562347173690796, + 0.5556216239929199, + 0.5551412105560303, + 0.5546855926513672, + 0.554517388343811, + 0.5544115304946899, + 0.5544315576553345, + 0.5544189214706421, + 0.5541485548019409, + 0.5542071461677551, + 0.5544220805168152, + 0.5545510649681091, + 0.5545435547828674, + 0.5545313358306885, + 0.5545893907546997, + 0.5545940399169922, + 0.5545541048049927, + 0.554665744304657, + 0.5547679662704468, + 0.5548238158226013, + 0.5548447370529175, + 0.5549745559692383, + 0.5552344918251038, + 0.5554050207138062, + 0.5555224418640137, + 0.5555222034454346, + 0.5556569695472717, + 0.5556782484054565, + 0.555759608745575, + 0.5558311939239502, + 0.5558602213859558, + 0.5558344721794128, + 0.5558265447616577, + 0.5558415055274963, + 0.5558870434761047, + 0.5559899210929871, + 0.5560117363929749, + 0.5560175776481628, + 0.5559782981872559, + 0.556072473526001, + 0.5560773611068726, + 0.556098222732544, + 0.5561559200286865, + 0.5561550259590149, + 0.5561347603797913, + 0.5561864376068115, + 0.5561371445655823, + 0.5561217665672302, + 0.5561251044273376, + 0.5561102032661438, + 0.5561025142669678, + 0.5561239719390869, + 0.5561983585357666, + 0.5562368631362915, + 0.5562867522239685, + 0.5562897324562073, + 0.5563392639160156, + 0.5563452243804932, + 0.5563333034515381, + 0.5563401579856873, + 0.5563548803329468, + 0.556351900100708, + 0.5563547611236572, + 0.5563892126083374, + 0.5564130544662476, + 0.5563898682594299, + 0.5562421679496765, + 0.5561904311180115, + 0.5561597347259521, + 0.5561371445655823, + 0.5561487674713135, + 0.5561352968215942, + 0.5561032891273499, + 0.5561174750328064, + 0.5561310052871704, + 0.5561124682426453, + 0.5561066269874573, + 0.5561291575431824, + 0.5561376810073853, + 0.5561423301696777, + 0.556159257888794, + 0.5561524629592896, + 0.5561360716819763, + 0.5560951828956604, + 0.5560580492019653, + 0.5560354590415955, + 0.5560299754142761, + 0.5560557842254639, + 0.5560573935508728, + 0.5560439229011536, + 0.5560564398765564, + 0.5560560822486877, + 0.5560553669929504, + 0.556060791015625, + 0.556081235408783, + 0.5560905933380127, + 0.5560944080352783, + 0.5560911893844604, + 0.5560874938964844, + 0.5560876131057739, + 0.5560891628265381, + 0.5560788512229919, + 0.5560904741287231, + 0.5560708045959473, + 0.5560656189918518, + 0.5560627579689026, + 0.5560514330863953, + 0.556043803691864, + 0.5560411810874939, + 0.5560407638549805, + 0.5560265779495239, + 0.5560235977172852, + 0.5560325980186462, + 0.5560321807861328, + 0.5560275912284851, + 0.556023895740509, + 0.5560228824615479, + 0.5560161471366882, + 0.555996298789978, + 0.5559833645820618, + 0.5559795498847961, + 0.5559757351875305, + 0.5559754371643066, + 0.5559735894203186, + 0.555976152420044, + 0.5559781789779663, + 0.555978000164032, + 0.555971622467041, + 0.5559578537940979, + 0.5559518933296204, + 0.5559501051902771, + 0.5559549927711487, + 0.5559545755386353, + 0.5559536218643188, + 0.5559516549110413, + 0.5559505820274353, + 0.5559508204460144, + 0.5559437274932861, + 0.5559419989585876, + 0.5559412240982056, + 0.5559414029121399, + 0.5559471845626831, + 0.5559492111206055, + 0.5559529662132263, + 0.5559535622596741, + 0.5559531450271606, + 0.5559549331665039, + 0.5559568405151367, + 0.5559579730033875, + 0.5559583902359009, + 0.5559578537940979, + 0.5559585690498352, + 0.5559585094451904, + 0.5559563636779785, + 0.5559573173522949, + 0.5559567809104919, + 0.5559576749801636, + 0.5559583306312561, + 0.5559585094451904, + 0.5559578537940979, + 0.5559577941894531, + 0.5559582114219666, + 0.5559569597244263, + 0.5559566020965576, + 0.555957019329071, + 0.5559565424919128, + 0.5559588074684143, + 0.5559602379798889, + 0.5559619069099426, + 0.5559626221656799, + 0.5559620261192322, + 0.555958092212677, + 0.5559570789337158, + 0.5559564828872681, + 0.555955708026886, + 0.5559564828872681, + 0.5559567213058472, + 0.5559565424919128, + 0.5559572577476501, + 0.5559572577476501, + 0.5559573173522949, + 0.5559572577476501, + 0.5559564232826233, + 0.5559567809104919, + 0.5559569001197815, + 0.5559573769569397, + 0.555958092212677, + 0.5559588074684143, + 0.5559590458869934, + 0.5559592247009277, + 0.5559595823287964, + 0.555959939956665, + 0.5559612512588501, + 0.5559611916542053, + 0.5559610724449158, + 0.5559601783752441, + 0.5559600591659546, + 0.5559598803520203, + 0.5559604167938232, + 0.5559604167938232, + 0.555960476398468, + 0.5559607148170471, + 0.5559608936309814, + 0.5559610724449158, + 0.5559606552124023, + 0.5559597015380859, + 0.5559594035148621, + 0.5559595227241516, + 0.5559599995613098, + 0.5559607744216919, + 0.5559614896774292, + 0.5559613108634949, + 0.5559610724449158, + 0.5559608936309814, + 0.5559607744216919, + 0.5559607744216919, + 0.5559608936309814, + 0.555961012840271, + 0.5559610724449158, + 0.5559608340263367, + 0.5559607744216919, + 0.5559605360031128, + 0.5559607148170471, + 0.5559607744216919, + 0.5559608340263367, + 0.5559608936309814, + 0.5559609532356262, + 0.5559607744216919, + 0.5559607744216919, + 0.5559607744216919, + 0.5559607744216919, + 0.5559608340263367, + 0.5559608340263367, + 0.5559605956077576, + 0.5559605956077576, + 0.5559607148170471, + 0.5559607744216919, + 0.5559607744216919, + 0.5559607744216919, + 0.5559606552124023, + 0.5559605956077576, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.5559602975845337, + 0.5559601783752441, + 0.5559601783752441, + 0.5559602975845337, + 0.5559601783752441, + 0.5559601783752441, + 0.5559601783752441, + 0.5559601783752441, + 0.5559601783752441, + 0.5559601783752441, + 0.5559601783752441, + 0.5559602975845337, + 0.5559602975845337, + 0.5559603571891785, + 0.5559604167938232, + 0.5559603571891785, + 0.5559603571891785, + 0.5559602975845337, + 0.5559601783752441, + 0.5559601783752441, + 0.5559602379798889, + 0.5559602379798889, + 0.5559602379798889, + 0.5559602975845337, + 0.5559604167938232, + 0.5559604167938232, + 0.555960476398468, + 0.5559605360031128, + 0.5559605956077576, + 0.5559607744216919, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559607148170471, + 0.5559607148170471, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559606552124023, + 0.5559607148170471, + 0.5559607148170471, + 0.5559607148170471, + 0.5559607148170471, + 0.5559607744216919, + 0.5559607148170471, + 0.5559607148170471, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559607148170471, + 0.5559607148170471, + 0.5559607148170471, + 0.5559607148170471, + 0.5559607148170471, + 0.5559607148170471, + 0.5559606552124023, + 0.5559605956077576, + 0.5559605956077576, + 0.5559606552124023, + 0.5559605956077576, + 0.5559606552124023, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605360031128, + 0.5559605360031128, + 0.5559605360031128, + 0.5559605360031128, + 0.5559605360031128, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605360031128, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559606552124023, + 0.5559607744216919, + 0.5559607744216919, + 0.5559607744216919, + 0.5559607744216919, + 0.5559607744216919, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608936309814, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559607744216919, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559607744216919, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559607744216919, + 0.5559607744216919, + 0.5559607744216919, + 0.5559608340263367, + 0.5559608340263367, + 0.5559607744216919, + 0.5559607744216919, + 0.5559607744216919, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608936309814, + 0.5559608936309814, + 0.5559608936309814, + 0.5559608936309814, + 0.5559608936309814, + 0.5559608936309814, + 0.5559608340263367, + 0.5559608340263367, + 0.5559608936309814, + 0.5559608936309814, + 0.5559608936309814, + 0.5559607744216919, + 0.5559607744216919, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559607148170471, + 0.5559607148170471, + 0.5559607744216919, + 0.5559607744216919, + 0.5559607148170471, + 0.5559606552124023, + 0.5559607744216919, + 0.5559607148170471, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559606552124023, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605360031128, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605956077576, + 0.5559605360031128, + 0.5559605360031128, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.555960476398468, + 0.5559604167938232, + 0.5559603571891785, + 0.5559603571891785, + 0.5559603571891785, + 0.5559602975845337, + 0.5559602975845337, + 0.5559602975845337, + 0.5559602975845337, + 0.5559602975845337, + 0.5559602975845337, + 0.5559602975845337, + 0.5559602975845337, + 0.5559603571891785, + 0.5559602975845337, + 0.5559603571891785, + 0.5559603571891785, + 0.5559603571891785, + 0.5559603571891785, + 0.5559603571891785, + 0.5559604167938232, + 0.5559604167938232, + 0.5559603571891785, + 0.5559603571891785 + ] +} \ No newline at end of file diff --git a/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/history.npz b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/history.npz new file mode 100644 index 0000000..c2f5df0 Binary files /dev/null and b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/history.npz differ diff --git a/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/metrics.json b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/metrics.json new file mode 100644 index 0000000..8480c5e --- /dev/null +++ b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/metrics.json @@ -0,0 +1,12 @@ +{ + "train": { + "mse": 0.4639959931373596, + "mae": 0.5509255528450012, + "r2": 0.5360039472579956 + }, + "validation": { + "mse": 0.5559603571891785, + "mae": 0.6042522192001343, + "r2": 0.40301668643951416 + } +} \ No newline at end of file diff --git a/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/model.pt b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/model.pt new file mode 100644 index 0000000..1cd0bd7 Binary files /dev/null and b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/model.pt differ diff --git a/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/training_loss.png b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/training_loss.png new file mode 100644 index 0000000..a604b44 Binary files /dev/null and b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/output/training_loss.png differ diff --git a/MLtasks/tasks/linreg_lvl5_diabetes_adamw/task.py b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/task.py new file mode 100644 index 0000000..ce5d3bf --- /dev/null +++ b/MLtasks/tasks/linreg_lvl5_diabetes_adamw/task.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +""" +Linear Regression on Diabetes dataset using PyTorch + AdamW. + +Implements the pytorch_task_v1 protocol in a single self-contained file. +""" + +import json +import os +import sys +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import DataLoader, TensorDataset +from sklearn.datasets import load_diabetes +from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + +def get_task_metadata(): + return { + "task_name": "linreg_lvl5_diabetes_adamw", + "series": "Linear Regression", + "task_type": "regression", + "dataset": "sklearn.load_diabetes", + "input_dim": 10, + "output_dim": 1, + "optimizer": "AdamW", + } + + +def set_seed(seed=42): + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def get_device(): + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +def _standardize(train_x, val_x): + mean = train_x.mean(axis=0, keepdims=True) + std = train_x.std(axis=0, keepdims=True) + 1e-8 + return (train_x - mean) / std, (val_x - mean) / std + + +def _standardize_target(train_y, val_y): + mean = train_y.mean(axis=0, keepdims=True) + std = train_y.std(axis=0, keepdims=True) + 1e-8 + return (train_y - mean) / std, (val_y - mean) / std + + +def make_dataloaders(batch_size=32, train_ratio=0.8, seed=42): + set_seed(seed) + data = load_diabetes() + x = data.data.astype(np.float32) + y = data.target.astype(np.float32).reshape(-1, 1) + + indices = np.random.permutation(len(x)) + split = int(len(x) * train_ratio) + train_idx, val_idx = indices[:split], indices[split:] + + x_train, y_train = x[train_idx], y[train_idx] + x_val, y_val = x[val_idx], y[val_idx] + x_train, x_val = _standardize(x_train, x_val) + y_train, y_val = _standardize_target(y_train, y_val) + + train_ds = TensorDataset(torch.from_numpy(x_train), torch.from_numpy(y_train)) + val_ds = TensorDataset(torch.from_numpy(x_val), torch.from_numpy(y_val)) + + train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True) + val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False) + return train_loader, val_loader, x_train, y_train, x_val, y_val + + +class LinearModel(nn.Module): + def __init__(self, input_dim): + super().__init__() + self.linear = nn.Linear(input_dim, 1) + + def forward(self, x): + return self.linear(x) + + +def build_model(input_dim, device): + return LinearModel(input_dim).to(device) + + +def train(model, train_loader, val_loader, device, epochs=600, lr=0.005, weight_decay=1e-4): + criterion = nn.MSELoss() + optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay) + scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="min", factor=0.5, patience=20) + + history = {"train_loss": [], "val_loss": []} + + for epoch in range(epochs): + model.train() + running = 0.0 + for xb, yb in train_loader: + xb, yb = xb.to(device), yb.to(device) + optimizer.zero_grad() + pred = model(xb) + loss = criterion(pred, yb) + loss.backward() + optimizer.step() + running += loss.item() + + train_loss = running / len(train_loader) + val_metrics = evaluate(model, val_loader, device) + val_loss = val_metrics["mse"] + scheduler.step(val_loss) + + history["train_loss"].append(train_loss) + history["val_loss"].append(val_loss) + + if (epoch + 1) % 50 == 0: + print(f"Epoch {epoch+1:03d} | train_mse={train_loss:.3f} | val_mse={val_loss:.3f}") + + return history + + +def evaluate(model, data_loader, device): + model.eval() + preds, targets = [], [] + + with torch.no_grad(): + for xb, yb in data_loader: + xb = xb.to(device) + out = model(xb).cpu().numpy() + preds.append(out) + targets.append(yb.numpy()) + + y_pred = np.vstack(preds) + y_true = np.vstack(targets) + + mse = float(mean_squared_error(y_true, y_pred)) + mae = float(mean_absolute_error(y_true, y_pred)) + r2 = float(r2_score(y_true, y_pred)) + + return { + "mse": mse, + "mae": mae, + "r2": r2, + } + + +def predict(model, x, device): + model.eval() + with torch.no_grad(): + x_t = torch.as_tensor(x, dtype=torch.float32, device=device) + return model(x_t).cpu().numpy() + +def _get_series(history, keys): + for k in keys: + v = history.get(k) + if isinstance(v, (list, tuple, np.ndarray)) and len(v) > 0: + return np.asarray(v, dtype=np.float32), k + return None, None + + +def plot_training_curves(history, out_dir, prefix="training"): + os.makedirs(out_dir, exist_ok=True) + + # Loss curve + train_loss, train_loss_name = _get_series(history, ["train_loss", "loss_train"]) + val_loss, val_loss_name = _get_series(history, ["val_loss", "valid_loss", "loss_val"]) + + if train_loss is not None or val_loss is not None: + n = len(train_loss) if train_loss is not None else len(val_loss) + epochs = np.arange(1, n + 1) + plt.figure(figsize=(8, 5)) + if train_loss is not None: + plt.plot(epochs, train_loss, label=train_loss_name) + if val_loss is not None: + plt.plot(epochs, val_loss, label=val_loss_name) + plt.xlabel("Epoch") + plt.ylabel("Loss") + plt.title("Training/Validation Loss") + plt.grid(alpha=0.3) + plt.legend() + plt.tight_layout() + plt.savefig(os.path.join(out_dir, f"{prefix}_loss.png"), dpi=150) + plt.close() + + # Accuracy curve (if available) + train_acc, train_acc_name = _get_series(history, ["train_acc", "train_accuracy"]) + val_acc, val_acc_name = _get_series(history, ["val_acc", "val_accuracy", "valid_accuracy"]) + + if train_acc is not None or val_acc is not None: + n = len(train_acc) if train_acc is not None else len(val_acc) + epochs = np.arange(1, n + 1) + plt.figure(figsize=(8, 5)) + if train_acc is not None: + plt.plot(epochs, train_acc, label=train_acc_name) + if val_acc is not None: + plt.plot(epochs, val_acc, label=val_acc_name) + plt.xlabel("Epoch") + plt.ylabel("Accuracy") + plt.title("Training/Validation Accuracy") + plt.grid(alpha=0.3) + plt.legend() + plt.tight_layout() + plt.savefig(os.path.join(out_dir, f"{prefix}_accuracy.png"), dpi=150) + plt.close() + + +def _to_serializable_history(history): + serializable = {} + for k, v in history.items(): + if isinstance(v, np.ndarray): + serializable[k] = v.astype(float).tolist() + elif isinstance(v, (list, tuple)): + serializable[k] = [ + float(x) if isinstance(x, (np.floating, np.integer)) else x for x in v + ] + elif isinstance(v, (np.floating, np.integer)): + serializable[k] = float(v) + else: + serializable[k] = v + return serializable + + +def save_artifacts(model, history, train_metrics, val_metrics, out_dir): + os.makedirs(out_dir, exist_ok=True) + torch.save(model.state_dict(), os.path.join(out_dir, "model.pt")) + + with open(os.path.join(out_dir, "history.json"), "w", encoding="utf-8") as f: + json.dump(_to_serializable_history(history), f, indent=2) + + with open(os.path.join(out_dir, "metrics.json"), "w", encoding="utf-8") as f: + json.dump({"train": train_metrics, "validation": val_metrics}, f, indent=2) + + plot_training_curves(history, out_dir, prefix="training") + + +def main(): + set_seed(42) + device = get_device() + print(f"Using device: {device}") + + metadata = get_task_metadata() + train_loader, val_loader, _, _, _, _ = make_dataloaders(batch_size=32, train_ratio=0.8, seed=42) + model = build_model(input_dim=metadata["input_dim"], device=device) + + history = train(model, train_loader, val_loader, device, epochs=600, lr=0.005, weight_decay=1e-4) + + train_metrics = evaluate(model, train_loader, device) + val_metrics = evaluate(model, val_loader, device) + + print("Train metrics:", train_metrics) + print("Validation metrics:", val_metrics) + + quality_checks = [ + val_metrics["r2"] > 0.30, + val_metrics["mse"] < 0.70, + abs(train_metrics["r2"] - val_metrics["r2"]) < 0.20, + history["train_loss"][-1] < history["train_loss"][0], + ] + + out_dir = os.path.join(os.path.dirname(__file__), "output") + save_artifacts(model, history, train_metrics, val_metrics, out_dir) + + passed = all(quality_checks) + print("PASS" if passed else "FAIL") + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/history.json b/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/history.json new file mode 100644 index 0000000..110283c --- /dev/null +++ b/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/history.json @@ -0,0 +1,145 @@ +{ + "train_loss": [ + 3.5897890249888103, + 3.131904458999634, + 2.711401581764221, + 2.319922192891439, + 1.959775487581889, + 1.6391074419021607, + 1.390454371770223, + 1.2203098972638449, + 1.1431001702944437, + 1.1200846552848815, + 1.118646748860677, + 1.1189670801162719, + 1.1187000373999278, + 1.118459403514862, + 1.118268318970998, + 1.1187069376309713, + 1.1183401823043824, + 1.118359422683716, + 1.1187021573384603, + 1.1189219276110332, + 1.1192628622055054, + 1.1195661087830862, + 1.118770186106364, + 1.1181447039047876, + 1.1186843276023866, + 1.1190517952044805, + 1.1192253053188324, + 1.1186881909767787, + 1.1186952551205953, + 1.118510506550471, + 1.1186392645041148, + 1.118793535232544, + 1.1189868648846943, + 1.1194673577944438, + 1.1191764195760092, + 1.1181732773780824, + 1.1200468381245932, + 1.1184199313322702, + 1.118701414267222, + 1.1188559412956238, + 1.1183897296587626, + 1.1186635037263235, + 1.1183933357397715, + 1.1183469653129579, + 1.1187721371650696, + 1.1189401348431904, + 1.1203146735827128, + 1.1199185013771058, + 1.119171965122223, + 1.1187200645605724, + 1.1184797545274099, + 1.1185637871424357, + 1.1189297318458558, + 1.1184063315391541, + 1.119272518157959, + 1.119778670867284, + 1.118913867076238, + 1.1192801515261332, + 1.119943916797638, + 1.1196907043457032, + 1.1182441333929698, + 1.1194041033585866, + 1.1190656920274098, + 1.1197932382424673, + 1.1199735244115194, + 1.1194872379302978, + 1.1195213635762533, + 1.1188689152399698, + 1.1193968196709951 + ], + "val_loss": [ + 3.103549897670746, + 2.6998029947280884, + 2.314393162727356, + 1.959760844707489, + 1.6500760912895203, + 1.3964251279830933, + 1.200258806347847, + 1.0925510972738266, + 1.0569090694189072, + 1.0553615540266037, + 1.0568431913852692, + 1.0561397671699524, + 1.0571985095739365, + 1.0570406764745712, + 1.056505724787712, + 1.0554130524396896, + 1.0562988072633743, + 1.0564957559108734, + 1.0565817654132843, + 1.0558569878339767, + 1.0576764792203903, + 1.0556329935789108, + 1.056065410375595, + 1.0571072101593018, + 1.0562696903944016, + 1.0558846443891525, + 1.0558826178312302, + 1.058008000254631, + 1.0567311346530914, + 1.05598184466362, + 1.0561548322439194, + 1.0573669224977493, + 1.056314930319786, + 1.0547645390033722, + 1.0570828765630722, + 1.0567996799945831, + 1.0572088658809662, + 1.0559954941272736, + 1.055975764989853, + 1.0573516935110092, + 1.0568968504667282, + 1.0569550842046738, + 1.055527076125145, + 1.0559988170862198, + 1.0573871284723282, + 1.0576574504375458, + 1.0560543984174728, + 1.0588538199663162, + 1.0558133870363235, + 1.0569309443235397, + 1.0568413138389587, + 1.0559211671352386, + 1.0578734576702118, + 1.0555291771888733, + 1.0559721440076828, + 1.0568474382162094, + 1.057193636894226, + 1.0563195794820786, + 1.0551100224256516, + 1.0583595484495163, + 1.0561405718326569, + 1.056247964501381, + 1.0564449280500412, + 1.0560431778430939, + 1.056742176413536, + 1.057654693722725, + 1.0554387867450714, + 1.0563873201608658, + 1.056645706295967 + ], + "best_val_huber": 1.0547645390033722 +} \ No newline at end of file diff --git a/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/history.npz b/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/history.npz new file mode 100644 index 0000000..e858a21 Binary files /dev/null and b/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/history.npz differ diff --git a/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/metrics.json b/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/metrics.json new file mode 100644 index 0000000..8889963 --- /dev/null +++ b/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/metrics.json @@ -0,0 +1,12 @@ +{ + "train": { + "mse": 17.650461196899414, + "mae": 1.4194393157958984, + "r2": 0.4908198118209839 + }, + "validation": { + "mse": 16.794803619384766, + "mae": 1.3302769660949707, + "r2": 0.49092918634414673 + } +} \ No newline at end of file diff --git a/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/model.pt b/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/model.pt new file mode 100644 index 0000000..2a07fe0 Binary files /dev/null and b/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/model.pt differ diff --git a/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/training_loss.png b/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/training_loss.png new file mode 100644 index 0000000..57ea14f Binary files /dev/null and b/MLtasks/tasks/linreg_lvl6_huber_earlystop/output/training_loss.png differ diff --git a/MLtasks/tasks/linreg_lvl6_huber_earlystop/task.py b/MLtasks/tasks/linreg_lvl6_huber_earlystop/task.py new file mode 100644 index 0000000..9780377 --- /dev/null +++ b/MLtasks/tasks/linreg_lvl6_huber_earlystop/task.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +""" +Robust Linear Regression with outliers using PyTorch Huber loss + early stopping. + +Implements the pytorch_task_v1 protocol in one file. +""" + +import json +import os +import sys +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import DataLoader, TensorDataset +from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + +def get_task_metadata(): + return { + "task_name": "linreg_lvl6_huber_earlystop", + "series": "Linear Regression", + "task_type": "regression", + "dataset": "synthetic_outlier_regression", + "input_dim": 6, + "output_dim": 1, + "loss": "HuberLoss", + "features": ["early_stopping", "gradient_clipping"], + } + + +def set_seed(seed=42): + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def get_device(): + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +def _standardize(train_x, val_x): + mean = train_x.mean(axis=0, keepdims=True) + std = train_x.std(axis=0, keepdims=True) + 1e-8 + return (train_x - mean) / std, (val_x - mean) / std + + +def make_dataloaders(n_samples=1200, train_ratio=0.8, batch_size=64, seed=42): + set_seed(seed) + + x = np.random.randn(n_samples, 6).astype(np.float32) + true_w = np.array([1.8, -2.2, 0.7, 1.1, -0.4, 2.5], dtype=np.float32) + y = x @ true_w + 1.2 + np.random.randn(n_samples).astype(np.float32) * 0.5 + + outlier_mask = np.random.rand(n_samples) < 0.1 + y[outlier_mask] += np.random.randn(outlier_mask.sum()).astype(np.float32) * 15.0 + + y = y.reshape(-1, 1) + + idx = np.random.permutation(n_samples) + split = int(n_samples * train_ratio) + tr_idx, va_idx = idx[:split], idx[split:] + + x_train, y_train = x[tr_idx], y[tr_idx] + x_val, y_val = x[va_idx], y[va_idx] + + x_train, x_val = _standardize(x_train, x_val) + + train_ds = TensorDataset(torch.from_numpy(x_train), torch.from_numpy(y_train)) + val_ds = TensorDataset(torch.from_numpy(x_val), torch.from_numpy(y_val)) + + train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True) + val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False) + return train_loader, val_loader, x_train, y_train, x_val, y_val + + +class LinearModel(nn.Module): + def __init__(self, input_dim): + super().__init__() + self.linear = nn.Linear(input_dim, 1) + + def forward(self, x): + return self.linear(x) + + +def build_model(input_dim, device): + return LinearModel(input_dim).to(device) + + +def train(model, train_loader, val_loader, device, epochs=300, lr=0.02, patience=35): + criterion = nn.HuberLoss(delta=1.0) + optimizer = torch.optim.Adam(model.parameters(), lr=lr) + + best_val = float("inf") + best_state = None + bad_epochs = 0 + history = {"train_loss": [], "val_loss": []} + + for epoch in range(epochs): + model.train() + train_running = 0.0 + for xb, yb in train_loader: + xb, yb = xb.to(device), yb.to(device) + optimizer.zero_grad() + pred = model(xb) + loss = criterion(pred, yb) + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=2.0) + optimizer.step() + train_running += loss.item() + + train_loss = train_running / len(train_loader) + + model.eval() + val_running = 0.0 + with torch.no_grad(): + for xb, yb in val_loader: + xb, yb = xb.to(device), yb.to(device) + val_running += criterion(model(xb), yb).item() + val_loss = val_running / len(val_loader) + + history["train_loss"].append(train_loss) + history["val_loss"].append(val_loss) + + if val_loss < best_val: + best_val = val_loss + bad_epochs = 0 + best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()} + else: + bad_epochs += 1 + + if (epoch + 1) % 50 == 0: + print(f"Epoch {epoch+1:03d} | train_huber={train_loss:.3f} | val_huber={val_loss:.3f}") + + if bad_epochs >= patience: + print(f"Early stopping at epoch {epoch + 1}") + break + + if best_state is not None: + model.load_state_dict(best_state) + + history["best_val_huber"] = best_val + return history + + +def evaluate(model, data_loader, device): + model.eval() + preds, targets = [], [] + + with torch.no_grad(): + for xb, yb in data_loader: + xb = xb.to(device) + out = model(xb).cpu().numpy() + preds.append(out) + targets.append(yb.numpy()) + + y_pred = np.vstack(preds) + y_true = np.vstack(targets) + + mse = float(mean_squared_error(y_true, y_pred)) + mae = float(mean_absolute_error(y_true, y_pred)) + r2 = float(r2_score(y_true, y_pred)) + + return { + "mse": mse, + "mae": mae, + "r2": r2, + } + + +def predict(model, x, device): + model.eval() + with torch.no_grad(): + x_t = torch.as_tensor(x, dtype=torch.float32, device=device) + return model(x_t).cpu().numpy() + + +def _get_series(history, keys): + for k in keys: + v = history.get(k) + if isinstance(v, (list, tuple, np.ndarray)) and len(v) > 0: + return np.asarray(v, dtype=np.float32), k + return None, None + + +def plot_training_curves(history, out_dir, prefix="training"): + os.makedirs(out_dir, exist_ok=True) + + # Loss curve + train_loss, train_loss_name = _get_series(history, ["train_loss", "loss_train"]) + val_loss, val_loss_name = _get_series(history, ["val_loss", "valid_loss", "loss_val"]) + + if train_loss is not None or val_loss is not None: + n = len(train_loss) if train_loss is not None else len(val_loss) + epochs = np.arange(1, n + 1) + plt.figure(figsize=(8, 5)) + if train_loss is not None: + plt.plot(epochs, train_loss, label=train_loss_name) + if val_loss is not None: + plt.plot(epochs, val_loss, label=val_loss_name) + plt.xlabel("Epoch") + plt.ylabel("Loss") + plt.title("Training/Validation Loss") + plt.grid(alpha=0.3) + plt.legend() + plt.tight_layout() + plt.savefig(os.path.join(out_dir, f"{prefix}_loss.png"), dpi=150) + plt.close() + + # Accuracy curve (if available) + train_acc, train_acc_name = _get_series(history, ["train_acc", "train_accuracy"]) + val_acc, val_acc_name = _get_series(history, ["val_acc", "val_accuracy", "valid_accuracy"]) + + if train_acc is not None or val_acc is not None: + n = len(train_acc) if train_acc is not None else len(val_acc) + epochs = np.arange(1, n + 1) + plt.figure(figsize=(8, 5)) + if train_acc is not None: + plt.plot(epochs, train_acc, label=train_acc_name) + if val_acc is not None: + plt.plot(epochs, val_acc, label=val_acc_name) + plt.xlabel("Epoch") + plt.ylabel("Accuracy") + plt.title("Training/Validation Accuracy") + plt.grid(alpha=0.3) + plt.legend() + plt.tight_layout() + plt.savefig(os.path.join(out_dir, f"{prefix}_accuracy.png"), dpi=150) + plt.close() + + +def _to_serializable_history(history): + serializable = {} + for k, v in history.items(): + if isinstance(v, np.ndarray): + serializable[k] = v.astype(float).tolist() + elif isinstance(v, (list, tuple)): + serializable[k] = [ + float(x) if isinstance(x, (np.floating, np.integer)) else x for x in v + ] + elif isinstance(v, (np.floating, np.integer)): + serializable[k] = float(v) + else: + serializable[k] = v + return serializable + + +def save_artifacts(model, history, train_metrics, val_metrics, out_dir): + os.makedirs(out_dir, exist_ok=True) + torch.save(model.state_dict(), os.path.join(out_dir, "model.pt")) + + with open(os.path.join(out_dir, "history.json"), "w", encoding="utf-8") as f: + json.dump(_to_serializable_history(history), f, indent=2) + + with open(os.path.join(out_dir, "metrics.json"), "w", encoding="utf-8") as f: + json.dump({"train": train_metrics, "validation": val_metrics}, f, indent=2) + + plot_training_curves(history, out_dir, prefix="training") + + +def main(): + set_seed(42) + device = get_device() + print(f"Using device: {device}") + + metadata = get_task_metadata() + train_loader, val_loader, _, _, _, _ = make_dataloaders(batch_size=64, train_ratio=0.8, seed=42) + model = build_model(metadata["input_dim"], device) + + history = train(model, train_loader, val_loader, device, epochs=300, lr=0.02, patience=35) + + train_metrics = evaluate(model, train_loader, device) + val_metrics = evaluate(model, val_loader, device) + + print("Train metrics:", train_metrics) + print("Validation metrics:", val_metrics) + + checks = [ + val_metrics["r2"] > 0.45, + val_metrics["mse"] < 30.0, + abs(train_metrics["r2"] - val_metrics["r2"]) < 0.25, + history["train_loss"][-1] < history["train_loss"][0], + ] + + out_dir = os.path.join(os.path.dirname(__file__), "output") + save_artifacts(model, history, train_metrics, val_metrics, out_dir) + + passed = all(checks) + print("PASS" if passed else "FAIL") + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/history.json b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/history.json new file mode 100644 index 0000000..457e05e --- /dev/null +++ b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/history.json @@ -0,0 +1,446 @@ +{ + "train_loss": [ + 0.5073547234137853, + 0.22672603726387025, + 0.15516155511140822, + 0.12763650740186375, + 0.1155952682097753, + 0.10369119296471278, + 0.09483178580800693, + 0.09080838685234388, + 0.08606504698594411, + 0.0910578524072965, + 0.07783325109630823, + 0.07686039432883263, + 0.08319019638001919, + 0.07121753456691901, + 0.07125453737874826, + 0.07586194699009259, + 0.06607930219421784, + 0.07142463798324267, + 0.06828118016322454, + 0.06519998908042908, + 0.06235339976847172, + 0.06252189800143242, + 0.06147883584101995, + 0.05947241395091017, + 0.059723781421780586, + 0.06317601477106412, + 0.06105540469288826, + 0.05770603281756242, + 0.058030868073304494, + 0.05797859914600849, + 0.05605210692932208, + 0.055756806209683415, + 0.055944766476750375, + 0.05509379133582115, + 0.054679508320987225, + 0.05495563019067049, + 0.057665705929199854, + 0.05355084662636121, + 0.053559480048716065, + 0.0528527148378392, + 0.0525420514245828, + 0.052720465573171775, + 0.05536800175905228, + 0.055104576113323374, + 0.05902053353687128, + 0.05139524694532156, + 0.0535520372291406, + 0.051551987789571284, + 0.05091729735334714, + 0.05085006318986416, + 0.05399079496661822, + 0.049810146416227025, + 0.05013930431256692, + 0.0503958648070693, + 0.05001893186320861, + 0.05031182747334242, + 0.08825779718657335, + 0.04863240603978435, + 0.049170256468156974, + 0.04908208533500632, + 0.04879018229742845, + 0.053138617301980655, + 0.048609616979956624, + 0.051365281889835995, + 0.04878011687348286, + 0.048788262406984965, + 0.049738709628582, + 0.05983035707225402, + 0.04733126908540726, + 0.047287091116110486, + 0.04709421861916781, + 0.05173868748048941, + 0.04656514944508672, + 0.04907220074286064, + 0.04690245535845558, + 0.04881545218328635, + 0.04651853907853365, + 0.046146411231408516, + 0.051139604610701404, + 0.046286731213331225, + 0.04678092437485854, + 0.04550116012493769, + 0.04540415660788615, + 0.04745627827942371, + 0.04607548067967097, + 0.04540101618816455, + 0.0481151952718695, + 0.0491765228100121, + 0.045234628145893414, + 0.04963627364486456, + 0.049100724545617896, + 0.04453195808455348, + 0.047107872056464356, + 0.04595598864058654, + 0.045043893220523996, + 0.04433615887537599, + 0.04400392475848396, + 0.046380297963817915, + 0.047350772904853025, + 0.04853178138534228, + 0.04650462735444307, + 0.04417539524535338, + 0.0472575414304932, + 0.04568983608235915, + 0.04777248998483022, + 0.04700271546219786, + 0.046660127987464266, + 0.04327828368792931, + 0.04316717932621638, + 0.043356384088595705, + 0.04323099550480644, + 0.043060819432139394, + 0.04394925559560458, + 0.04645733858148257, + 0.04276991927375396, + 0.055089855628709, + 0.042276743054389956, + 0.04305669184153279, + 0.04238467952236533, + 0.043567873847981296, + 0.042313296472032864, + 0.04193481355905533, + 0.04335773512721062, + 0.04443619474768638, + 0.045089468670388065, + 0.04196361415088177, + 0.04205141632507245, + 0.041368159279227254, + 0.04492904115468264, + 0.041134511058529215, + 0.041480775984625025, + 0.043092035812636216, + 0.04197159682710965, + 0.04126122413824002, + 0.04572493036588033, + 0.04146993656953176, + 0.04107534841944774, + 0.04115012337764104, + 0.043724243094523746, + 0.041005444899201396, + 0.045752899100383125, + 0.04153161843617757, + 0.04115377987424532, + 0.04139802816013495, + 0.04249849338084459, + 0.04078914256145557, + 0.04032570409278075, + 0.07013922370970249, + 0.042558192772169906, + 0.04109765018026034, + 0.042184264461199446, + 0.04008589759469032, + 0.039613004525502524, + 0.040184245848407345, + 0.04200637129445871, + 0.039976172925283514, + 0.040114417051275574, + 0.03980108027656873, + 0.04300130599488815, + 0.039322691472868126, + 0.039385651548703514, + 0.04153607953339815, + 0.042611552961170675, + 0.04209835131963094, + 0.04424624939759572, + 0.03930395692586899, + 0.039411594377209744, + 0.038997974681357545, + 0.04268838663895925, + 0.03983918596059084, + 0.039030187545965114, + 0.039062858062485856, + 0.03876046175137162, + 0.038923592989643416, + 0.03926635744671027, + 0.03855900305012862, + 0.042243967422594626, + 0.03845088997234901, + 0.040026499330997466, + 0.038217696640640494, + 0.0383204133870701, + 0.03829634711146355, + 0.03862310343732436, + 0.05626788968220353, + 0.0388500331590573, + 0.06662165733675161, + 0.03844869639724493, + 0.04044370868553718, + 0.05267198116828998, + 0.03776582212497791, + 0.03821957496305307, + 0.060816226278742155, + 0.04194761564334234, + 0.03880071751773358, + 0.04025698291758696, + 0.03755000761399666, + 0.03729380583390594, + 0.039695079997181894, + 0.03843379343549411, + 0.037292201754947504, + 0.03741502957418561, + 0.03898871627946695, + 0.03704038488989075, + 0.03695434605081876, + 0.03714511205131809, + 0.03720792202899854, + 0.03689210200682282, + 0.03705873098224401, + 0.04163930757592122, + 0.03716012972096602, + 0.036890121859808764, + 0.03697395094980796, + 0.0371783080821236, + 0.0373286260291934, + 0.039532751372704907, + 0.036606912563244505, + 0.03695353946338097, + 0.036661192495375874, + 0.03751643281430006, + 0.036648237891495225 + ], + "val_loss": [ + 0.2860754355788231, + 0.18635230883955956, + 0.15614060312509537, + 0.14293508231639862, + 0.13436429016292095, + 0.1280137449502945, + 0.12370234169065952, + 0.12003643997013569, + 0.11613162606954575, + 0.11394867114722729, + 0.11112092435359955, + 0.10851684398949146, + 0.10691925324499607, + 0.10547382570803165, + 0.10366330668330193, + 0.1027941107749939, + 0.10152437351644039, + 0.10064230486750603, + 0.09918933734297752, + 0.09791560471057892, + 0.09694525226950645, + 0.0959358848631382, + 0.09583133831620216, + 0.09591496270149946, + 0.09514427557587624, + 0.09425134304910898, + 0.09403619542717934, + 0.09376310091465712, + 0.09359710291028023, + 0.09291134215891361, + 0.09229649230837822, + 0.09169899392873049, + 0.09148352593183517, + 0.09102088585495949, + 0.09128598682582378, + 0.09097931068390608, + 0.09021184965968132, + 0.09100121539086103, + 0.09101455472409725, + 0.09034751821309328, + 0.08944759890437126, + 0.0890338085591793, + 0.08864570502191782, + 0.0892579210922122, + 0.08775199297815561, + 0.08809818420559168, + 0.08815301582217216, + 0.08721166104078293, + 0.08726168051362038, + 0.08672521449625492, + 0.08715423755347729, + 0.08730197697877884, + 0.08709765784442425, + 0.08752891048789024, + 0.08746403269469738, + 0.08760622888803482, + 0.08713068719953299, + 0.08954589627683163, + 0.09033292159438133, + 0.09031018149107695, + 0.0894417492672801, + 0.08872675150632858, + 0.0889656264334917, + 0.08923231530934572, + 0.08902142103761435, + 0.08916457742452621, + 0.08923298213630915, + 0.08884127251803875, + 0.08771484158933163, + 0.08788817841559649, + 0.08807993121445179, + 0.08812897931784391, + 0.0885092867538333, + 0.08862946555018425, + 0.08806703891605139, + 0.08891137782484293, + 0.08887508418411016, + 0.08853406552225351, + 0.08905604016035795, + 0.08886023983359337, + 0.08851058222353458, + 0.0884133018553257, + 0.08932049479335546, + 0.08932754583656788, + 0.08977743238210678, + 0.0901889344677329, + 0.08967507909983397, + 0.08922043442726135, + 0.0890556201338768, + 0.08895074296742678, + 0.08930143062025309, + 0.0921499328687787, + 0.09311666339635849, + 0.09256904385983944, + 0.09209482092410326, + 0.09090907126665115, + 0.09215560927987099, + 0.09212474897503853, + 0.09064077399671078, + 0.09010792523622513, + 0.0921840239316225, + 0.09308810345828533, + 0.09397644642740488, + 0.09202526323497295, + 0.09090881980955601, + 0.09249714575707912, + 0.09564935974776745, + 0.09195096511393785, + 0.09186985716223717, + 0.09333938267081976, + 0.09196844696998596, + 0.0927006658166647, + 0.093511663377285, + 0.09536054264754057, + 0.08974315039813519, + 0.08995975460857153, + 0.09616039413958788, + 0.09742549993097782, + 0.09491786360740662, + 0.09578236285597086, + 0.09385533723980188, + 0.09504812117666006, + 0.09686016943305731, + 0.0966332321986556, + 0.098082160577178, + 0.0961492657661438, + 0.0968292010948062, + 0.0986358867958188, + 0.09952172450721264, + 0.09892898332327604, + 0.09795824438333511, + 0.099725641310215, + 0.09890205599367619, + 0.10031496174633503, + 0.09899795614182949, + 0.10040813125669956, + 0.10220851935446262, + 0.10299027245491743, + 0.10634172707796097, + 0.10358721017837524, + 0.10467332415282726, + 0.09960769768804312, + 0.10013823304325342, + 0.10495061706751585, + 0.1029449813067913, + 0.10938005708158016, + 0.10476208385080099, + 0.10546547453850508, + 0.11857481859624386, + 0.11869812570512295, + 0.11384661123156548, + 0.10983691923320293, + 0.11139682307839394, + 0.10825286526232958, + 0.11258183140307665, + 0.11230179015547037, + 0.11012817174196243, + 0.10760761052370071, + 0.10975281335413456, + 0.11086225230246782, + 0.11295258346945047, + 0.11334462091326714, + 0.11545501835644245, + 0.1152386711910367, + 0.11155535839498043, + 0.11044031009078026, + 0.10994032025337219, + 0.10901130829006433, + 0.1104082241654396, + 0.10903155989944935, + 0.11031796783208847, + 0.11433012969791889, + 0.11473425850272179, + 0.11334675829857588, + 0.11403365340083838, + 0.11471323017030954, + 0.11343594919890165, + 0.1153411716222763, + 0.11495445296168327, + 0.11498389393091202, + 0.11541918478906155, + 0.1152451392263174, + 0.11716452706605196, + 0.11709957290440798, + 0.1198013573884964, + 0.11890153307467699, + 0.13234778679907322, + 0.1274654045701027, + 0.13075017370283604, + 0.12802676297724247, + 0.12521168868988752, + 0.1272519063204527, + 0.13914460875093937, + 0.13652893342077732, + 0.12846440449357033, + 0.12741978187114, + 0.12591337971389294, + 0.12639134377241135, + 0.11944071482867002, + 0.11772667244076729, + 0.11976429540663958, + 0.12154733203351498, + 0.12566371448338032, + 0.12558680586516857, + 0.12467233370989561, + 0.12415540684014559, + 0.12550705298781395, + 0.1261441633105278, + 0.12758961785584688, + 0.12541485484689474, + 0.12730634585022926, + 0.12797126546502113, + 0.12621173076331615, + 0.1287756823003292, + 0.12885782588273287, + 0.1255217818543315, + 0.12625402584671974, + 0.12643590942025185, + 0.13108772691339254, + 0.13061813451349735 + ] +} \ No newline at end of file diff --git a/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/history.npz b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/history.npz new file mode 100644 index 0000000..9a77bcc Binary files /dev/null and b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/history.npz differ diff --git a/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/metrics.json b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/metrics.json new file mode 100644 index 0000000..997e247 --- /dev/null +++ b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/metrics.json @@ -0,0 +1,22 @@ +{ + "train": { + "mse": 0.007893201895058155, + "r2": 0.965849757194519, + "bce_loss": 0.03550373991020024, + "accuracy": 0.9912087912087912, + "f1": 0.993127147766323, + "precision": 0.9897260273972602, + "recall": 0.996551724137931, + "auc": 0.9992058516196447 + }, + "validation": { + "mse": 0.04029029607772827, + "r2": 0.8337209224700928, + "bce_loss": 0.13061813451349735, + "accuracy": 0.9385964912280702, + "f1": 0.9481481481481482, + "precision": 0.9411764705882353, + "recall": 0.9552238805970149, + "auc": 0.9904731660844712 + } +} \ No newline at end of file diff --git a/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/model.pt b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/model.pt new file mode 100644 index 0000000..8739cd2 Binary files /dev/null and b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/model.pt differ diff --git a/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/training_loss.png b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/training_loss.png new file mode 100644 index 0000000..93f425e Binary files /dev/null and b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/output/training_loss.png differ diff --git a/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/task.py b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/task.py new file mode 100644 index 0000000..5b8a266 --- /dev/null +++ b/MLtasks/tasks/logreg_lvl5_breast_cancer_l1/task.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +""" +Binary Logistic Regression on Breast Cancer dataset with L1 regularization. + +Implements the pytorch_task_v1 protocol in one file. +""" + +import json +import os +import sys +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import DataLoader, TensorDataset +from sklearn.datasets import load_breast_cancer +from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, roc_auc_score, mean_squared_error, r2_score +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + +def get_task_metadata(): + return { + "task_name": "logreg_lvl5_breast_cancer_l1", + "series": "Logistic Regression", + "task_type": "binary_classification", + "dataset": "sklearn.load_breast_cancer", + "input_dim": 30, + "output_dim": 1, + "features": ["L1_regularization", "BCEWithLogitsLoss"], + } + + +def set_seed(seed=42): + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def get_device(): + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +def _standardize(train_x, val_x): + mean = train_x.mean(axis=0, keepdims=True) + std = train_x.std(axis=0, keepdims=True) + 1e-8 + return (train_x - mean) / std, (val_x - mean) / std + + +def make_dataloaders(batch_size=32, train_ratio=0.8, seed=42): + set_seed(seed) + data = load_breast_cancer() + x = data.data.astype(np.float32) + y = data.target.astype(np.float32).reshape(-1, 1) + + idx = np.random.permutation(len(x)) + split = int(len(x) * train_ratio) + tr_idx, va_idx = idx[:split], idx[split:] + + x_train, y_train = x[tr_idx], y[tr_idx] + x_val, y_val = x[va_idx], y[va_idx] + + x_train, x_val = _standardize(x_train, x_val) + + train_ds = TensorDataset(torch.from_numpy(x_train), torch.from_numpy(y_train)) + val_ds = TensorDataset(torch.from_numpy(x_val), torch.from_numpy(y_val)) + + train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True) + val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False) + return train_loader, val_loader, x_train, y_train, x_val, y_val + + +class LogisticModel(nn.Module): + def __init__(self, input_dim): + super().__init__() + self.linear = nn.Linear(input_dim, 1) + + def forward(self, x): + return self.linear(x) + + +def build_model(input_dim, device): + return LogisticModel(input_dim).to(device) + + +def train(model, train_loader, val_loader, device, epochs=220, lr=0.01, l1_lambda=2e-4): + criterion = nn.BCEWithLogitsLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=lr) + history = {"train_loss": [], "val_loss": []} + + for epoch in range(epochs): + model.train() + running = 0.0 + + for xb, yb in train_loader: + xb, yb = xb.to(device), yb.to(device) + optimizer.zero_grad() + logits = model(xb) + bce = criterion(logits, yb) + l1_penalty = torch.abs(model.linear.weight).sum() + loss = bce + l1_lambda * l1_penalty + loss.backward() + optimizer.step() + running += loss.item() + + train_loss = running / len(train_loader) + val_metrics = evaluate(model, val_loader, device) + + history["train_loss"].append(train_loss) + history["val_loss"].append(val_metrics["bce_loss"]) + + if (epoch + 1) % 50 == 0: + print( + f"Epoch {epoch+1:03d} | train_loss={train_loss:.4f} " + f"| val_bce={val_metrics['bce_loss']:.4f} | val_acc={val_metrics['accuracy']:.4f}" + ) + + return history + + +def evaluate(model, data_loader, device): + model.eval() + criterion = nn.BCEWithLogitsLoss() + + logits_all, y_all = [], [] + total_loss = 0.0 + + with torch.no_grad(): + for xb, yb in data_loader: + xb, yb = xb.to(device), yb.to(device) + logits = model(xb) + total_loss += criterion(logits, yb).item() + logits_all.append(logits.cpu().numpy()) + y_all.append(yb.cpu().numpy()) + + logits_np = np.vstack(logits_all) + y_true = np.vstack(y_all) + probs = torch.sigmoid(torch.from_numpy(logits_np)).numpy() + y_pred = (probs >= 0.5).astype(np.float32) + + mse = float(mean_squared_error(y_true, probs)) + r2 = float(r2_score(y_true, probs)) + acc = float(accuracy_score(y_true, y_pred)) + f1 = float(f1_score(y_true, y_pred)) + precision = float(precision_score(y_true, y_pred, zero_division=0)) + recall = float(recall_score(y_true, y_pred, zero_division=0)) + auc = float(roc_auc_score(y_true, probs)) + + return { + "mse": mse, + "r2": r2, + "bce_loss": float(total_loss / len(data_loader)), + "accuracy": acc, + "f1": f1, + "precision": precision, + "recall": recall, + "auc": auc, + } + + +def predict(model, x, device): + model.eval() + with torch.no_grad(): + x_t = torch.as_tensor(x, dtype=torch.float32, device=device) + logits = model(x_t) + probs = torch.sigmoid(logits) + return probs.cpu().numpy() + + +def _get_series(history, keys): + for k in keys: + v = history.get(k) + if isinstance(v, (list, tuple, np.ndarray)) and len(v) > 0: + return np.asarray(v, dtype=np.float32), k + return None, None + + +def plot_training_curves(history, out_dir, prefix="training"): + os.makedirs(out_dir, exist_ok=True) + + # Loss curve + train_loss, train_loss_name = _get_series(history, ["train_loss", "loss_train"]) + val_loss, val_loss_name = _get_series(history, ["val_loss", "valid_loss", "loss_val"]) + + if train_loss is not None or val_loss is not None: + n = len(train_loss) if train_loss is not None else len(val_loss) + epochs = np.arange(1, n + 1) + plt.figure(figsize=(8, 5)) + if train_loss is not None: + plt.plot(epochs, train_loss, label=train_loss_name) + if val_loss is not None: + plt.plot(epochs, val_loss, label=val_loss_name) + plt.xlabel("Epoch") + plt.ylabel("Loss") + plt.title("Training/Validation Loss") + plt.grid(alpha=0.3) + plt.legend() + plt.tight_layout() + plt.savefig(os.path.join(out_dir, f"{prefix}_loss.png"), dpi=150) + plt.close() + + # Accuracy curve (if available) + train_acc, train_acc_name = _get_series(history, ["train_acc", "train_accuracy"]) + val_acc, val_acc_name = _get_series(history, ["val_acc", "val_accuracy", "valid_accuracy"]) + + if train_acc is not None or val_acc is not None: + n = len(train_acc) if train_acc is not None else len(val_acc) + epochs = np.arange(1, n + 1) + plt.figure(figsize=(8, 5)) + if train_acc is not None: + plt.plot(epochs, train_acc, label=train_acc_name) + if val_acc is not None: + plt.plot(epochs, val_acc, label=val_acc_name) + plt.xlabel("Epoch") + plt.ylabel("Accuracy") + plt.title("Training/Validation Accuracy") + plt.grid(alpha=0.3) + plt.legend() + plt.tight_layout() + plt.savefig(os.path.join(out_dir, f"{prefix}_accuracy.png"), dpi=150) + plt.close() + + +def _to_serializable_history(history): + serializable = {} + for k, v in history.items(): + if isinstance(v, np.ndarray): + serializable[k] = v.astype(float).tolist() + elif isinstance(v, (list, tuple)): + serializable[k] = [ + float(x) if isinstance(x, (np.floating, np.integer)) else x for x in v + ] + elif isinstance(v, (np.floating, np.integer)): + serializable[k] = float(v) + else: + serializable[k] = v + return serializable + + +def save_artifacts(model, history, train_metrics, val_metrics, out_dir): + os.makedirs(out_dir, exist_ok=True) + torch.save(model.state_dict(), os.path.join(out_dir, "model.pt")) + + with open(os.path.join(out_dir, "history.json"), "w", encoding="utf-8") as f: + json.dump(_to_serializable_history(history), f, indent=2) + + with open(os.path.join(out_dir, "metrics.json"), "w", encoding="utf-8") as f: + json.dump({"train": train_metrics, "validation": val_metrics}, f, indent=2) + + plot_training_curves(history, out_dir, prefix="training") + + +def main(): + set_seed(42) + device = get_device() + print(f"Using device: {device}") + + metadata = get_task_metadata() + train_loader, val_loader, _, _, _, _ = make_dataloaders(batch_size=32, train_ratio=0.8, seed=42) + model = build_model(metadata["input_dim"], device) + + history = train(model, train_loader, val_loader, device, epochs=220, lr=0.01, l1_lambda=2e-4) + + train_metrics = evaluate(model, train_loader, device) + val_metrics = evaluate(model, val_loader, device) + + print("Train metrics:", train_metrics) + print("Validation metrics:", val_metrics) + + checks = [ + val_metrics["accuracy"] > 0.93, + val_metrics["f1"] > 0.93, + val_metrics["auc"] > 0.97, + abs(train_metrics["accuracy"] - val_metrics["accuracy"]) < 0.08, + ] + + out_dir = os.path.join(os.path.dirname(__file__), "output") + save_artifacts(model, history, train_metrics, val_metrics, out_dir) + + passed = all(checks) + print("PASS" if passed else "FAIL") + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/history.json b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/history.json new file mode 100644 index 0000000..c5eeb5b --- /dev/null +++ b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/history.json @@ -0,0 +1,366 @@ +{ + "train_loss": [ + 0.8352899700403214, + 0.5784546919167042, + 0.5573406778275967, + 0.5476696006953716, + 0.5282124951481819, + 0.5197043232619762, + 0.5059540942311287, + 0.5117000304162502, + 0.5083412081003189, + 0.5034436136484146, + 0.49999890848994255, + 0.4932052716612816, + 0.49262023344635963, + 0.4933450147509575, + 0.4904259853065014, + 0.4997929520905018, + 0.489526841789484, + 0.49215466156601906, + 0.4891430102288723, + 0.4890429228544235, + 0.5006928406655788, + 0.49490122124552727, + 0.4906842075288296, + 0.48736078292131424, + 0.4855901896953583, + 0.48564156144857407, + 0.49699728190898895, + 0.48711542412638664, + 0.49135059118270874, + 0.4900742471218109, + 0.4902191609144211, + 0.4886529669165611, + 0.49198681488633156, + 0.4846149757504463, + 0.49367600306868553, + 0.48488829657435417, + 0.4908243902027607, + 0.4893101081252098, + 0.48893267288804054, + 0.4903526194393635, + 0.4862045980989933, + 0.48605259880423546, + 0.48847462236881256, + 0.490227073431015, + 0.49207934737205505, + 0.4892636388540268, + 0.4860970340669155, + 0.4917004480957985, + 0.4829515367746353, + 0.48681338503956795, + 0.48183421045541763, + 0.4882308840751648, + 0.4860247112810612, + 0.49031949788331985, + 0.4869874119758606, + 0.4839169606566429, + 0.48507362231612206, + 0.4852241463959217, + 0.4907388910651207, + 0.4861534535884857, + 0.48280681297183037, + 0.4884617552161217, + 0.4861406348645687, + 0.48521551862359047, + 0.48735514655709267, + 0.4859876073896885, + 0.49227577075362206, + 0.48493364453315735, + 0.48611877858638763, + 0.4881702773272991, + 0.48579680174589157, + 0.4932532571256161, + 0.4867297299206257, + 0.4875848852097988, + 0.48482435569167137, + 0.48632482811808586, + 0.49333327636122704, + 0.48959388583898544, + 0.4861641079187393, + 0.4835391901433468, + 0.4836774356663227, + 0.4897551089525223, + 0.4829469658434391, + 0.48429662734270096, + 0.4873156063258648, + 0.48422880098223686, + 0.48835085704922676, + 0.4835480786859989, + 0.4888598322868347, + 0.4888424873352051, + 0.48646894097328186, + 0.48515579476952553, + 0.4942018687725067, + 0.4829057976603508, + 0.4824255593121052, + 0.4864325635135174, + 0.48794102668762207, + 0.48395296931266785, + 0.4964497424662113, + 0.4856218509376049, + 0.48411130905151367, + 0.49115975573658943, + 0.48939407616853714, + 0.4853826314210892, + 0.48449568450450897, + 0.4908837154507637, + 0.48721547424793243, + 0.48516684025526047, + 0.49309469386935234, + 0.4831163175404072, + 0.4914641007781029, + 0.48885805532336235, + 0.48119737952947617, + 0.48499925434589386, + 0.4831254184246063, + 0.48779072239995, + 0.4836696460843086, + 0.4911133460700512, + 0.48449182137846947, + 0.4937284104526043, + 0.49110272154212, + 0.4874686114490032, + 0.48929940164089203, + 0.4827985130250454, + 0.4826558791100979, + 0.4881819859147072, + 0.48825545608997345, + 0.48817649856209755, + 0.4837500564754009, + 0.4866022430360317, + 0.4821549914777279, + 0.48959139734506607, + 0.49022040888667107, + 0.48331068456172943, + 0.48714376986026764, + 0.4820771627128124, + 0.49116911739110947, + 0.4858187586069107, + 0.48145389929413795, + 0.4848716966807842, + 0.4870399981737137, + 0.48495638370513916, + 0.48356421291828156, + 0.49223617836833, + 0.48088568821549416, + 0.48237528279423714, + 0.4868098422884941, + 0.48357944563031197, + 0.48308734595775604, + 0.4824477732181549, + 0.4895947612822056, + 0.48935914784669876, + 0.4910712391138077, + 0.48207150399684906, + 0.4840298220515251, + 0.4811982437968254, + 0.4841928854584694, + 0.48159557580947876, + 0.4853235110640526, + 0.4978737495839596, + 0.4841132164001465, + 0.48482849448919296, + 0.48915036767721176, + 0.4824967123568058, + 0.48345566540956497, + 0.48276905342936516, + 0.48517104983329773, + 0.4897347427904606, + 0.481914259493351, + 0.4892885945737362, + 0.4823150932788849, + 0.48213908448815346, + 0.48570096865296364, + 0.4881662502884865, + 0.48484668135643005, + 0.4886109158396721, + 0.4856961593031883, + 0.4889055974781513, + 0.4877813532948494, + 0.4844284951686859 + ], + "val_loss": [ + 0.6249738335609436, + 0.498450368642807, + 0.4466194659471512, + 0.41704078018665314, + 0.39467403292655945, + 0.3771253973245621, + 0.36848805844783783, + 0.3546007126569748, + 0.33846840262413025, + 0.332307904958725, + 0.3287721872329712, + 0.321393221616745, + 0.3113609254360199, + 0.30403560400009155, + 0.3011365234851837, + 0.30462752282619476, + 0.29020069539546967, + 0.29132868349552155, + 0.29794108867645264, + 0.2904362976551056, + 0.29328177869319916, + 0.2805250734090805, + 0.27768905460834503, + 0.2837628275156021, + 0.28999462723731995, + 0.2830507755279541, + 0.2825104892253876, + 0.27706629037857056, + 0.2766125649213791, + 0.28230176866054535, + 0.29265259206295013, + 0.28157611191272736, + 0.27106302976608276, + 0.2795771658420563, + 0.278623104095459, + 0.2701523154973984, + 0.2713504731655121, + 0.27070844173431396, + 0.27499687671661377, + 0.2697601765394211, + 0.2787284255027771, + 0.27880506217479706, + 0.2747831791639328, + 0.2626102715730667, + 0.27394992113113403, + 0.2697294354438782, + 0.2684774547815323, + 0.2753974497318268, + 0.26663637161254883, + 0.26402704417705536, + 0.27241556346416473, + 0.2756720632314682, + 0.26875966787338257, + 0.27550989389419556, + 0.26740434765815735, + 0.2689508944749832, + 0.27197907865047455, + 0.27691952884197235, + 0.269258588552475, + 0.26221229135990143, + 0.2678312808275223, + 0.27349628508090973, + 0.27611328661441803, + 0.26763835549354553, + 0.2714805603027344, + 0.2700466960668564, + 0.2663128077983856, + 0.2749969959259033, + 0.2693764865398407, + 0.2717929482460022, + 0.278842568397522, + 0.2710244804620743, + 0.25942157208919525, + 0.2672186940908432, + 0.2761337459087372, + 0.2746458947658539, + 0.2656659036874771, + 0.27115392684936523, + 0.2700684666633606, + 0.27200302481651306, + 0.2671108841896057, + 0.26809047162532806, + 0.26846517622470856, + 0.27219249308109283, + 0.2715156078338623, + 0.27442941069602966, + 0.269666388630867, + 0.2753506004810333, + 0.265880823135376, + 0.26970648765563965, + 0.27248798310756683, + 0.271660178899765, + 0.2670249342918396, + 0.2694484144449234, + 0.2670091986656189, + 0.2721700668334961, + 0.26897694170475006, + 0.2728455662727356, + 0.272429957985878, + 0.2710406929254532, + 0.27297206223011017, + 0.26881732046604156, + 0.2657306268811226, + 0.26301246881484985, + 0.2650594487786293, + 0.2707550376653671, + 0.2664535790681839, + 0.2673831433057785, + 0.2724008411169052, + 0.275252103805542, + 0.27052731812000275, + 0.26965150237083435, + 0.27087707817554474, + 0.26906172931194305, + 0.2698809951543808, + 0.2697933465242386, + 0.26954124867916107, + 0.2734246402978897, + 0.27024659514427185, + 0.2664816826581955, + 0.26635926961898804, + 0.2659122124314308, + 0.2709209620952606, + 0.27031391859054565, + 0.2688979208469391, + 0.2690268009901047, + 0.26851803064346313, + 0.26847919821739197, + 0.26808692514896393, + 0.2704381197690964, + 0.2710452228784561, + 0.27219295501708984, + 0.27062903344631195, + 0.26767052710056305, + 0.26644618809223175, + 0.2660006731748581, + 0.26646852493286133, + 0.26887187361717224, + 0.268538698554039, + 0.26844705641269684, + 0.2686927765607834, + 0.2693353146314621, + 0.2700244039297104, + 0.27104751765727997, + 0.27104319632053375, + 0.2701449990272522, + 0.2686065137386322, + 0.26783977448940277, + 0.2680230438709259, + 0.2680181711912155, + 0.2682025730609894, + 0.26865915954113007, + 0.2689538449048996, + 0.26969514787197113, + 0.2692503184080124, + 0.26914332807064056, + 0.26932740211486816, + 0.2689661979675293, + 0.2687140852212906, + 0.26916515827178955, + 0.26908278465270996, + 0.2684461772441864, + 0.26852600276470184, + 0.2683270424604416, + 0.2683120667934418, + 0.2683003842830658, + 0.2681221663951874, + 0.2682217210531235, + 0.2682558447122574, + 0.2682235687971115, + 0.2681256979703903, + 0.2681456059217453, + 0.2681797593832016, + 0.2682591527700424, + 0.2682824730873108, + 0.268296554684639, + 0.2682979106903076, + 0.26830093562602997, + 0.2683042138814926, + 0.26830457150936127 + ] +} \ No newline at end of file diff --git a/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/history.npz b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/history.npz new file mode 100644 index 0000000..57fbab7 Binary files /dev/null and b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/history.npz differ diff --git a/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/metrics.json b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/metrics.json new file mode 100644 index 0000000..2be1444 --- /dev/null +++ b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/metrics.json @@ -0,0 +1,16 @@ +{ + "train": { + "mse": 0.03740174323320389, + "r2": 0.831692099571228, + "ce_loss": 0.2420116774737835, + "accuracy": 0.9583333333333334, + "f1_macro": 0.9566807313642757 + }, + "validation": { + "mse": 0.04032169654965401, + "r2": 0.8185523748397827, + "ce_loss": 0.26830457150936127, + "accuracy": 0.9666666666666667, + "f1_macro": 0.9710144927536232 + } +} \ No newline at end of file diff --git a/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/model.pt b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/model.pt new file mode 100644 index 0000000..3328f63 Binary files /dev/null and b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/model.pt differ diff --git a/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/training_loss.png b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/training_loss.png new file mode 100644 index 0000000..cdda4f9 Binary files /dev/null and b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/output/training_loss.png differ diff --git a/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/task.py b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/task.py new file mode 100644 index 0000000..ec5e135 --- /dev/null +++ b/MLtasks/tasks/logreg_lvl6_iris_label_smoothing/task.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +""" +Multiclass Logistic Regression on Iris with label smoothing and scheduler. + +Implements the pytorch_task_v1 protocol in one file. +""" + +import json +import os +import sys +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import DataLoader, TensorDataset +from sklearn.datasets import load_iris +from sklearn.metrics import accuracy_score, f1_score, mean_squared_error, r2_score +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + +def get_task_metadata(): + return { + "task_name": "logreg_lvl6_iris_label_smoothing", + "series": "Logistic Regression", + "task_type": "multiclass_classification", + "dataset": "sklearn.load_iris", + "input_dim": 4, + "num_classes": 3, + "features": ["label_smoothing", "cosine_scheduler"], + } + + +def set_seed(seed=42): + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def get_device(): + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +def _standardize(train_x, val_x): + mean = train_x.mean(axis=0, keepdims=True) + std = train_x.std(axis=0, keepdims=True) + 1e-8 + return (train_x - mean) / std, (val_x - mean) / std + + +def make_dataloaders(batch_size=16, train_ratio=0.8, seed=42): + set_seed(seed) + data = load_iris() + x = data.data.astype(np.float32) + y = data.target.astype(np.int64) + + idx = np.random.permutation(len(x)) + split = int(len(x) * train_ratio) + tr_idx, va_idx = idx[:split], idx[split:] + + x_train, y_train = x[tr_idx], y[tr_idx] + x_val, y_val = x[va_idx], y[va_idx] + x_train, x_val = _standardize(x_train, x_val) + + train_ds = TensorDataset(torch.from_numpy(x_train), torch.from_numpy(y_train)) + val_ds = TensorDataset(torch.from_numpy(x_val), torch.from_numpy(y_val)) + + train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True) + val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False) + return train_loader, val_loader, x_train, y_train, x_val, y_val + + +class SoftmaxLogReg(nn.Module): + def __init__(self, input_dim, num_classes): + super().__init__() + self.linear = nn.Linear(input_dim, num_classes) + + def forward(self, x): + return self.linear(x) + + +def build_model(input_dim, num_classes, device): + return SoftmaxLogReg(input_dim, num_classes).to(device) + + +def train(model, train_loader, val_loader, device, epochs=180, lr=0.04): + criterion = nn.CrossEntropyLoss(label_smoothing=0.1) + optimizer = torch.optim.Adam(model.parameters(), lr=lr) + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs) + + history = {"train_loss": [], "val_loss": []} + + for epoch in range(epochs): + model.train() + running = 0.0 + for xb, yb in train_loader: + xb, yb = xb.to(device), yb.to(device) + optimizer.zero_grad() + logits = model(xb) + loss = criterion(logits, yb) + loss.backward() + optimizer.step() + running += loss.item() + scheduler.step() + + train_loss = running / len(train_loader) + val_metrics = evaluate(model, val_loader, device) + history["train_loss"].append(train_loss) + history["val_loss"].append(val_metrics["ce_loss"]) + + if (epoch + 1) % 40 == 0: + print( + f"Epoch {epoch+1:03d} | train_loss={train_loss:.4f} " + f"| val_ce={val_metrics['ce_loss']:.4f} | val_f1={val_metrics['f1_macro']:.4f}" + ) + + return history + + +def evaluate(model, data_loader, device): + model.eval() + criterion = nn.CrossEntropyLoss(label_smoothing=0.0) + + logits_all, y_all = [], [] + total_ce = 0.0 + + with torch.no_grad(): + for xb, yb in data_loader: + xb, yb = xb.to(device), yb.to(device) + logits = model(xb) + total_ce += criterion(logits, yb).item() + logits_all.append(logits.cpu().numpy()) + y_all.append(yb.cpu().numpy()) + + logits_np = np.vstack(logits_all) + y_true = np.concatenate(y_all) + probs = torch.softmax(torch.from_numpy(logits_np), dim=1).numpy() + y_pred = probs.argmax(axis=1) + + y_true_oh = np.eye(probs.shape[1], dtype=np.float32)[y_true] + mse = float(mean_squared_error(y_true_oh, probs)) + r2 = float(r2_score(y_true_oh.reshape(-1), probs.reshape(-1))) + + return { + "mse": mse, + "r2": r2, + "ce_loss": float(total_ce / len(data_loader)), + "accuracy": float(accuracy_score(y_true, y_pred)), + "f1_macro": float(f1_score(y_true, y_pred, average="macro")), + } + + +def predict(model, x, device): + model.eval() + with torch.no_grad(): + x_t = torch.as_tensor(x, dtype=torch.float32, device=device) + probs = torch.softmax(model(x_t), dim=1) + return probs.cpu().numpy() + + +def _get_series(history, keys): + for k in keys: + v = history.get(k) + if isinstance(v, (list, tuple, np.ndarray)) and len(v) > 0: + return np.asarray(v, dtype=np.float32), k + return None, None + + +def plot_training_curves(history, out_dir, prefix="training"): + os.makedirs(out_dir, exist_ok=True) + + # Loss curve + train_loss, train_loss_name = _get_series(history, ["train_loss", "loss_train"]) + val_loss, val_loss_name = _get_series(history, ["val_loss", "valid_loss", "loss_val"]) + + if train_loss is not None or val_loss is not None: + n = len(train_loss) if train_loss is not None else len(val_loss) + epochs = np.arange(1, n + 1) + plt.figure(figsize=(8, 5)) + if train_loss is not None: + plt.plot(epochs, train_loss, label=train_loss_name) + if val_loss is not None: + plt.plot(epochs, val_loss, label=val_loss_name) + plt.xlabel("Epoch") + plt.ylabel("Loss") + plt.title("Training/Validation Loss") + plt.grid(alpha=0.3) + plt.legend() + plt.tight_layout() + plt.savefig(os.path.join(out_dir, f"{prefix}_loss.png"), dpi=150) + plt.close() + + # Accuracy curve (if available) + train_acc, train_acc_name = _get_series(history, ["train_acc", "train_accuracy"]) + val_acc, val_acc_name = _get_series(history, ["val_acc", "val_accuracy", "valid_accuracy"]) + + if train_acc is not None or val_acc is not None: + n = len(train_acc) if train_acc is not None else len(val_acc) + epochs = np.arange(1, n + 1) + plt.figure(figsize=(8, 5)) + if train_acc is not None: + plt.plot(epochs, train_acc, label=train_acc_name) + if val_acc is not None: + plt.plot(epochs, val_acc, label=val_acc_name) + plt.xlabel("Epoch") + plt.ylabel("Accuracy") + plt.title("Training/Validation Accuracy") + plt.grid(alpha=0.3) + plt.legend() + plt.tight_layout() + plt.savefig(os.path.join(out_dir, f"{prefix}_accuracy.png"), dpi=150) + plt.close() + + +def _to_serializable_history(history): + serializable = {} + for k, v in history.items(): + if isinstance(v, np.ndarray): + serializable[k] = v.astype(float).tolist() + elif isinstance(v, (list, tuple)): + serializable[k] = [ + float(x) if isinstance(x, (np.floating, np.integer)) else x for x in v + ] + elif isinstance(v, (np.floating, np.integer)): + serializable[k] = float(v) + else: + serializable[k] = v + return serializable + + +def save_artifacts(model, history, train_metrics, val_metrics, out_dir): + os.makedirs(out_dir, exist_ok=True) + torch.save(model.state_dict(), os.path.join(out_dir, "model.pt")) + + with open(os.path.join(out_dir, "history.json"), "w", encoding="utf-8") as f: + json.dump(_to_serializable_history(history), f, indent=2) + + with open(os.path.join(out_dir, "metrics.json"), "w", encoding="utf-8") as f: + json.dump({"train": train_metrics, "validation": val_metrics}, f, indent=2) + + plot_training_curves(history, out_dir, prefix="training") + + +def main(): + set_seed(42) + device = get_device() + print(f"Using device: {device}") + + metadata = get_task_metadata() + train_loader, val_loader, _, _, _, _ = make_dataloaders(batch_size=16, train_ratio=0.8, seed=42) + model = build_model(metadata["input_dim"], metadata["num_classes"], device) + + history = train(model, train_loader, val_loader, device, epochs=180, lr=0.04) + + train_metrics = evaluate(model, train_loader, device) + val_metrics = evaluate(model, val_loader, device) + + print("Train metrics:", train_metrics) + print("Validation metrics:", val_metrics) + + checks = [ + val_metrics["accuracy"] > 0.90, + val_metrics["f1_macro"] > 0.90, + abs(train_metrics["accuracy"] - val_metrics["accuracy"]) < 0.12, + history["train_loss"][-1] < history["train_loss"][0], + ] + + out_dir = os.path.join(os.path.dirname(__file__), "output") + save_artifacts(model, history, train_metrics, val_metrics, out_dir) + + passed = all(checks) + print("PASS" if passed else "FAIL") + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/MLtasks/tasks/mlp_lvl5_ensemble_distillation/task.py b/MLtasks/tasks/mlp_lvl5_ensemble_distillation/task.py new file mode 100644 index 0000000..092312d --- /dev/null +++ b/MLtasks/tasks/mlp_lvl5_ensemble_distillation/task.py @@ -0,0 +1,650 @@ +""" +Knowledge Distillation: Ensemble Teacher -> Student + +Mathematical Formulation: +- Distillation Loss: L = alpha*CE(student, target) + (1-alpha)*KL(p_s, p_t) + where p_s = softmax(z_s/T), p_t = softmax(z_t/T) + T is temperature (typically 3-20) +- Teacher Ensemble: Average logits from N independent trained teachers +- Student Model: Compact architecture with fewer parameters that learns to mimic ensemble + +This task demonstrates: +1. Training multiple teacher models independently +2. Creating an ensemble predictor (average logits) +3. Knowledge distillation with temperature scaling +4. Measuring distillation efficiency and student compression +""" + +import os +import sys +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, TensorDataset +import torchvision +import torchvision.transforms as transforms +import json +from pathlib import Path + +# Set seeds for reproducibility +torch.manual_seed(42) +np.random.seed(42) + +# Output directory for artifacts +OUTPUT_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'output', 'mlp_lvl5_ensemble_distillation') +os.makedirs(OUTPUT_DIR, exist_ok=True) + + +def get_task_metadata(): + """Return task metadata.""" + return { + 'task_name': 'knowledge_distillation_ensemble', + 'description': 'Train ensemble of teacher MLPs and distill into compact student', + 'n_teachers': 3, + 'teacher_architecture': '784->256->128->10', + 'student_architecture': '784->128->64->10', + 'temperature': 4, + 'alpha': 0.5, + 'task_type': 'image_classification', + 'dataset': 'MNIST' + } + + +def set_seed(seed=42): + """Set random seeds for reproducibility.""" + torch.manual_seed(seed) + np.random.seed(seed) + + +def get_device(): + """Get device (CPU or GPU).""" + return torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + +def make_dataloaders(batch_size=32, download=True): + """ + Create MNIST dataloaders for train and validation. + + Args: + batch_size: Batch size + download: Whether to download MNIST + + Returns: + train_loader, val_loader + """ + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ]) + + # Download and load MNIST + mnist_train = torchvision.datasets.MNIST( + root='./data', train=True, download=download, transform=transform + ) + mnist_test = torchvision.datasets.MNIST( + root='./data', train=False, download=download, transform=transform + ) + + # Split training into 80/20 train/val + n_train = int(0.8 * len(mnist_train)) + indices = torch.randperm(len(mnist_train)) + train_indices = indices[:n_train] + val_indices = indices[n_train:] + + train_subset = torch.utils.data.Subset(mnist_train, train_indices) + val_subset = torch.utils.data.Subset(mnist_train, val_indices) + + train_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True) + val_loader = DataLoader(val_subset, batch_size=batch_size, shuffle=False) + + return train_loader, val_loader + + +class TeacherMLP(nn.Module): + """Teacher MLP: 784 -> 256 -> 128 -> 10""" + + def __init__(self, dropout=0.2): + super(TeacherMLP, self).__init__() + self.fc1 = nn.Linear(784, 256) + self.fc2 = nn.Linear(256, 128) + self.fc3 = nn.Linear(128, 10) + self.relu = nn.ReLU() + self.dropout = nn.Dropout(dropout) + self.device = get_device() + self.to(self.device) + + def forward(self, x): + x = x.view(x.size(0), -1) # Flatten + x = self.relu(self.fc1(x)) + x = self.dropout(x) + x = self.relu(self.fc2(x)) + x = self.dropout(x) + x = self.fc3(x) + return x + + +class StudentMLP(nn.Module): + """Student MLP: 784 -> 128 -> 64 -> 10 (Compact)""" + + def __init__(self, dropout=0.2): + super(StudentMLP, self).__init__() + self.fc1 = nn.Linear(784, 128) + self.fc2 = nn.Linear(128, 64) + self.fc3 = nn.Linear(64, 10) + self.relu = nn.ReLU() + self.dropout = nn.Dropout(dropout) + self.device = get_device() + self.to(self.device) + + def forward(self, x): + x = x.view(x.size(0), -1) # Flatten + x = self.relu(self.fc1(x)) + x = self.dropout(x) + x = self.relu(self.fc2(x)) + x = self.dropout(x) + x = self.fc3(x) + return x + + +def build_model(model_type='teacher', **kwargs): + """Build a model (teacher or student).""" + if model_type == 'teacher': + return TeacherMLP(**kwargs) + elif model_type == 'student': + return StudentMLP(**kwargs) + else: + raise ValueError(f"Unknown model type: {model_type}") + + +def count_parameters(model): + """Count total trainable parameters.""" + return sum(p.numel() for p in model.parameters() if p.requires_grad) + + +def train_teacher(model, train_loader, val_loader, device, epochs=10, lr=0.001): + """ + Train a single teacher model to high accuracy. + + Args: + model: Teacher model + train_loader: Training data loader + val_loader: Validation data loader + device: Device to use + epochs: Number of epochs + lr: Learning rate + + Returns: + dict with training history + """ + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=lr) + + history = {'train_loss': [], 'train_acc': [], 'val_loss': [], 'val_acc': []} + + for epoch in range(epochs): + # Training + model.train() + train_loss = 0.0 + train_correct = 0 + train_total = 0 + + for images, labels in train_loader: + images, labels = images.to(device), labels.to(device) + + optimizer.zero_grad() + outputs = model(images) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + train_loss += loss.item() + _, predicted = torch.max(outputs.data, 1) + train_total += labels.size(0) + train_correct += (predicted == labels).sum().item() + + train_loss /= len(train_loader) + train_acc = train_correct / train_total + history['train_loss'].append(train_loss) + history['train_acc'].append(train_acc) + + # Validation + model.eval() + val_loss = 0.0 + val_correct = 0 + val_total = 0 + + with torch.no_grad(): + for images, labels in val_loader: + images, labels = images.to(device), labels.to(device) + outputs = model(images) + loss = criterion(outputs, labels) + + val_loss += loss.item() + _, predicted = torch.max(outputs.data, 1) + val_total += labels.size(0) + val_correct += (predicted == labels).sum().item() + + val_loss /= len(val_loader) + val_acc = val_correct / val_total + history['val_loss'].append(val_loss) + history['val_acc'].append(val_acc) + + if (epoch + 1) % 5 == 0: + print(f"Teacher Epoch [{epoch+1}/{epochs}], Train Loss: {train_loss:.4f}, " + f"Train Acc: {train_acc:.4f}, Val Acc: {val_acc:.4f}") + + return history + + +def train_with_distillation(student, teachers, train_loader, val_loader, device, + temperature=4, alpha=0.5, epochs=20, lr=0.001): + """ + Train student model with knowledge distillation from ensemble of teachers. + + Distillation Loss: L = alpha*CE(student, target) + (1-alpha)*KL(p_s, p_t) + + Args: + student: Student model + teachers: List of teacher models (all in eval mode) + train_loader: Training data loader + val_loader: Validation data loader + device: Device to use + temperature: Temperature for softmax scaling + alpha: Weight for distillation loss (1-alpha for CE loss) + epochs: Number of epochs + lr: Learning rate + + Returns: + dict with training history + """ + criterion_ce = nn.CrossEntropyLoss() + criterion_kl = nn.KLDivLoss(reduction='batchmean') + optimizer = optim.Adam(student.parameters(), lr=lr) + + # Set teachers to eval mode + for teacher in teachers: + teacher.eval() + + history = {'train_loss': [], 'train_acc': [], 'val_loss': [], 'val_acc': []} + + for epoch in range(epochs): + # Training + student.train() + train_loss = 0.0 + train_correct = 0 + train_total = 0 + + for images, labels in train_loader: + images, labels = images.to(device), labels.to(device) + + # Get teacher ensemble predictions (average logits) + teacher_logits = [] + with torch.no_grad(): + for teacher in teachers: + logits = teacher(images) + teacher_logits.append(logits) + teacher_logits_ensemble = torch.mean(torch.stack(teacher_logits), dim=0) + + # Student prediction + student_logits = student(images) + + # Distillation loss + ce_loss = criterion_ce(student_logits, labels) + + # KL divergence loss (using temperature) + p_student = torch.nn.functional.log_softmax(student_logits / temperature, dim=1) + p_teacher = torch.nn.functional.softmax(teacher_logits_ensemble / temperature, dim=1) + kl_loss = criterion_kl(p_student, p_teacher) + + # Combined loss + loss = alpha * ce_loss + (1 - alpha) * kl_loss + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + train_loss += loss.item() + _, predicted = torch.max(student_logits.data, 1) + train_total += labels.size(0) + train_correct += (predicted == labels).sum().item() + + train_loss /= len(train_loader) + train_acc = train_correct / train_total + history['train_loss'].append(train_loss) + history['train_acc'].append(train_acc) + + # Validation + student.eval() + val_loss = 0.0 + val_correct = 0 + val_total = 0 + + with torch.no_grad(): + for images, labels in train_loader: + images, labels = images.to(device), labels.to(device) + + # Ensemble prediction + teacher_logits = [] + for teacher in teachers: + logits = teacher(images) + teacher_logits.append(logits) + teacher_logits_ensemble = torch.mean(torch.stack(teacher_logits), dim=0) + + # Student prediction + student_logits = student(images) + + # Loss + ce_loss = criterion_ce(student_logits, labels) + p_student = torch.nn.functional.log_softmax(student_logits / temperature, dim=1) + p_teacher = torch.nn.functional.softmax(teacher_logits_ensemble / temperature, dim=1) + kl_loss = criterion_kl(p_student, p_teacher) + loss = alpha * ce_loss + (1 - alpha) * kl_loss + + val_loss += loss.item() + _, predicted = torch.max(student_logits.data, 1) + val_total += labels.size(0) + val_correct += (predicted == labels).sum().item() + + val_loss /= len(val_loader) + val_acc = val_correct / val_total + history['val_loss'].append(val_loss) + history['val_acc'].append(val_acc) + + if (epoch + 1) % 5 == 0: + print(f"Distillation Epoch [{epoch+1}/{epochs}], Train Loss: {train_loss:.4f}, " + f"Train Acc: {train_acc:.4f}, Val Acc: {val_acc:.4f}") + + return history + + +def train(train_loader, val_loader, device, model_type='teacher', **kwargs): + """ + Train wrapper function for compatibility with protocol. + """ + model = build_model(model_type=model_type) + history = train_teacher(model, train_loader, val_loader, device) + return model, history + + +def evaluate(model, data_loader, device, teachers=None, temperature=4, return_dict=True): + """ + Evaluate model on data loader. + + Computes: accuracy, CE loss, and distillation efficiency metrics. + + Args: + model: Model to evaluate (student or teacher) + data_loader: Data loader + device: Device + teachers: Optional list of teachers for ensemble comparison + temperature: Temperature for distillation + return_dict: Whether to return as dict + + Returns: + dict with metrics or float (accuracy) + """ + model.eval() + criterion = nn.CrossEntropyLoss() + + correct = 0 + total = 0 + loss = 0.0 + all_preds = [] + all_targets = [] + + with torch.no_grad(): + for images, labels in data_loader: + images, labels = images.to(device), labels.to(device) + + outputs = model(images) + batch_loss = criterion(outputs, labels) + loss += batch_loss.item() + + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + all_preds.append(predicted.cpu().numpy()) + all_targets.append(labels.cpu().numpy()) + + accuracy = correct / total + avg_loss = loss / len(data_loader) + + if not return_dict: + return accuracy + + metrics = { + 'accuracy': accuracy, + 'ce_loss': avg_loss, + 'correct': correct, + 'total': total + } + + # If teachers provided, compute ensemble comparison + if teachers is not None: + ensemble_correct = 0 + with torch.no_grad(): + for images, labels in data_loader: + images, labels = images.to(device), labels.to(device) + + # Ensemble prediction + teacher_logits = [] + for teacher in teachers: + logits = teacher(images) + teacher_logits.append(logits) + ensemble_logits = torch.mean(torch.stack(teacher_logits), dim=0) + + _, predicted = torch.max(ensemble_logits, 1) + ensemble_correct += (predicted == labels).sum().item() + + ensemble_accuracy = ensemble_correct / total + metrics['ensemble_accuracy'] = ensemble_accuracy + metrics['accuracy_ratio'] = accuracy / ensemble_accuracy if ensemble_accuracy > 0 else 0 + + return metrics + + +def predict(model, x, device): + """ + Make predictions on input. + + Args: + model: Model + x: Input tensor or numpy array + device: Device + + Returns: + Predictions (class labels) + """ + model.eval() + if isinstance(x, np.ndarray): + x = torch.FloatTensor(x) + x = x.to(device) + + with torch.no_grad(): + outputs = model(x) + _, predictions = torch.max(outputs, 1) + + return predictions.cpu().numpy() + + +def save_artifacts(teachers, student, histories, metrics, output_dir=OUTPUT_DIR): + """ + Save all models and metrics. + + Args: + teachers: List of teacher models + student: Student model + histories: Dict with training histories + metrics: Dict with evaluation metrics + output_dir: Output directory + """ + os.makedirs(output_dir, exist_ok=True) + + # Save teachers + for i, teacher in enumerate(teachers): + path = os.path.join(output_dir, f'teacher_{i}.pt') + torch.save(teacher.state_dict(), path) + print(f"Saved teacher {i} to {path}") + + # Save student + student_path = os.path.join(output_dir, 'student.pt') + torch.save(student.state_dict(), student_path) + print(f"Saved student to {student_path}") + + # Save metrics + metrics_path = os.path.join(output_dir, 'metrics.json') + with open(metrics_path, 'w') as f: + json.dump(metrics, f, indent=2) + print(f"Saved metrics to {metrics_path}") + + # Save histories + history_path = os.path.join(output_dir, 'histories.json') + with open(history_path, 'w') as f: + json.dump(histories, f, indent=2) + print(f"Saved histories to {history_path}") + + +if __name__ == '__main__': + """ + Main training and evaluation pipeline: + 1. Load MNIST data + 2. Train 3 independent teacher models + 3. Create ensemble predictor + 4. Train student model with distillation + 5. Evaluate both and compute efficiency metrics + 6. Assert quality thresholds + 7. Exit 0 on success, 1 on failure + """ + + try: + device = get_device() + print(f"Using device: {device}") + + # Load data + print("\nLoading MNIST data...") + train_loader, val_loader = make_dataloaders(batch_size=64, download=True) + + # Train teachers + print("\n" + "="*60) + print("Training Teacher Models") + print("="*60) + teachers = [] + teacher_histories = {} + + for i in range(3): + print(f"\nTraining Teacher {i+1}/3...") + teacher = build_model('teacher') + history = train_teacher(teacher, train_loader, val_loader, device, epochs=10, lr=0.001) + teachers.append(teacher) + teacher_histories[f'teacher_{i}'] = history + teacher_acc = history['val_acc'][-1] + print(f"Teacher {i} final validation accuracy: {teacher_acc:.4f}") + + # Evaluate teachers individually + print("\n" + "="*60) + print("Evaluating Teachers") + print("="*60) + teacher_metrics = {} + for i, teacher in enumerate(teachers): + metrics = evaluate(teacher, val_loader, device, return_dict=True) + teacher_metrics[f'teacher_{i}'] = metrics + print(f"Teacher {i} - Accuracy: {metrics['accuracy']:.4f}") + + # Evaluate ensemble + print("\nEvaluating Teacher Ensemble...") + ensemble_correct = 0 + ensemble_total = 0 + with torch.no_grad(): + for images, labels in val_loader: + images, labels = images.to(device), labels.to(device) + teacher_logits = [] + for teacher in teachers: + teacher.eval() + logits = teacher(images) + teacher_logits.append(logits) + ensemble_logits = torch.mean(torch.stack(teacher_logits), dim=0) + _, predicted = torch.max(ensemble_logits, 1) + ensemble_correct += (predicted == labels).sum().item() + ensemble_total += labels.size(0) + + ensemble_accuracy = ensemble_correct / ensemble_total + print(f"Ensemble accuracy: {ensemble_accuracy:.4f}") + + # Train student with distillation + print("\n" + "="*60) + print("Training Student Model (Distillation)") + print("="*60) + student = build_model('student') + distillation_history = train_with_distillation( + student, teachers, train_loader, val_loader, device, + temperature=4, alpha=0.5, epochs=20, lr=0.001 + ) + + # Evaluate student + print("\n" + "="*60) + print("Evaluating Student Model") + print("="*60) + student_metrics = evaluate(student, val_loader, device, teachers=teachers, + temperature=4, return_dict=True) + print(f"Student accuracy: {student_metrics['accuracy']:.4f}") + print(f"Ensemble accuracy: {student_metrics['ensemble_accuracy']:.4f}") + print(f"Student/Ensemble ratio: {student_metrics['accuracy_ratio']:.4f}") + + # Compute parameter counts + teacher_params = count_parameters(teachers[0]) + student_params = count_parameters(student) + compression_ratio = student_params / (teacher_params * 3) + + print(f"\nParameter efficiency:") + print(f"Teacher params (single): {teacher_params}") + print(f"Total teacher params (3x): {teacher_params * 3}") + print(f"Student params: {student_params}") + print(f"Compression ratio: {compression_ratio:.4f}") + + # Save artifacts + all_histories = { + 'teachers': teacher_histories, + 'distillation': distillation_history + } + all_metrics = { + 'teachers': teacher_metrics, + 'ensemble_accuracy': ensemble_accuracy, + 'student': student_metrics, + 'compression_ratio': compression_ratio + } + + save_artifacts(teachers, student, all_histories, all_metrics) + + # Assertions for quality + print("\n" + "="*60) + print("Quality Assertions") + print("="*60) + + assert student_metrics['accuracy'] > 0.92, \ + f"Student accuracy {student_metrics['accuracy']:.4f} must be > 0.92" + print("✓ Student accuracy > 0.92") + + assert student_metrics['accuracy'] > ensemble_accuracy * 0.85, \ + f"Student accuracy should be >= 85% of ensemble accuracy" + print("✓ Student achieves >= 85% of ensemble performance") + + assert ensemble_accuracy > 0.97, \ + f"Ensemble accuracy {ensemble_accuracy:.4f} must be > 0.97" + print("✓ Ensemble accuracy > 0.97") + + assert compression_ratio < 1.0, \ + f"Student should have fewer params than single teacher" + print(f"✓ Student is compressed (ratio: {compression_ratio:.4f})") + + print("\n" + "="*60) + print("SUCCESS: All assertions passed!") + print("="*60) + sys.exit(0) + + except Exception as e: + print(f"\nERROR: {str(e)}") + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/MLtasks/tasks/nlp_lvl1_text_embedding/task.py b/MLtasks/tasks/nlp_lvl1_text_embedding/task.py new file mode 100644 index 0000000..0550d0a --- /dev/null +++ b/MLtasks/tasks/nlp_lvl1_text_embedding/task.py @@ -0,0 +1,603 @@ +""" +Word Embeddings with Skip-gram and Sentiment Classification + +Mathematical Formulation: +- Skip-gram Objective: maximize sum_w log p(w_context | w_target) +- Negative Sampling: log(sigmoid(v_c · v_t)) + sum_i E_neg[log(sigmoid(-v_i · v_t))] +- Similarity: cosine similarity between word vectors +- Classification: Logistic regression on sentence embeddings + +This task demonstrates: +1. Training word embeddings from scratch using Skip-gram with negative sampling +2. Evaluating embedding quality through word similarity +3. Building a sentiment classifier on top of embeddings +4. Handling text processing and vocabulary management +""" + +import os +import sys +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, Dataset +import json +from pathlib import Path +from collections import defaultdict, Counter +import random + +# Set seeds for reproducibility +torch.manual_seed(42) +np.random.seed(42) +random.seed(42) + +# Output directory for artifacts +OUTPUT_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'output', 'nlp_lvl1_text_embedding') +os.makedirs(OUTPUT_DIR, exist_ok=True) + + +def get_task_metadata(): + """Return task metadata.""" + return { + 'task_name': 'word_embeddings_sentiment', + 'description': 'Skip-gram word embeddings + sentiment classification', + 'embedding_dim': 100, + 'context_window': 5, + 'negative_samples': 15, + 'task_type': 'nlp_sentiment', + 'vocab_size_target': 5000 + } + + +def set_seed(seed=42): + """Set random seeds for reproducibility.""" + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + +def get_device(): + """Get device (CPU or GPU).""" + return torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + +class Vocabulary: + """Build and manage vocabulary.""" + + def __init__(self, min_freq=2): + self.word2id = {} + self.id2word = {} + self.word_freq = Counter() + self.min_freq = min_freq + self.size = 0 + + def add_word(self, word): + """Add word to vocabulary.""" + self.word_freq[word] += 1 + + def build_vocab(self, max_vocab=5000): + """Build vocabulary from word frequencies.""" + # Add special tokens + self.word2id[''] = 0 + self.word2id[''] = 1 + self.id2word[0] = '' + self.id2word[1] = '' + + current_id = 2 + for word, freq in self.word_freq.most_common(max_vocab): + if freq >= self.min_freq and current_id < max_vocab: + self.word2id[word] = current_id + self.id2word[current_id] = word + current_id += 1 + + self.size = len(self.word2id) + + def get_id(self, word): + """Get word ID.""" + return self.word2id.get(word, 0) # 0 is + + def get_word(self, word_id): + """Get word from ID.""" + return self.id2word.get(word_id, '') + + +class SkipGramDataset(Dataset): + """Skip-gram dataset for word embeddings.""" + + def __init__(self, sentences, vocab, window_size=5, neg_samples=15): + self.pairs = [] + self.neg_samples = neg_samples + self.vocab = vocab + + # Build (target, context) pairs + for sentence in sentences: + tokens = [vocab.get_id(w) for w in sentence.split()] + # Remove tokens (0) that exceed threshold + tokens = [t for t in tokens if t != 0 or random.random() < 0.1] + + for i, target in enumerate(tokens): + # Context window + start = max(0, i - window_size) + end = min(len(tokens), i + window_size + 1) + + for j in range(start, end): + if i != j: + context = tokens[j] + self.pairs.append((target, context)) + + # Pre-compute negative sampling distribution (unigram distribution) + freq_sum = sum(vocab.word_freq.values()) + self.neg_samples_dist = np.array([ + vocab.word_freq.get(vocab.get_word(i), 1) ** 0.75 / freq_sum + for i in range(vocab.size) + ]) + self.neg_samples_dist = self.neg_samples_dist / self.neg_samples_dist.sum() + + def __len__(self): + return len(self.pairs) + + def __getitem__(self, idx): + target, context = self.pairs[idx] + + # Sample negative examples + negatives = np.random.choice( + self.vocab.size, size=self.neg_samples, p=self.neg_samples_dist + ) + + return target, context, torch.LongTensor(negatives) + + +def make_dataloaders(batch_size=64): + """ + Create toy sentiment corpus and dataloaders for embedding training and classification. + + Returns: + embedding_loader, sentiment_data (for final classification evaluation) + """ + # Toy sentiment dataset + positive_reviews = [ + "this movie is great and amazing", + "i loved this film very much", + "fantastic performance by actors", + "excellent direction and cinematography", + "absolutely wonderful movie experience", + "best film i have seen this year", + "incredible story and great actors", + "loved every minute of this film", + "amazing plot and perfect ending", + "outstanding performance throughout" + ] + + negative_reviews = [ + "this movie is terrible and boring", + "i hated this film very much", + "awful performance by actors", + "terrible direction and cinematography", + "absolutely horrible movie experience", + "worst film i have seen this year", + "terrible story and bad actors", + "hated every minute of this film", + "boring plot and disappointing ending", + "awful performance throughout" + ] + + # Build vocabulary + vocab = Vocabulary(min_freq=1) + + all_texts = positive_reviews + negative_reviews + for text in all_texts: + for word in text.split(): + vocab.add_word(word) + + vocab.build_vocab(max_vocab=5000) + + # Create embedding training dataset + embedding_dataset = SkipGramDataset(all_texts, vocab, window_size=5, neg_samples=15) + embedding_loader = DataLoader(embedding_dataset, batch_size=batch_size, shuffle=True) + + # Create sentiment classification data + sentiment_data = { + 'positive': positive_reviews, + 'negative': negative_reviews, + 'labels': [1] * len(positive_reviews) + [0] * len(negative_reviews) + } + + return embedding_loader, sentiment_data, vocab + + +class SkipGramModel(nn.Module): + """Skip-gram embedding model with negative sampling.""" + + def __init__(self, vocab_size, embedding_dim): + super(SkipGramModel, self).__init__() + self.vocab_size = vocab_size + self.embedding_dim = embedding_dim + + # Embedding matrices + self.target_embedding = nn.Embedding(vocab_size, embedding_dim) + self.context_embedding = nn.Embedding(vocab_size, embedding_dim) + + # Initialize + nn.init.uniform_(self.target_embedding.weight, -0.5/embedding_dim, 0.5/embedding_dim) + nn.init.uniform_(self.context_embedding.weight, -0.5/embedding_dim, 0.5/embedding_dim) + + def forward(self, target, context, negatives): + """ + Compute negative sampling loss. + + Args: + target: Target word IDs (B,) + context: Context word IDs (B,) + negatives: Negative sample IDs (B, neg_samples) + + Returns: + Loss (scalar) + """ + # Positive pair + target_emb = self.target_embedding(target) # (B, D) + context_emb = self.context_embedding(context) # (B, D) + + pos_score = torch.sum(target_emb * context_emb, dim=1) # (B,) + pos_loss = -torch.log(torch.sigmoid(pos_score)) # (B,) + + # Negative pairs + neg_emb = self.context_embedding(negatives) # (B, neg_samples, D) + neg_score = torch.matmul(target_emb.unsqueeze(1), neg_emb.transpose(1, 2)).squeeze(1) # (B, neg_samples) + neg_loss = -torch.log(torch.sigmoid(-neg_score)) # (B, neg_samples) + + loss = pos_loss + torch.sum(neg_loss, dim=1) + return loss.mean() + + def get_embeddings(self): + """Get learned word embeddings.""" + return self.target_embedding.weight.data + + +def build_model(vocab_size, embedding_dim=100, **kwargs): + """Build Skip-gram model.""" + return SkipGramModel(vocab_size, embedding_dim) + + +def train_embeddings(model, embedding_loader, device, epochs=10, lr=0.025): + """ + Train word embeddings using Skip-gram with negative sampling. + + Args: + model: SkipGramModel + embedding_loader: DataLoader for embedding training + device: Device + epochs: Number of epochs + lr: Learning rate + + Returns: + Training history + """ + optimizer = optim.Adam(model.parameters(), lr=lr) + + history = {'loss': []} + + for epoch in range(epochs): + epoch_loss = 0.0 + for target, context, negatives in embedding_loader: + target, context = target.to(device), context.to(device) + negatives = negatives.to(device) + + optimizer.zero_grad() + loss = model(target, context, negatives) + loss.backward() + optimizer.step() + + epoch_loss += loss.item() + + avg_loss = epoch_loss / len(embedding_loader) + history['loss'].append(avg_loss) + + if (epoch + 1) % 2 == 0: + print(f"Embedding Epoch [{epoch+1}/{epochs}], Loss: {avg_loss:.4f}") + + return history + + +def compute_word_similarity(embeddings, vocab, word1, word2): + """Compute cosine similarity between two words.""" + id1 = vocab.get_id(word1) + id2 = vocab.get_id(word2) + + emb1 = embeddings[id1] + emb2 = embeddings[id2] + + similarity = torch.nn.functional.cosine_similarity(emb1.unsqueeze(0), emb2.unsqueeze(0)).item() + return similarity + + +def get_similar_words(embeddings, vocab, word, top_k=5): + """Get top-k similar words using embedding similarity.""" + word_id = vocab.get_id(word) + word_emb = embeddings[word_id] # (D,) + + # Compute similarities with all words + similarities = torch.nn.functional.cosine_similarity( + word_emb.unsqueeze(0), embeddings, dim=1 + ) # (vocab_size,) + + # Get top-k (excluding the word itself) + top_ids = torch.argsort(similarities, descending=True)[:top_k+1] + top_words = [] + for wid in top_ids: + if wid != word_id: + top_words.append((vocab.get_word(wid.item()), similarities[wid].item())) + + return top_words[:top_k] + + +class SentimentClassifier(nn.Module): + """Sentiment classifier on top of embeddings.""" + + def __init__(self, embedding_dim, frozen_embeddings=None): + super(SentimentClassifier, self).__init__() + self.fc1 = nn.Linear(embedding_dim, 64) + self.fc2 = nn.Linear(64, 1) + self.relu = nn.ReLU() + + def forward(self, x): + x = self.relu(self.fc1(x)) + x = torch.sigmoid(self.fc2(x)) + return x + + +def evaluate_sentiment(embeddings, vocab, sentiment_data, device, epochs=20, lr=0.01): + """ + Train and evaluate sentiment classifier on top of embeddings. + + Args: + embeddings: Word embeddings (vocab_size, embedding_dim) + vocab: Vocabulary + sentiment_data: Dict with reviews and labels + device: Device + epochs: Training epochs + lr: Learning rate + + Returns: + dict with metrics + """ + classifier = SentimentClassifier(embeddings.size(1)) + classifier.to(device) + + optimizer = optim.Adam(classifier.parameters(), lr=lr) + criterion = nn.BCELoss() + + # Prepare data + all_reviews = sentiment_data['positive'] + sentiment_data['negative'] + labels = torch.tensor(sentiment_data['labels'], dtype=torch.float32).to(device) + + # Convert reviews to embeddings (average word embeddings) + review_embeddings = [] + for review in all_reviews: + tokens = [vocab.get_id(w) for w in review.split()] + if tokens: + review_emb = embeddings[tokens].mean(dim=0) + else: + review_emb = torch.zeros(embeddings.size(1), device=device) + review_embeddings.append(review_emb) + + review_embeddings = torch.stack(review_embeddings).to(device) + + # Train + for epoch in range(epochs): + classifier.train() + optimizer.zero_grad() + + logits = classifier(review_embeddings) + loss = criterion(logits, labels.unsqueeze(1)) + loss.backward() + optimizer.step() + + if (epoch + 1) % 5 == 0: + print(f" Sentiment Epoch [{epoch+1}/{epochs}], Loss: {loss.item():.4f}") + + # Evaluate + classifier.eval() + with torch.no_grad(): + logits = classifier(review_embeddings) + predictions = (logits > 0.5).long().squeeze() + accuracy = (predictions == labels.long()).float().mean().item() + + return { + 'accuracy': accuracy, + 'loss': loss.item() + } + + +def train(embedding_loader, device, epochs=10, lr=0.025, **kwargs): + """Train wrapper for protocol.""" + model = build_model(5000, embedding_dim=100) + model.to(device) + history = train_embeddings(model, embedding_loader, device, epochs=epochs, lr=lr) + return model, history + + +def predict(embeddings, vocab, word_list): + """Predict similar words for a list of words.""" + results = {} + for word in word_list: + similar = get_similar_words(embeddings, vocab, word, top_k=5) + results[word] = similar + return results + + +def evaluate(model, embedding_loader, device, vocab=None, return_dict=True): + """Evaluate model (embedding quality metrics).""" + embeddings = model.get_embeddings() + + # Test word similarities + test_pairs = [ + ('great', 'good'), + ('bad', 'terrible'), + ('movie', 'film') + ] + + similarities = [] + for w1, w2 in test_pairs: + sim = compute_word_similarity(embeddings, vocab, w1, w2) + similarities.append(sim) + + avg_similarity = np.mean(similarities) + + if not return_dict: + return avg_similarity + + return { + 'avg_similarity': avg_similarity, + 'embedding_norm': embeddings.norm().item() + } + + +def save_artifacts(model, embeddings, vocab, histories, metrics, output_dir=OUTPUT_DIR): + """Save models and artifacts.""" + os.makedirs(output_dir, exist_ok=True) + + # Save model + model_path = os.path.join(output_dir, 'embedding_model.pt') + torch.save(model.state_dict(), model_path) + print(f"Saved model to {model_path}") + + # Save embeddings + embeddings_path = os.path.join(output_dir, 'embeddings.pt') + torch.save(embeddings, embeddings_path) + print(f"Saved embeddings to {embeddings_path}") + + # Save vocabulary + vocab_path = os.path.join(output_dir, 'vocab.json') + with open(vocab_path, 'w') as f: + json.dump({ + 'word2id': vocab.word2id, + 'id2word': {str(k): v for k, v in vocab.id2word.items()} + }, f, indent=2) + print(f"Saved vocabulary to {vocab_path}") + + # Save metrics + metrics_path = os.path.join(output_dir, 'metrics.json') + # Convert tensors to python types for json serialization + metrics_serializable = {} + for k, v in metrics.items(): + if isinstance(v, dict): + metrics_serializable[k] = {k2: float(v2) if isinstance(v2, (torch.Tensor, np.ndarray)) else v2 + for k2, v2 in v.items()} + else: + metrics_serializable[k] = float(v) if isinstance(v, (torch.Tensor, np.ndarray)) else v + + with open(metrics_path, 'w') as f: + json.dump(metrics_serializable, f, indent=2) + print(f"Saved metrics to {metrics_path}") + + +if __name__ == '__main__': + """ + Main pipeline: + 1. Build vocabulary and embedding dataset + 2. Train Skip-gram embeddings + 3. Evaluate embedding quality (word similarity) + 4. Train sentiment classifier on embeddings + 5. Assert quality thresholds + """ + + try: + device = get_device() + print(f"Using device: {device}") + + # Load data + print("\nPreparing embedding and sentiment data...") + embedding_loader, sentiment_data, vocab = make_dataloaders(batch_size=16) + + print(f"Vocabulary size: {vocab.size}") + print(f"Embedding training pairs: {len(embedding_loader.dataset)}") + + # Train embeddings + print("\n" + "="*60) + print("Training Word Embeddings (Skip-gram with Negative Sampling)") + print("="*60) + + model = build_model(vocab.size, embedding_dim=100) + model.to(device) + embedding_history = train_embeddings(model, embedding_loader, device, epochs=10, lr=0.025) + + # Get learned embeddings + embeddings = model.get_embeddings() + + # Evaluate embedding quality + print("\n" + "="*60) + print("Evaluating Embedding Quality") + print("="*60) + + print("\nSimilar words to 'great':") + similar_great = get_similar_words(embeddings, vocab, 'great', top_k=5) + for word, sim in similar_great: + print(f" {word}: {sim:.4f}") + + print("\nSimilar words to 'bad':") + similar_bad = get_similar_words(embeddings, vocab, 'bad', top_k=5) + for word, sim in similar_bad: + print(f" {word}: {sim:.4f}") + + print("\nSimilar words to 'movie':") + similar_movie = get_similar_words(embeddings, vocab, 'movie', top_k=5) + for word, sim in similar_movie: + print(f" {word}: {sim:.4f}") + + # Compute word pair similarities + print("\nWord pair similarities:") + pairs = [('great', 'good'), ('bad', 'terrible'), ('movie', 'film')] + pair_similarities = [] + for w1, w2 in pairs: + sim = compute_word_similarity(embeddings, vocab, w1, w2) + pair_similarities.append(sim) + print(f" '{w1}' <-> '{w2}': {sim:.4f}") + + avg_pair_sim = np.mean(pair_similarities) + + # Train sentiment classifier + print("\n" + "="*60) + print("Training Sentiment Classifier on Embeddings") + print("="*60) + + sentiment_metrics = evaluate_sentiment(embeddings, vocab, sentiment_data, device, epochs=20, lr=0.01) + print(f"\nSentiment classification accuracy: {sentiment_metrics['accuracy']:.4f}") + + # Collect metrics + all_metrics = { + 'embedding_training_loss': embedding_history['loss'][-1], + 'avg_word_pair_similarity': float(avg_pair_sim), + 'embedding_norm': float(embeddings.norm()), + 'sentiment_accuracy': sentiment_metrics['accuracy'], + 'vocab_size': vocab.size + } + + save_artifacts(model, embeddings, vocab, embedding_history, all_metrics) + + # Assertions for quality + print("\n" + "="*60) + print("Quality Assertions") + print("="*60) + + assert avg_pair_sim > 0.3, \ + f"Average word pair similarity {avg_pair_sim:.4f} should be > 0.3" + print("✓ Word embeddings show reasonable semantic similarity") + + assert sentiment_metrics['accuracy'] > 0.7, \ + f"Sentiment accuracy {sentiment_metrics['accuracy']:.4f} should be > 0.7" + print("✓ Sentiment classifier achieves > 70% accuracy on embeddings") + + assert embeddings.norm() > 0, \ + "Embeddings should have non-zero magnitude" + print("✓ Embeddings are properly initialized") + + print("\n" + "="*60) + print("SUCCESS: All assertions passed!") + print("="*60) + sys.exit(0) + + except Exception as e: + print(f"\nERROR: {str(e)}") + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/MLtasks/tasks/nlp_lvl2_bigframe_embeddings/task.py b/MLtasks/tasks/nlp_lvl2_bigframe_embeddings/task.py new file mode 100644 index 0000000..8680662 --- /dev/null +++ b/MLtasks/tasks/nlp_lvl2_bigframe_embeddings/task.py @@ -0,0 +1,779 @@ +# -*- coding: utf-8 -*- +""" +BigQuery Bigframe Text Embeddings with Semantic Similarity Task + +Mathematical Formulation: +- Embedding Generation: e_i = Embed(text_i) where Embed is BigQuery Vertex AI Embeddings +- Cosine Similarity: sim(e_i, e_j) = (e_i · e_j) / (||e_i|| ||e_j||) +- Semantic Search: Find k-nearest neighbors using embedding similarity +- Neural Network Classifier: y = sigmoid(MLP(concatenate_embeddings(text_pairs))) +- Contrastive Loss: L = -log(exp(sim(e_pos) / tau) / sum_j exp(sim(e_j) / tau)) + where tau is temperature scaling parameter + +This task demonstrates: +1. Loading text data from BigQuery Bigframe with proper authentication handling +2. Generating embeddings using BigQuery Vertex AI Embeddings API +3. Building semantic similarity models using neural networks +4. Evaluating embedding quality through semantic textual similarity (STS) benchmarks +5. Fallback to local embeddings when BigQuery is unavailable (for testing) +""" + +import os +import sys +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, Dataset +import json +from pathlib import Path +import random +import hashlib +from typing import List, Dict, Tuple, Optional +import warnings + +# Suppress warnings +warnings.filterwarnings('ignore') + +# Set seeds for reproducibility +torch.manual_seed(42) +np.random.seed(42) +random.seed(42) + +# Output directory for artifacts +OUTPUT_DIR = os.path.join( + os.path.dirname(__file__), '..', '..', '..', 'output', 'nlp_lvl2_bigframe_embeddings' +) +os.makedirs(OUTPUT_DIR, exist_ok=True) + + +def get_task_metadata(): + """Return task metadata.""" + return { + 'task_name': 'bigframe_embeddings_semantic_similarity', + 'description': 'BigQuery Bigframe embeddings with semantic similarity neural network', + 'embedding_dim': 768, # Vertex AI embeddings dimension + 'task_type': 'nlp_semantic_similarity', + 'use_bigquery': bool(os.getenv('GOOGLE_APPLICATION_CREDENTIALS')), + 'fallback_mode': 'local_simulated_embeddings' + } + + +def set_seed(seed=42): + """Set random seeds for reproducibility.""" + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + +def get_device(): + """Get device (CPU or GPU).""" + return torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + +class BigFrameDataLoader: + """ + Handles loading text data from BigQuery Bigframe. + + This class demonstrates: + - Connecting to BigQuery with Bigframe + - Loading data using SQL queries + - Fallback to local data when BigQuery is unavailable + """ + + def __init__(self, use_bigquery: bool = False): + self.use_bigquery = use_bigquery and os.getenv('GOOGLE_APPLICATION_CREDENTIALS') + + if self.use_bigquery: + try: + import bigframes + import pandas as pd + self.bigframes = bigframes + self.pd = pd + print("[BigFrame] Using BigQuery Bigframe for data loading") + except ImportError: + print("[BigFrame] WARNING: bigframes library not found. Falling back to local data.") + self.use_bigquery = False + + def load_data(self) -> Tuple[List[str], List[str], List[int]]: + """ + Load text data from BigQuery Bigframe or local fallback. + + Returns: + Tuple of (texts1, texts2, similarity_labels) + - texts1: List of first texts + - texts2: List of second texts + - similarity_labels: Binary labels (1 = similar, 0 = dissimilar) + """ + if self.use_bigquery: + return self._load_from_bigquery() + else: + return self._load_local_fallback() + + def _load_from_bigquery(self) -> Tuple[List[str], List[str], List[int]]: + """ + Load data from BigQuery using Bigframe API. + + Example query structure: + SELECT text1, text2, similarity_score FROM `project.dataset.sts_benchmark` + WHERE similarity_score IS NOT NULL LIMIT 1000 + """ + try: + # Set up BigFrame session + session = self.bigframes.Session( + credentials_path=os.getenv('GOOGLE_APPLICATION_CREDENTIALS') + ) + + # Query STS (Semantic Textual Similarity) benchmark data + query = """ + SELECT text1, text2, similarity_score + FROM `google-research-datasets.sts_benchmark.train` + WHERE similarity_score IS NOT NULL + LIMIT 500 + """ + + df = session.sql(query) + + # Convert to local DataFrame + df_local = df.to_pandas() + + texts1 = df_local['text1'].tolist() + texts2 = df_local['text2'].tolist() + # Normalize similarity scores (0-5) to binary labels + similarity_labels = [ + 1 if score >= 3.0 else 0 + for score in df_local['similarity_score'].tolist() + ] + + print(f"[BigFrame] Loaded {len(texts1)} text pairs from BigQuery") + return texts1, texts2, similarity_labels + + except Exception as e: + print(f"[BigFrame] ERROR loading from BigQuery: {e}") + print("[BigFrame] Falling back to local data") + return self._load_local_fallback() + + def _load_local_fallback(self) -> Tuple[List[str], List[str], List[int]]: + """ + Fallback to local semantic similarity dataset. + + This provides a representative sample for testing and demonstration. + """ + # Expanded Semantic Textual Similarity pairs (similar and dissimilar) + similar_pairs = [ + ("A man is playing a large flute", "A man is playing a flute"), + ("A woman is eating something", "A woman is consuming food"), + ("A boy is jumping on the trampoline", "A boy is jumping on a bed"), + ("Three people are dancing", "Multiple people are performing a dance"), + ("The cars are driving in the opposite directions", "The cars are driving in different ways"), + ("The grass is green", "The green grass is beautiful"), + ("A dog is running in the park", "A canine is jogging in the garden"), + ("People are dancing at a wedding", "Guests are performing at a celebration"), + ("The cat is sleeping on the bed", "A feline is resting on furniture"), + ("Two women are playing volleyball", "Women are engaged in a volleyball match"), + ("The sun is shining brightly", "The sun is bright today"), + ("A child is playing with a toy", "A young person is playing with a plaything"), + ("The weather is cloudy", "It is overcast outside"), + ("She is running quickly", "She is moving fast"), + ("The book is on the shelf", "The novel is placed on the bookshelf"), + ("A car is parked outside", "A vehicle is standing outside"), + ("The river flows to the sea", "Water flows downstream to the ocean"), + ("A bird is flying in the sky", "A bird is soaring through the air"), + ("He is wearing a red shirt", "He has a red shirt on"), + ("The flower is blooming", "The flower is in bloom"), + ("A cup of coffee is on the table", "Coffee is sitting on the table in a cup"), + ("The door is closed", "The door is shut"), + ("She is smiling happily", "She has a happy smile"), + ("The computer is working", "The computer is functioning properly"), + ("A student is studying hard", "A pupil is working diligently"), + ("The movie is interesting", "The film is captivating"), + ("He is driving a car", "He is operating a vehicle"), + ("The music is loud", "The sound is intense"), + ("A tree is tall", "A tree has great height"), + ("The food is delicious", "The meal tastes wonderful"), + ("A person is walking", "A human is moving on foot"), + ("The window is open", "The window is ajar"), + ("She is happy", "She is in a good mood"), + ("The water is cold", "The water is chilly"), + ("A dog barks loudly", "A dog is barking"), + ("The road is long", "The road extends far"), + ("A person is tired", "A person is exhausted"), + ("The room is bright", "The room is well-lit"), + ("A child is playing outside", "A child is outdoors playing"), + ("The night is dark", "The night is pitch black"), + ("A man is tall", "A man has a great stature"), + ("The job is difficult", "The job is challenging"), + ("She is intelligent", "She is smart"), + ("The house is clean", "The house is tidy"), + ("A baby is crying", "An infant is weeping"), + ("The road is wet", "The road is damp"), + ("A person is strong", "A person is powerful"), + ("The movie was entertaining", "The film was enjoyable"), + ("She is wearing a blue dress", "She has on a blue skirt"), + ("A cat is sleeping", "A cat is resting"), + ] + + dissimilar_pairs = [ + ("A man is riding a horse", "The weather is sunny today"), + ("The car is red", "Birds are flying in the sky"), + ("She likes apples", "He prefers mathematics"), + ("The book is on the table", "The dog is barking loudly"), + ("I enjoy reading novels", "Cars are made of metal"), + ("The sun is shining", "Computers process data"), + ("Water is flowing downhill", "Music is playing softly"), + ("The building is tall", "Cats can climb trees"), + ("She is wearing a blue dress", "Coffee is hot"), + ("The clock shows noon", "Flowers bloom in spring"), + ("Pizza is delicious", "The elephant is large"), + ("Programming is fun", "Tennis is a sport"), + ("The ocean is blue", "Iron is dense"), + ("A piano is an instrument", "Ice cream is cold"), + ("The forest is green", "Diamonds are precious"), + ("Basketball is played indoors", "Lightning is powerful"), + ("She likes chocolate", "Mathematics is abstract"), + ("The rainbow is colorful", "Submarines are underwater"), + ("A bicycle has wheels", "The earth rotates"), + ("Milk is white", "Stars shine at night"), + ("The desert is hot", "Penguins live in cold regions"), + ("Dance is an art form", "Numbers are infinite"), + ("A fish swims", "Mountains touch the sky"), + ("Wine is fermented", "The sky has clouds"), + ("A phone can ring", "Gravity pulls things down"), + ("Poetry uses words", "Atoms are tiny"), + ("The ocean has waves", "Rockets fly to space"), + ("Silver is shiny", "The internet connects people"), + ("A camera takes pictures", "Wind moves the trees"), + ("Glass is transparent", "Thunder follows lightning"), + ("The stadium is large", "Magnets attract metal"), + ("A microscope magnifies", "Robots are mechanical"), + ("Pepper is spicy", "Evolution changes species"), + ("The highway is busy", "Bacteria are microorganisms"), + ("A painting is colorful", "Electricity powers devices"), + ("The forest is quiet", "Fossils tell us about the past"), + ("A bridge connects places", "Genes carry information"), + ("Marble is hard", "Photosynthesis produces oxygen"), + ("The beach has sand", "Volcanoes erupt lava"), + ("A lighthouse guides ships", "Rainbows form after rain"), + ("Silk is smooth", "Pollution damages the environment"), + ("The volcano erupts", "Hurricanes cause damage"), + ("A microscope shows details", "Black holes are in space"), + ("Diamonds sparkle", "Tornadoes are dangerous"), + ("The laboratory experiments", "Earthquakes shake the ground"), + ("A museum displays art", "Tsunamis are ocean waves"), + ("Steel is strong", "Avalanches slide down mountains"), + ("The carnival is fun", "Wildfires spread quickly"), + ("A garden has flowers", "Deserts have little water"), + ("The symphony plays music", "Glaciers are melting"), + ] + + texts1 = [] + texts2 = [] + labels = [] + + # Add similar pairs (label=1) + for text1, text2 in similar_pairs: + texts1.append(text1) + texts2.append(text2) + labels.append(1) + + # Add dissimilar pairs (label=0) + for text1, text2 in dissimilar_pairs: + texts1.append(text1) + texts2.append(text2) + labels.append(0) + + # Shuffle + combined = list(zip(texts1, texts2, labels)) + random.shuffle(combined) + texts1, texts2, labels = zip(*combined) + + print(f"[BigFrame] Loaded {len(texts1)} text pairs from local fallback") + return list(texts1), list(texts2), list(labels) + + +class EmbeddingGenerator: + """ + Generate embeddings using BigQuery Vertex AI Embeddings API or local fallback. + """ + + def __init__(self, embedding_dim: int = 768, use_bigquery: bool = False): + self.embedding_dim = embedding_dim + self.use_bigquery = use_bigquery and os.getenv('GOOGLE_APPLICATION_CREDENTIALS') + self.embedding_cache = {} + + if self.use_bigquery: + try: + from vertexai.language_models import TextEmbeddingModel + self.model = TextEmbeddingModel.from_pretrained("textembedding-gecko@001") + print("[Embeddings] Using Vertex AI TextEmbedding model") + except ImportError: + print("[Embeddings] WARNING: Vertex AI not available. Using local embeddings.") + self.use_bigquery = False + + def embed_text(self, text: str) -> np.ndarray: + """ + Generate embedding for a text string. + + Args: + text: Input text string + + Returns: + Embedding vector of shape (embedding_dim,) + """ + if text in self.embedding_cache: + return self.embedding_cache[text] + + if self.use_bigquery: + embedding = self._embed_with_vertex_ai(text) + else: + embedding = self._embed_locally(text) + + self.embedding_cache[text] = embedding + return embedding + + def _embed_with_vertex_ai(self, text: str) -> np.ndarray: + """Generate embedding using Vertex AI TextEmbedding API.""" + try: + embeddings = self.model.get_embeddings([text]) + embedding = np.array(embeddings[0].values, dtype=np.float32) + return embedding + except Exception as e: + print(f"[Embeddings] Error with Vertex AI: {e}. Using local fallback.") + return self._embed_locally(text) + + def _embed_locally(self, text: str) -> np.ndarray: + """ + Generate simulated embeddings locally using hash-based approach. + + For demonstration: creates deterministic embeddings based on text content + that preserve semantic similarity through character n-gram statistics. + """ + # Create deterministic embedding from text + text_lower = text.lower() + + # Initialize embedding + embedding = np.zeros(self.embedding_dim, dtype=np.float32) + + # Use character n-grams to create features + for i in range(len(text_lower) - 2): + trigram = text_lower[i:i+3] + # Hash-based feature index + feature_idx = (sum(ord(c) for c in trigram) * 7919) % self.embedding_dim + embedding[feature_idx] += 1.0 + + # Add word-level features + words = text_lower.split() + for word in words: + feature_idx = ((sum(ord(c) for c in word) * 7919) % (self.embedding_dim // 2)) + (self.embedding_dim // 2) + embedding[feature_idx % self.embedding_dim] += 0.5 + + # Normalize + norm = np.linalg.norm(embedding) + if norm > 0: + embedding = embedding / norm + + # Add some noise for realism (but deterministic) + np.random.seed((sum(ord(c) for c in text) * 7919) % (2**31)) + embedding += np.random.normal(0, 0.01, self.embedding_dim).astype(np.float32) + + # Renormalize + norm = np.linalg.norm(embedding) + if norm > 0: + embedding = embedding / norm + + return embedding.astype(np.float32) + + def embed_texts_batch(self, texts: List[str]) -> np.ndarray: + """ + Generate embeddings for a batch of texts. + + Args: + texts: List of text strings + + Returns: + Array of shape (len(texts), embedding_dim) + """ + embeddings = np.array([ + self.embed_text(text) for text in texts + ], dtype=np.float32) + return embeddings + + +class SemanticSimilarityDataset(Dataset): + """Dataset for semantic similarity pairs with pre-computed embeddings.""" + + def __init__( + self, + texts1: List[str], + texts2: List[str], + labels: List[int], + embedding_generator: EmbeddingGenerator + ): + self.texts1 = texts1 + self.texts2 = texts2 + self.labels = labels + self.embedding_generator = embedding_generator + + # Pre-compute all embeddings + print("[Dataset] Pre-computing embeddings for all text pairs...") + all_texts = list(set(texts1 + texts2)) + self.embeddings_cache = {} + for text in all_texts: + self.embeddings_cache[text] = embedding_generator.embed_text(text) + print(f"[Dataset] Cached {len(self.embeddings_cache)} unique embeddings") + + def __len__(self): + return len(self.texts1) + + def __getitem__(self, idx): + emb1 = torch.FloatTensor(self.embeddings_cache[self.texts1[idx]]) + emb2 = torch.FloatTensor(self.embeddings_cache[self.texts2[idx]]) + label = torch.LongTensor([self.labels[idx]]) + + return emb1, emb2, label.squeeze() + + +def make_dataloaders(batch_size=16, use_bigquery: Optional[bool] = None): + """ + Create dataloaders for semantic similarity task. + + Pipeline: + 1. Load text pairs from BigQuery Bigframe (or local fallback) + 2. Generate embeddings using Vertex AI or local method + 3. Create semantic similarity dataset + 4. Split into train/validation + + Returns: + Tuple of (train_loader, val_loader, embedding_generator) + """ + # Auto-detect BigQuery usage from credentials unless explicitly provided. + if use_bigquery is None: + use_bigquery = bool(os.getenv('GOOGLE_APPLICATION_CREDENTIALS')) + + # Load data from BigQuery Bigframe + bigframe_loader = BigFrameDataLoader(use_bigquery=use_bigquery) + texts1, texts2, labels = bigframe_loader.load_data() + + # Generate embeddings + embedding_generator = EmbeddingGenerator(embedding_dim=768, use_bigquery=use_bigquery) + + # Create dataset + dataset = SemanticSimilarityDataset(texts1, texts2, labels, embedding_generator) + + # Split into train/validation (80/20) + train_size = int(0.8 * len(dataset)) + val_size = len(dataset) - train_size + + # Create a generator with fixed seed for reproducibility + generator = torch.Generator() + generator.manual_seed(42) + + train_dataset, val_dataset = torch.utils.data.random_split( + dataset, [train_size, val_size], generator=generator + ) + + train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, generator=generator) + val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False) + + return train_loader, val_loader, embedding_generator + + +class SemanticSimilarityModel(nn.Module): + """ + Neural network for semantic similarity prediction. + + Architecture: + - Input: Concatenated embeddings of two texts [emb1, emb2, emb1*emb2, |emb1-emb2|] + - Hidden layers with ReLU activation and dropout + - Output: Binary classification (similar or not) + """ + + def __init__(self, embedding_dim: int = 768, hidden_dim: int = 512): + super(SemanticSimilarityModel, self).__init__() + + # Input: concatenated similarities features + # Features: [emb1, emb2, emb1*emb2, |emb1-emb2|] = 4*embedding_dim + input_dim = 4 * embedding_dim + + self.network = nn.Sequential( + nn.Linear(input_dim, hidden_dim), + nn.ReLU(), + nn.Dropout(0.6), + nn.Linear(hidden_dim, hidden_dim // 2), + nn.ReLU(), + nn.Dropout(0.5), + nn.Linear(hidden_dim // 2, 256), + nn.ReLU(), + nn.Dropout(0.4), + nn.Linear(256, 2) # Binary classification + ) + + def forward(self, emb1, emb2): + """ + Compute similarity score for embedding pairs. + + Args: + emb1: Embedding 1 of shape (B, D) + emb2: Embedding 2 of shape (B, D) + + Returns: + Logits of shape (B, 2) + """ + # Create similarity features + elementwise_mult = emb1 * emb2 + elementwise_diff = torch.abs(emb1 - emb2) + + # Concatenate all features + combined = torch.cat([emb1, emb2, elementwise_mult, elementwise_diff], dim=1) + + logits = self.network(combined) + return logits + + +def build_model(embedding_dim: int = 768, hidden_dim: int = 512, **kwargs): + """Build semantic similarity model.""" + return SemanticSimilarityModel(embedding_dim, hidden_dim) + + +def train(model, train_loader, device, epochs=20, lr=0.001): + """ + Train semantic similarity model. + + Args: + model: SemanticSimilarityModel + train_loader: DataLoader for training + device: PyTorch device + epochs: Number of training epochs + lr: Learning rate + + Returns: + Training history + """ + optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=2e-4) + criterion = nn.CrossEntropyLoss() + + history = {'loss': [], 'accuracy': []} + + model.train() + for epoch in range(epochs): + total_loss = 0.0 + correct = 0 + total = 0 + + for emb1, emb2, labels in train_loader: + emb1, emb2, labels = emb1.to(device), emb2.to(device), labels.to(device) + + optimizer.zero_grad() + logits = model(emb1, emb2) + loss = criterion(logits, labels) + loss.backward() + optimizer.step() + + total_loss += loss.item() + + # Compute accuracy + preds = torch.argmax(logits, dim=1) + correct += (preds == labels).sum().item() + total += labels.size(0) + + avg_loss = total_loss / len(train_loader) + avg_accuracy = correct / total + + history['loss'].append(avg_loss) + history['accuracy'].append(avg_accuracy) + + if (epoch + 1) % 5 == 0: + print(f"Epoch [{epoch+1}/{epochs}], Loss: {avg_loss:.4f}, Accuracy: {avg_accuracy:.4f}") + + return history + + +def evaluate(model, data_loader, device): + """ + Evaluate semantic similarity model. + + Args: + model: SemanticSimilarityModel + data_loader: DataLoader for evaluation + device: PyTorch device + + Returns: + Dictionary with evaluation metrics + """ + model.eval() + criterion = nn.CrossEntropyLoss() + + total_loss = 0.0 + correct = 0 + total = 0 + + with torch.no_grad(): + for emb1, emb2, labels in data_loader: + emb1, emb2, labels = emb1.to(device), emb2.to(device), labels.to(device) + + logits = model(emb1, emb2) + loss = criterion(logits, labels) + + total_loss += loss.item() + + preds = torch.argmax(logits, dim=1) + correct += (preds == labels).sum().item() + total += labels.size(0) + + accuracy = correct / total + avg_loss = total_loss / len(data_loader) + + return { + 'accuracy': accuracy, + 'loss': avg_loss, + 'correct': correct, + 'total': total + } + + +def predict(model, emb1, emb2, device): + """ + Predict similarity for embedding pairs. + + Args: + model: SemanticSimilarityModel + emb1: Embedding 1 + emb2: Embedding 2 + device: PyTorch device + + Returns: + Predicted similarity (0 or 1) + """ + model.eval() + with torch.no_grad(): + emb1 = torch.FloatTensor(emb1).unsqueeze(0).to(device) + emb2 = torch.FloatTensor(emb2).unsqueeze(0).to(device) + logits = model(emb1, emb2) + pred = torch.argmax(logits, dim=1).item() + + return pred + + +def save_artifacts(model, embedding_generator, history, metrics, output_dir=OUTPUT_DIR): + """ + Save model, embeddings, and metrics. + + Args: + model: Trained model + embedding_generator: Embedding generator with cache + history: Training history + metrics: Evaluation metrics + output_dir: Output directory + """ + os.makedirs(output_dir, exist_ok=True) + + # Save model + model_path = os.path.join(output_dir, 'semantic_similarity_model.pt') + torch.save(model.state_dict(), model_path) + print(f"[Artifacts] Saved model to {model_path}") + + # Save metrics + metrics_path = os.path.join(output_dir, 'metrics.json') + metrics_to_save = { + 'validation_accuracy': metrics['accuracy'], + 'validation_loss': metrics['loss'], + 'training_history': { + 'loss': [float(l) for l in history['loss']], + 'accuracy': [float(a) for a in history['accuracy']] + } + } + with open(metrics_path, 'w') as f: + json.dump(metrics_to_save, f, indent=2) + print(f"[Artifacts] Saved metrics to {metrics_path}") + + # Save embedding cache summary + cache_summary_path = os.path.join(output_dir, 'embedding_cache_summary.json') + cache_summary = { + 'cached_texts': len(embedding_generator.embedding_cache), + 'embedding_dim': embedding_generator.embedding_dim + } + with open(cache_summary_path, 'w') as f: + json.dump(cache_summary, f, indent=2) + print(f"[Artifacts] Saved embedding cache summary to {cache_summary_path}") + + +if __name__ == '__main__': + print("\n" + "="*80) + print("BigQuery Bigframe Embeddings - Semantic Similarity Task") + print("="*80) + + try: + # Setup + device = get_device() + set_seed(42) + + metadata = get_task_metadata() + print(f"\n[Config] Task: {metadata['task_name']}") + print(f"[Config] Embedding Dimension: {metadata['embedding_dim']}") + print(f"[Config] Using BigQuery: {metadata['use_bigquery']}") + print(f"[Config] Device: {device}") + + # Create dataloaders + print("\n[Data] Loading data from BigQuery Bigframe...") + train_loader, val_loader, embedding_generator = make_dataloaders(batch_size=16) + + # Build model + print("\n[Model] Building semantic similarity model...") + model = build_model(embedding_dim=768, hidden_dim=512) + model = model.to(device) + + total_params = sum(p.numel() for p in model.parameters()) + print(f"[Model] Total parameters: {total_params:,}") + + # Train + print("\n[Training] Starting training...") + history = train(model, train_loader, device, epochs=20, lr=0.001) + + # Evaluate on validation set + print("\n[Evaluation] Evaluating on validation set...") + val_metrics = evaluate(model, val_loader, device) + + print(f"\n[Results] Validation Accuracy: {val_metrics['accuracy']:.4f}") + print(f"[Results] Validation Loss: {val_metrics['loss']:.4f}") + + # Evaluate on training set + train_metrics = evaluate(model, train_loader, device) + print(f"\n[Results] Training Accuracy: {train_metrics['accuracy']:.4f}") + print(f"[Results] Training Loss: {train_metrics['loss']:.4f}") + + # Quality assertions + print("\n[Assertions] Checking quality thresholds...") + + assert val_metrics['accuracy'] > 0.60, \ + f"Validation accuracy {val_metrics['accuracy']:.4f} must be > 0.60" + print("✓ Validation accuracy > 0.60") + + assert train_metrics['accuracy'] > 0.65, \ + f"Training accuracy {train_metrics['accuracy']:.4f} must be > 0.65" + print("✓ Training accuracy > 0.65") + + assert val_metrics['loss'] < 1.0, \ + f"Validation loss {val_metrics['loss']:.4f} must be < 1.0" + print("✓ Validation loss < 1.0") + + assert len(history['accuracy']) > 0, "Training history empty" + print("✓ Training history recorded") + + # Save artifacts + print("\n[Saving] Persisting model and artifacts...") + save_artifacts(model, embedding_generator, history, val_metrics) + + print("\n" + "="*80) + print("SUCCESS: All assertions passed!") + print("="*80) + sys.exit(0) + + except AssertionError as e: + print(f"\n✗ ASSERTION FAILED: {e}") + print("="*80) + sys.exit(1) + + except Exception as e: + print(f"\n✗ ERROR: {e}") + import traceback + traceback.print_exc() diff --git a/MLtasks/tasks/nlp_lvl3_bigframe_llm_classification/task.py b/MLtasks/tasks/nlp_lvl3_bigframe_llm_classification/task.py new file mode 100644 index 0000000..f52286d --- /dev/null +++ b/MLtasks/tasks/nlp_lvl3_bigframe_llm_classification/task.py @@ -0,0 +1,942 @@ +# -*- coding: utf-8 -*- +""" +BigQuery Bigframe LLM Text Classification Task + +Mathematical Formulation: +- LLM Feature Generation: f_i = LLM_encode(text_i) where LLM generates semantic features +- Context Augmentation: text_augmented_i = LLM_generate(context, text_i, num_variations) +- Classification Network: y = softmax(MLP(concat(f_original, f_augmented))) +- Cross-Entropy Loss: L = -sum_i y_i * log(pred_i) +- Accuracy: acc = (correct_predictions) / total_predictions + +This task demonstrates: +1. Loading text data from BigQuery Bigframe with proper SQL queries +2. Using BigQuery Vertex AI LLM API for feature generation and data augmentation +3. Building ensemble-style classifiers combining original and LLM-augmented features +4. Evaluating multi-class text classification with LLM-enriched representations +5. Proper error handling and fallback to local LLM simulation for testing environments +""" + +import os +import sys +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, Dataset +import json +from pathlib import Path +import random +from typing import List, Dict, Tuple, Optional +import warnings +import hashlib + +# Suppress warnings +warnings.filterwarnings('ignore') + +# Set seeds for reproducibility +torch.manual_seed(42) +np.random.seed(42) +random.seed(42) + +# Output directory for artifacts +OUTPUT_DIR = os.path.join( + os.path.dirname(__file__), '..', '..', '..', 'output', 'nlp_lvl3_bigframe_llm_classification' +) +os.makedirs(OUTPUT_DIR, exist_ok=True) + + +def get_task_metadata(): + """Return task metadata.""" + return { + 'task_name': 'bigframe_llm_text_classification', + 'description': 'BigQuery Bigframe with LLM features for text classification', + 'num_classes': 3, + 'feature_dim': 512, + 'task_type': 'nlp_classification_with_llm', + 'use_bigquery': bool(os.getenv('GOOGLE_APPLICATION_CREDENTIALS')), + 'use_llm': bool(os.getenv('GOOGLE_APPLICATION_CREDENTIALS')), + 'augmentation_ratio': 0.5 + } + + +def set_seed(seed=42): + """Set random seeds for reproducibility.""" + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + +def get_device(): + """Get device (CPU or GPU).""" + return torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + +class BigFrameDataLoader: + """ + Handles loading text classification data from BigQuery Bigframe. + + Demonstrates: + - Connecting to BigQuery with Bigframe + - Loading data using SQL queries with WHERE clauses and LIMIT + - Handling authentication and connection errors + - Fallback to local data when BigQuery is unavailable + """ + + def __init__(self, use_bigquery: bool = False): + self.use_bigquery = use_bigquery and os.getenv('GOOGLE_APPLICATION_CREDENTIALS') + + if self.use_bigquery: + try: + import bigframes + import pandas as pd + self.bigframes = bigframes + self.pd = pd + print("[BigFrame] Initialized BigQuery Bigframe connection") + except ImportError: + print("[BigFrame] WARNING: bigframes library not found. Using local data.") + self.use_bigquery = False + + def load_data(self) -> Tuple[List[str], List[int]]: + """ + Load text classification data from BigQuery Bigframe or local fallback. + + Returns: + Tuple of (texts, labels) where: + - texts: List of text strings + - labels: List of class labels (0, 1, or 2) + """ + if self.use_bigquery: + return self._load_from_bigquery() + else: + return self._load_local_fallback() + + def _load_from_bigquery(self) -> Tuple[List[str], List[int]]: + """ + Load data from BigQuery using Bigframe API. + + Example query structure for sentiment or topic classification: + SELECT text, label FROM `project.dataset.text_classification` + WHERE split = 'train' AND label IS NOT NULL + LIMIT 1000 + + Can also use AGG functions: + SELECT text, label, COUNT(*) as frequency + FROM `project.dataset.text_classification` + GROUP BY text, label + HAVING frequency > 10 + LIMIT 1000 + """ + try: + # Set up BigFrame session + session = self.bigframes.Session( + credentials_path=os.getenv('GOOGLE_APPLICATION_CREDENTIALS') + ) + + # Query DBpedia or other public classification dataset + # DBpedia has 14 classes: Company, EducationalInstitution, Artist, etc. + query = """ + SELECT abstract as text, dbo_type as label + FROM `bigquery-public-data.dbpedia.text_descriptions` + WHERE dbo_type IN ('Company', 'Person', 'Place') + AND abstract IS NOT NULL + AND LENGTH(abstract) BETWEEN 50 AND 500 + LIMIT 300 + """ + + df = session.sql(query) + + # Convert to local DataFrame + df_local = df.to_pandas() + + texts = df_local['text'].tolist() + + # Map string labels to integers + label_map = {'Company': 0, 'Person': 1, 'Place': 2} + labels = [label_map.get(label, 0) for label in df_local['label'].tolist()] + + print(f"[BigFrame] Loaded {len(texts)} texts from BigQuery") + return texts, labels + + except Exception as e: + print(f"[BigFrame] ERROR loading from BigQuery: {e}") + print("[BigFrame] Falling back to local data") + return self._load_local_fallback() + + def _load_local_fallback(self) -> Tuple[List[str], List[int]]: + """ + Fallback to local text classification dataset. + + Classes: + - 0: Technology/Science + - 1: Business/Economics + - 2: Entertainment/Sports + """ + + tech_texts = [ + "Artificial intelligence and machine learning are transforming industries globally.", + "Quantum computing could revolutionize cryptography and data processing.", + "Neural networks can now recognize patterns in complex data at superhuman levels.", + "Cloud computing provides scalable infrastructure for modern applications.", + "Blockchain technology enables decentralized and secure transactions.", + "Deep learning has achieved breakthrough results in computer vision tasks.", + "5G networks will enable faster and more reliable wireless communications.", + "Natural language processing powers modern conversational AI systems.", + "GPU acceleration dramatically improves computational performance.", + "Cybersecurity is critical for protecting digital infrastructure.", + "Data science enables organizations to extract insights from vast datasets.", + "Computer vision enables machines to interpret and analyze visual information.", + "Robotics and automation are revolutionizing manufacturing processes.", + "Internet of Things connects billions of devices worldwide.", + "Software development methodologies continue to evolve with new frameworks.", + "Programming languages influence how developers approach problem solving.", + "Distributed systems architecture enables scalable applications.", + "Algorithm optimization reduces computational complexity significantly.", + "Machine vision systems improve quality control in factories.", + "Advanced analytics predict future trends and patterns accurately.", + "Quantum encryption provides unbreakable security solutions.", + "Artificial neural networks mimic biological brain structures.", + "Big data analytics uncover hidden patterns in large datasets.", + "Software testing ensures applications work reliably.", + "DevOps practices accelerate software development cycles.", + "API design enables seamless integration between systems.", + "Database optimization improves application performance.", + "Cloud infrastructure reduces capital expenditure significantly.", + "Microservices architecture enables scalable system design.", + "Open source software accelerates innovation in technology.", + ] + + business_texts = [ + "Market valuations reached record highs as investors show confidence.", + "The stock market reacted positively to earnings reports from major corporations.", + "Mergers and acquisitions continue to reshape the competitive landscape.", + "Economic growth remains strong with unemployment at historic lows.", + "Interest rates influence investment decisions across all sectors.", + "Trade policies impact global supply chains and business operations.", + "Consumer spending drives economic activity in developed nations.", + "Corporate profits increased due to improved operational efficiency.", + "Financial markets showed resilience despite economic uncertainty.", + "Business expansion into emerging markets offers significant opportunities.", + "Revenue streams diversify as companies enter new market segments.", + "Quarterly earnings reports reveal company performance metrics.", + "Dividend payments reward shareholders for their investments.", + "Market competition intensifies pricing pressure across industries.", + "Supply chain management optimizes product distribution networks.", + "Sales forecasting predicts future revenue and demand patterns.", + "Cost reduction initiatives improve profit margins substantially.", + "Strategic partnerships create synergies between organizations.", + "Customer acquisition costs influence long-term profitability.", + "Business intelligence systems provide actionable insights rapidly.", + "Venture capital funding accelerates startup growth trajectories.", + "Product lifecycle management maximizes market opportunity.", + "Operational efficiency reduces overall business costs.", + "Market segmentation targets specific customer demographics.", + "Pricing strategies influence customer purchasing decisions.", + "Brand loyalty creates competitive advantages in markets.", + "Distribution channels expand product accessibility.", + "Customer service excellence differentiates market leaders.", + "Financial analysis informs strategic business decisions.", + "Human resources management attracts talented employees.", + ] + + entertainment_texts = [ + "The latest superhero film broke box office records on opening weekend.", + "Music festivals attract millions of fans from around the world.", + "Award shows celebrate the best performances in television and film.", + "Streaming services have revolutionized how people consume entertainment.", + "Professional sports leagues generate billions in revenue annually.", + "Celebrity gossip and news captivate audiences on social media.", + "Video games have become a dominant form of interactive entertainment.", + "Concert tours by famous artists sell out stadiums worldwide.", + "Television dramas receive critical acclaim for writing and acting.", + "Fashion shows showcase the latest designs from top designers.", + "Movie theaters experience declining attendance as streaming grows.", + "Reality television programs attract large viewership audiences.", + "Sports commentary provides entertaining analysis of athletic events.", + "Entertainment news covers celebrity activities and scandals extensively.", + "Video streaming platforms compete for exclusive content rights.", + "Animation technology creates increasingly realistic visual effects.", + "Live performances showcase talent in front of enthusiastic audiences.", + "Entertainment industry generates substantial economic activity.", + "Social media influences entertainment consumption and preferences.", + "Podcast creators build loyal audiences through regular episodes.", + "Film production budgets reach unprecedented levels annually.", + "Theater performances attract diverse audiences culturally.", + "Documentary films educate audiences about important topics.", + "Comedy shows provide laughter and entertainment value.", + "Dance performances showcase artistic expression beautifully.", + "Opera productions combine music and theatrical storytelling.", + "Circus entertainment thrills audiences with acrobatic feats.", + "Stand-up comedy defines modern entertainment culture.", + "Animated movies appeal to audiences of all ages.", + "Musical theater combines song, dance, and dramatic storytelling.", + ] + + texts = tech_texts + business_texts + entertainment_texts + labels = [0]*len(tech_texts) + [1]*len(business_texts) + [2]*len(entertainment_texts) + + # Shuffle + combined = list(zip(texts, labels)) + random.shuffle(combined) + texts, labels = zip(*combined) + + print(f"[BigFrame] Loaded {len(texts)} texts from local fallback") + return list(texts), list(labels) + + +class LLMFeatureGenerator: + """ + Generate semantic features and augmentations using BigQuery Vertex AI LLM API. + + Provides: + - Text classification features from LLM + - Data augmentation through paraphrasing + - Keyword/topic extraction + """ + + def __init__(self, use_llm: bool = False, feature_dim: int = 512): + self.use_llm = use_llm and os.getenv('GOOGLE_APPLICATION_CREDENTIALS') + self.feature_dim = feature_dim + self.feature_cache = {} + self.augmentation_cache = {} + + if self.use_llm: + try: + from vertexai.language_models import TextGenerationModel + self.llm_model = TextGenerationModel.from_pretrained("text-bison@001") + print("[LLM] Using Vertex AI TextGenerationModel for feature generation") + except ImportError: + print("[LLM] WARNING: Vertex AI not available. Using local LLM simulation.") + self.use_llm = False + + def generate_features(self, text: str) -> np.ndarray: + """ + Generate semantic features for text using LLM or local simulation. + + Args: + text: Input text + + Returns: + Feature vector of shape (feature_dim,) + """ + if text in self.feature_cache: + return self.feature_cache[text] + + if self.use_llm: + features = self._generate_with_llm(text) + else: + features = self._generate_locally(text) + + self.feature_cache[text] = features + return features + + def _generate_with_llm(self, text: str) -> np.ndarray: + """Generate features using Vertex AI LLM API.""" + try: + # Create prompt for feature extraction + prompt = f"""Extract key semantic features from this text (respond with a list of 10-15 keywords): + +Text: {text} + +Features:""" + + response = self.llm_model.predict( + prompt=prompt, + temperature=0.7, + max_output_tokens=100 + ) + + # Parse response to create feature vector + features = np.zeros(self.feature_dim, dtype=np.float32) + + keywords = response.text.split(',') + for i, keyword in enumerate(keywords): + keyword = keyword.strip().lower() + if keyword: + feature_idx = (hash(keyword) % self.feature_dim) + features[feature_idx] += 1.0 + + # Normalize + norm = np.linalg.norm(features) + if norm > 0: + features = features / norm + + return features.astype(np.float32) + + except Exception as e: + print(f"[LLM] Error: {e}. Using local fallback.") + return self._generate_locally(text) + + def _generate_locally(self, text: str) -> np.ndarray: + """ + Generate features locally using text statistics and keyword detection. + + Enhanced approach: Detects domain-specific keywords to create features + that better distinguish between Technology, Business, and Entertainment. + """ + features = np.zeros(self.feature_dim, dtype=np.float32) + + text_lower = text.lower() + + # Domain-specific keywords for feature generation + tech_keywords = [ + 'artificial', 'intelligence', 'machine', 'learning', 'algorithm', 'computing', + 'quantum', 'neural', 'network', 'cloud', 'blockchain', 'deep', 'gpu', + 'cybersecurity', 'data', 'science', 'software', 'programming', 'technology', + 'digital', 'automation', 'internet', 'optimization', 'api', 'database' + ] + + business_keywords = [ + 'market', 'stock', 'revenue', 'profit', 'financial', 'investment', 'earnings', + 'merger', 'acquisition', 'economic', 'business', 'corporate', 'sales', 'cost', + 'supply', 'demand', 'customer', 'growth', 'strategy', 'company', 'enterprise', + 'dividend', 'shareholder', 'pricing', 'distribution', 'vendor' + ] + + entertainment_keywords = [ + 'film', 'movie', 'music', 'sport', 'entertainment', 'celebrity', 'performance', + 'concert', 'theater', 'television', 'game', 'video', 'award', 'show', 'festival', + 'actor', 'actress', 'artist', 'streaming', 'audience', 'fan', 'drama', 'comedy' + ] + + # Count keyword occurrences and create features + tech_count = sum(text_lower.count(kw) for kw in tech_keywords) + business_count = sum(text_lower.count(kw) for kw in business_keywords) + entertainment_count = sum(text_lower.count(kw) for kw in entertainment_keywords) + + # Divide feature space into regions + third = self.feature_dim // 3 + + # Create distinctive feature patterns + # Tech features in first 1/3 of vector + if tech_count > 0: + tech_region = np.arange(third, dtype=np.float32) + tech_features = np.exp(-tech_region / (third / 2)) * tech_count + features[:third] = tech_features[:third] + + # Business features in middle 1/3 + if business_count > 0: + business_region = np.arange(third, dtype=np.float32) + business_features = np.exp(-business_region / (third / 2)) * business_count + features[third:2*third] = business_features[:third] + + # Entertainment features in last 1/3 + if entertainment_count > 0: + entertainment_region = np.arange(self.feature_dim - 2*third, dtype=np.float32) + entertainment_features = np.exp(-entertainment_region / (third / 2)) * entertainment_count + features[2*third:] = entertainment_features[:self.feature_dim - 2*third] + + # Add word-level features for fine-grained representation + words = text_lower.split() + for word in words: + # Hash word to feature space + feature_idx = (hash(word) % self.feature_dim) + features[feature_idx] += 0.1 + + # Normalize + norm = np.linalg.norm(features) + if norm > 0: + features = features / norm + + return features.astype(np.float32) + + def augment_text(self, text: str) -> str: + """ + Generate augmented/paraphrased version of text using LLM. + + Args: + text: Original text + + Returns: + Augmented text + """ + cache_key = f"aug_{text}" + if cache_key in self.augmentation_cache: + return self.augmentation_cache[cache_key] + + if self.use_llm: + augmented = self._augment_with_llm(text) + else: + augmented = self._augment_locally(text) + + self.augmentation_cache[cache_key] = augmented + return augmented + + def _augment_with_llm(self, text: str) -> str: + """Augment text using LLM.""" + try: + prompt = f"""Rewrite this text in a different way while preserving the meaning: + +Original: {text} + +Rewritten:""" + + response = self.llm_model.predict( + prompt=prompt, + temperature=0.8, + max_output_tokens=100 + ) + + return response.text.strip() + + except Exception as e: + return self._augment_locally(text) + + def _augment_locally(self, text: str) -> str: + """ + Local augmentation: simple synonym replacement simulation. + """ + # Simple synonym replacements for augmentation + synonym_map = { + 'very': 'quite, extremely', + 'good': 'excellent, great, fine', + 'bad': 'poor, terrible, awful', + 'important': 'significant, crucial, vital', + 'important': 'key, essential, critical', + 'large': 'big, huge, massive', + 'small': 'tiny, little, minimal', + 'fast': 'quick, rapid, swift', + 'slow': 'sluggish, gradual, unhurried', + 'happy': 'joyful, delighted, pleased', + 'sad': 'unhappy, miserable, gloomy', + } + + words = text.lower().split() + augmented_words = [] + + for word in words: + # Remove punctuation + clean_word = word.rstrip('.,!?;:') + + if clean_word in synonym_map: + # Use first synonym + synonyms = synonym_map[clean_word].split(',') + replacement = synonyms[0].strip() + augmented_words.append(replacement + word[len(clean_word):]) + else: + augmented_words.append(word) + + return ' '.join(augmented_words) + + +class LLMEnrichedDataset(Dataset): + """Dataset combining original and LLM-augmented features.""" + + def __init__( + self, + texts: List[str], + labels: List[int], + llm_generator: LLMFeatureGenerator, + augmentation_ratio: float = 0.5 + ): + self.texts = texts + self.labels = labels + self.llm_generator = llm_generator + self.augmentation_ratio = augmentation_ratio + + # Pre-compute all features + print("[Dataset] Pre-computing LLM features...") + self.original_features = [] + self.augmented_features = [] + + for i, text in enumerate(texts): + # Original text features + original_feat = llm_generator.generate_features(text) + self.original_features.append(original_feat) + + # Augmented text features + if random.random() < augmentation_ratio: + augmented_text = llm_generator.augment_text(text) + augmented_feat = llm_generator.generate_features(augmented_text) + else: + augmented_feat = original_feat.copy() + + self.augmented_features.append(augmented_feat) + + if (i + 1) % 10 == 0: + print(f" [{i+1}/{len(texts)}] features computed") + + print(f"[Dataset] Computed {len(texts)} feature pairs") + + def __len__(self): + return len(self.texts) + + def __getitem__(self, idx): + original_feat = torch.FloatTensor(self.original_features[idx]) + augmented_feat = torch.FloatTensor(self.augmented_features[idx]) + label = torch.LongTensor([self.labels[idx]]) + + return original_feat, augmented_feat, label.squeeze() + + +def make_dataloaders(batch_size=16): + """ + Create dataloaders for LLM-enhanced text classification. + + Pipeline: + 1. Load texts and labels from BigQuery Bigframe (or local fallback) + 2. Generate features using LLM or local simulation + 3. Create augmented versions of texts + 4. Split into train/validation + + Returns: + Tuple of (train_loader, val_loader, llm_generator) + """ + # Load data + print("[Data] Loading data from BigQuery Bigframe...") + bigframe_loader = BigFrameDataLoader(use_bigquery=False) # Set True if BigQuery available + texts, labels = bigframe_loader.load_data() + + print(f"[Data] Loaded {len(texts)} texts with {len(set(labels))} classes") + + # Initialize LLM generator + print("[LLM] Initializing LLM feature generator...") + llm_generator = LLMFeatureGenerator(use_llm=False, feature_dim=512) # Set True if LLM available + + # Create dataset + dataset = LLMEnrichedDataset(texts, labels, llm_generator, augmentation_ratio=0.5) + + # Split into train/validation (80/20) + train_size = int(0.8 * len(dataset)) + val_size = len(dataset) - train_size + + # Create a generator with fixed seed for reproducibility + generator = torch.Generator() + generator.manual_seed(42) + + train_dataset, val_dataset = torch.utils.data.random_split( + dataset, [train_size, val_size], generator=generator + ) + + train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, generator=generator) + val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False) + + return train_loader, val_loader, llm_generator + + +class LLMEnrichedClassifier(nn.Module): + """ + Text classifier combining original and LLM-augmented features. + + Architecture: + - Input: Concatenated [original_features, augmented_features] + - Hidden layers with ReLU and dropout + - Output: Multi-class logits + """ + + def __init__(self, feature_dim: int = 512, num_classes: int = 3, hidden_dim: int = 256): + super(LLMEnrichedClassifier, self).__init__() + + # Input: concatenated original and augmented features + input_dim = 2 * feature_dim + + self.network = nn.Sequential( + nn.Linear(input_dim, hidden_dim), + nn.ReLU(), + nn.BatchNorm1d(hidden_dim), + nn.Dropout(0.5), # Increased from 0.4 + + nn.Linear(hidden_dim, hidden_dim // 2), + nn.ReLU(), + nn.BatchNorm1d(hidden_dim // 2), + nn.Dropout(0.4), # Increased from 0.3 + + nn.Linear(hidden_dim // 2, 128), + nn.ReLU(), + nn.Dropout(0.35), # Increased from 0.2 + + nn.Linear(128, num_classes) + ) + + def forward(self, original_feat, augmented_feat): + """ + Classify text based on combined features. + + Args: + original_feat: Original text features of shape (B, feature_dim) + augmented_feat: Augmented text features of shape (B, feature_dim) + + Returns: + Logits of shape (B, num_classes) + """ + combined = torch.cat([original_feat, augmented_feat], dim=1) + logits = self.network(combined) + return logits + + +def build_model(feature_dim: int = 512, num_classes: int = 3, hidden_dim: int = 256, **kwargs): + """Build LLM-enriched classifier.""" + return LLMEnrichedClassifier(feature_dim, num_classes, hidden_dim) + + +def train(model, train_loader, device, epochs=25, lr=0.001): + """ + Train the classifier. + + Args: + model: LLMEnrichedClassifier + train_loader: DataLoader + device: PyTorch device + epochs: Number of training epochs + lr: Learning rate + + Returns: + Training history + """ + optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=2e-4) # Increased from 1e-5 + criterion = nn.CrossEntropyLoss() + + history = {'loss': [], 'accuracy': []} + + model.train() + for epoch in range(epochs): + total_loss = 0.0 + correct = 0 + total = 0 + + for original_feat, augmented_feat, labels in train_loader: + original_feat = original_feat.to(device) + augmented_feat = augmented_feat.to(device) + labels = labels.to(device) + + optimizer.zero_grad() + logits = model(original_feat, augmented_feat) + loss = criterion(logits, labels) + loss.backward() + optimizer.step() + + total_loss += loss.item() + + preds = torch.argmax(logits, dim=1) + correct += (preds == labels).sum().item() + total += labels.size(0) + + avg_loss = total_loss / len(train_loader) + avg_accuracy = correct / total + + history['loss'].append(avg_loss) + history['accuracy'].append(avg_accuracy) + + if (epoch + 1) % 5 == 0: + print(f"Epoch [{epoch+1}/{epochs}], Loss: {avg_loss:.4f}, Accuracy: {avg_accuracy:.4f}") + + return history + + +def evaluate(model, data_loader, device): + """ + Evaluate classifier on validation/test set. + + Args: + model: LLMEnrichedClassifier + data_loader: DataLoader + device: PyTorch device + + Returns: + Dictionary with evaluation metrics + """ + model.eval() + criterion = nn.CrossEntropyLoss() + + total_loss = 0.0 + correct = 0 + total = 0 + + class_correct = [0, 0, 0] + class_total = [0, 0, 0] + + with torch.no_grad(): + for original_feat, augmented_feat, labels in data_loader: + original_feat = original_feat.to(device) + augmented_feat = augmented_feat.to(device) + labels = labels.to(device) + + logits = model(original_feat, augmented_feat) + loss = criterion(logits, labels) + + total_loss += loss.item() + + preds = torch.argmax(logits, dim=1) + correct += (preds == labels).sum().item() + total += labels.size(0) + + # Per-class accuracy + for i in range(len(labels)): + label = labels[i].item() + class_total[label] += 1 + if preds[i] == labels[i]: + class_correct[label] += 1 + + accuracy = correct / total if total > 0 else 0 + avg_loss = total_loss / len(data_loader) + + class_accuracies = [ + class_correct[i] / class_total[i] if class_total[i] > 0 else 0 + for i in range(3) + ] + + return { + 'accuracy': accuracy, + 'loss': avg_loss, + 'correct': correct, + 'total': total, + 'class_accuracies': class_accuracies + } + + +def predict(model, original_feat, augmented_feat, device): + """ + Predict class for feature pair. + + Args: + model: LLMEnrichedClassifier + original_feat: Original features + augmented_feat: Augmented features + device: PyTorch device + + Returns: + Predicted class (0, 1, or 2) + """ + model.eval() + with torch.no_grad(): + original_feat = torch.FloatTensor(original_feat).unsqueeze(0).to(device) + augmented_feat = torch.FloatTensor(augmented_feat).unsqueeze(0).to(device) + logits = model(original_feat, augmented_feat) + pred = torch.argmax(logits, dim=1).item() + + return pred + + +def save_artifacts(model, llm_generator, history, metrics, output_dir=OUTPUT_DIR): + """ + Save model, features, and metrics. + + Args: + model: Trained model + llm_generator: LLM feature generator with caches + history: Training history + metrics: Evaluation metrics + output_dir: Output directory + """ + os.makedirs(output_dir, exist_ok=True) + + # Save model + model_path = os.path.join(output_dir, 'llm_classifier_model.pt') + torch.save(model.state_dict(), model_path) + print(f"[Artifacts] Saved model to {model_path}") + + # Save metrics + metrics_path = os.path.join(output_dir, 'metrics.json') + metrics_to_save = { + 'validation_accuracy': metrics['accuracy'], + 'validation_loss': metrics['loss'], + 'class_accuracies': metrics.get('class_accuracies', []), + 'training_history': { + 'loss': [float(l) for l in history['loss']], + 'accuracy': [float(a) for a in history['accuracy']] + } + } + with open(metrics_path, 'w') as f: + json.dump(metrics_to_save, f, indent=2) + print(f"[Artifacts] Saved metrics to {metrics_path}") + + # Save feature cache summary + cache_summary_path = os.path.join(output_dir, 'feature_cache_summary.json') + cache_summary = { + 'feature_cache_size': len(llm_generator.feature_cache), + 'augmentation_cache_size': len(llm_generator.augmentation_cache), + 'feature_dim': llm_generator.feature_dim + } + with open(cache_summary_path, 'w') as f: + json.dump(cache_summary, f, indent=2) + print(f"[Artifacts] Saved cache summary to {cache_summary_path}") + + +if __name__ == '__main__': + print("\n" + "="*80) + print("BigQuery Bigframe LLM - Text Classification Task") + print("="*80) + + try: + # Setup + device = get_device() + set_seed(42) + + metadata = get_task_metadata() + print(f"\n[Config] Task: {metadata['task_name']}") + print(f"[Config] Number of classes: {metadata['num_classes']}") + print(f"[Config] Feature dimension: {metadata['feature_dim']}") + print(f"[Config] Using BigQuery: {metadata['use_bigquery']}") + print(f"[Config] Using LLM: {metadata['use_llm']}") + print(f"[Config] Device: {device}") + + # Create dataloaders + print("\n[Data] Creating dataloaders with LLM augmentation...") + train_loader, val_loader, llm_generator = make_dataloaders(batch_size=16) + + # Build model + print("\n[Model] Building LLM-enriched classifier...") + model = build_model(feature_dim=512, num_classes=3, hidden_dim=256) + model = model.to(device) + + total_params = sum(p.numel() for p in model.parameters()) + print(f"[Model] Total parameters: {total_params:,}") + + # Train + print("\n[Training] Starting training with LLM-augmented features...") + history = train(model, train_loader, device, epochs=25, lr=0.001) + + # Evaluate on validation set + print("\n[Evaluation] Evaluating on validation set...") + val_metrics = evaluate(model, val_loader, device) + + print(f"\n[Results] Validation Accuracy: {val_metrics['accuracy']:.4f}") + print(f"[Results] Validation Loss: {val_metrics['loss']:.4f}") + print(f"[Results] Per-class accuracies: {[f'{acc:.4f}' for acc in val_metrics['class_accuracies']]}") + + # Evaluate on training set + train_metrics = evaluate(model, train_loader, device) + print(f"\n[Results] Training Accuracy: {train_metrics['accuracy']:.4f}") + print(f"[Results] Training Loss: {train_metrics['loss']:.4f}") + + # Quality assertions + print("\n[Assertions] Checking quality thresholds...") + + assert val_metrics['accuracy'] > 0.60, \ + f"Validation accuracy {val_metrics['accuracy']:.4f} must be > 0.60" + print("✓ Validation accuracy > 0.60") + + assert train_metrics['accuracy'] > 0.65, \ + f"Training accuracy {train_metrics['accuracy']:.4f} must be > 0.65" + print("✓ Training accuracy > 0.65") + + assert val_metrics['loss'] < 1.2, \ + f"Validation loss {val_metrics['loss']:.4f} must be < 1.2" + print("✓ Validation loss < 1.2") + + assert len(history['accuracy']) > 0, "Training history empty" + print("✓ Training history recorded") + + assert len(llm_generator.feature_cache) > 0, "Feature cache empty" + print(f"✓ Feature cache generated ({len(llm_generator.feature_cache)} features)") + + # Save artifacts + print("\n[Saving] Persisting model and artifacts...") + save_artifacts(model, llm_generator, history, val_metrics) + + print("\n" + "="*80) + print("SUCCESS: All assertions passed!") + print("="*80) + sys.exit(0) + + except AssertionError as e: + print(f"\n✗ ASSERTION FAILED: {e}") + print("="*80) + sys.exit(1) + + except Exception as e: + print(f"\n✗ ERROR: {e}") + import traceback + traceback.print_exc() + print("="*80) + sys.exit(1) diff --git a/MLtasks/tasks/robust_lvl1_adversarial_training/task.py b/MLtasks/tasks/robust_lvl1_adversarial_training/task.py new file mode 100644 index 0000000..3d1dc0a --- /dev/null +++ b/MLtasks/tasks/robust_lvl1_adversarial_training/task.py @@ -0,0 +1,700 @@ +""" +Adversarial Robustness: FGSM Attacks and Adversarial Training + +Mathematical Formulation: +- FGSM Perturbation: x_adv = x + eps * sign(∇_x J(θ, x, y)) + where J is the loss function and eps is the perturbation magnitude +- Adversarial Loss Mix: L_total = α*L_clean + (1-α)*L_adversarial + where L_clean is loss on clean samples and L_adversarial on attacked samples +- Robustness Metrics: clean accuracy, adversarial accuracy, robustness gap + +This task demonstrates: +1. Implementing FGSM attack from scratch +2. Training a baseline CNN without adversarial defense +3. Training with adversarial examples (adversarial training) +4. Comparing baseline vs robust model performance +5. Visualizing adversarial perturbations +""" + +import os +import sys +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, Dataset +import torchvision +import torchvision.transforms as transforms +import matplotlib.pyplot as plt +import json +from pathlib import Path + +# Set seeds for reproducibility +torch.manual_seed(42) +np.random.seed(42) + +# Output directory for artifacts +OUTPUT_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'output', 'robust_lvl1_adversarial_training') +os.makedirs(OUTPUT_DIR, exist_ok=True) + + +def get_task_metadata(): + """Return task metadata.""" + return { + 'task_name': 'adversarial_robustness_fgsm', + 'description': 'FGSM attacks and adversarial training for robustness', + 'attack_method': 'FGSM', + 'epsilon': 0.3, # 8/255 ≈ 0.03 normalized, but using 0.3 for MNIST visibility + 'task_type': 'adversarial_robustness', + 'dataset': 'MNIST' + } + + +def set_seed(seed=42): + """Set random seeds for reproducibility.""" + torch.manual_seed(seed) + np.random.seed(seed) + + +def get_device(): + """Get device (CPU or GPU).""" + return torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + +def make_dataloaders(batch_size=64, download=True): + """ + Create MNIST dataloaders for train and validation. + + Args: + batch_size: Batch size + download: Whether to download MNIST + + Returns: + train_loader, val_loader + """ + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ]) + + # Download and load MNIST + mnist_train = torchvision.datasets.MNIST( + root='./data', train=True, download=download, transform=transform + ) + mnist_test = torchvision.datasets.MNIST( + root='./data', train=False, download=download, transform=transform + ) + + # Split training into 80/20 train/val + n_train = int(0.8 * len(mnist_train)) + indices = torch.randperm(len(mnist_train)) + train_indices = indices[:n_train] + val_indices = indices[n_train:] + + train_subset = torch.utils.data.Subset(mnist_train, train_indices) + val_subset = torch.utils.data.Subset(mnist_train, val_indices) + + train_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True) + val_loader = DataLoader(val_subset, batch_size=batch_size, shuffle=False) + + return train_loader, val_loader + + +class SimpleCNN(nn.Module): + """Simple CNN for MNIST classification.""" + + def __init__(self): + super(SimpleCNN, self).__init__() + self.conv1 = nn.Conv2d(1, 32, kernel_size=5, padding=2) + self.conv2 = nn.Conv2d(32, 64, kernel_size=5, padding=2) + self.fc1 = nn.Linear(64 * 7 * 7, 128) + self.fc2 = nn.Linear(128, 10) + self.relu = nn.ReLU() + self.pool = nn.MaxPool2d(2, 2) + self.dropout = nn.Dropout(0.5) + + def forward(self, x): + # Input: (batch, 1, 28, 28) + x = self.pool(self.relu(self.conv1(x))) # -> (batch, 32, 14, 14) + x = self.pool(self.relu(self.conv2(x))) # -> (batch, 64, 7, 7) + x = x.view(x.size(0), -1) # -> (batch, 2816) + x = self.relu(self.fc1(x)) # -> (batch, 128) + x = self.dropout(x) + x = self.fc2(x) # -> (batch, 10) + return x + + +def build_model(**kwargs): + """Build CNN model.""" + return SimpleCNN() + + +class FGSM: + """Fast Gradient Sign Method (FGSM) attack.""" + + def __init__(self, model, epsilon=0.3, device='cpu'): + self.model = model + self.epsilon = epsilon + self.device = device + + def generate(self, x, y): + """ + Generate adversarial examples using FGSM. + + Args: + x: Input images (batch, channels, height, width) + y: True labels (batch,) + + Returns: + x_adv: Adversarial images (batch, channels, height, width) + perturbation: Perturbation (batch, channels, height, width) + """ + x = x.clone().detach().to(self.device) + y = y.clone().detach().to(self.device) + + # Compute gradients + x.requires_grad = True + + logits = self.model(x) + criterion = nn.CrossEntropyLoss() + loss = criterion(logits, y) + + # Compute gradient of loss w.r.t. x + grad = torch.autograd.grad(loss, x, create_graph=False)[0] + + # Apply FGSM: x_adv = x + eps * sign(grad) + perturbation = self.epsilon * torch.sign(grad) + x_adv = x + perturbation + + # Clip to valid image range + x_adv = torch.clamp(x_adv, -3, 3) # Clamp to valid normalized range + + return x_adv.detach(), perturbation.detach() + + +def train_standard(model, train_loader, val_loader, device, epochs=10, lr=0.001): + """ + Train model with standard cross-entropy loss (no adversarial training). + + Args: + model: CNN model + train_loader: Training data loader + val_loader: Validation data loader + device: Device + epochs: Number of epochs + lr: Learning rate + + Returns: + model, training history + """ + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=lr) + + history = { + 'train_loss': [], + 'train_acc': [], + 'val_loss': [], + 'val_acc': [] + } + + for epoch in range(epochs): + # Training + model.train() + train_loss = 0.0 + train_correct = 0 + train_total = 0 + + for images, labels in train_loader: + images, labels = images.to(device), labels.to(device) + + optimizer.zero_grad() + outputs = model(images) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + train_loss += loss.item() + _, predicted = torch.max(outputs.data, 1) + train_total += labels.size(0) + train_correct += (predicted == labels).sum().item() + + train_loss /= len(train_loader) + train_acc = train_correct / train_total + history['train_loss'].append(train_loss) + history['train_acc'].append(train_acc) + + # Validation + model.eval() + val_loss = 0.0 + val_correct = 0 + val_total = 0 + + with torch.no_grad(): + for images, labels in val_loader: + images, labels = images.to(device), labels.to(device) + outputs = model(images) + loss = criterion(outputs, labels) + + val_loss += loss.item() + _, predicted = torch.max(outputs.data, 1) + val_total += labels.size(0) + val_correct += (predicted == labels).sum().item() + + val_loss /= len(val_loader) + val_acc = val_correct / val_total + history['val_loss'].append(val_loss) + history['val_acc'].append(val_acc) + + if (epoch + 1) % 2 == 0: + print(f"Standard Epoch [{epoch+1}/{epochs}], Train Loss: {train_loss:.4f}, " + f"Train Acc: {train_acc:.4f}, Val Acc: {val_acc:.4f}") + + return model, history + + +def train_adversarial(model, train_loader, val_loader, device, attack_epsilon=0.3, + lambda_adv=0.5, epochs=10, lr=0.001): + """ + Train model with adversarial examples (adversarial training). + + Combines standard and adversarial loss: + L_total = lambda_adv * L_clean + (1 - lambda_adv) * L_adversarial + + Args: + model: CNN model + train_loader: Training data loader + val_loader: Validation data loader + device: Device + attack_epsilon: FGSM epsilon for perturbations + lambda_adv: Weight for adversarial loss in mix + epochs: Number of epochs + lr: Learning rate + + Returns: + model, training history + """ + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=lr) + attacker = FGSM(model, epsilon=attack_epsilon, device=device) + + history = { + 'train_loss': [], + 'train_acc': [], + 'val_loss': [], + 'val_acc': [] + } + + for epoch in range(epochs): + # Training + model.train() + train_loss = 0.0 + train_correct = 0 + train_total = 0 + + for images, labels in train_loader: + images, labels = images.to(device), labels.to(device) + + # Clean loss + optimizer.zero_grad() + outputs_clean = model(images) + loss_clean = criterion(outputs_clean, labels) + + # Adversarial loss (FGSM needs input gradients) + images_adv, _ = attacker.generate(images, labels) + + outputs_adv = model(images_adv) + loss_adv = criterion(outputs_adv, labels) + + # Combined loss + loss = lambda_adv * loss_clean + (1 - lambda_adv) * loss_adv + + loss.backward() + optimizer.step() + + train_loss += loss.item() + _, predicted = torch.max(outputs_clean.data, 1) + train_total += labels.size(0) + train_correct += (predicted == labels).sum().item() + + train_loss /= len(train_loader) + train_acc = train_correct / train_total + history['train_loss'].append(train_loss) + history['train_acc'].append(train_acc) + + # Validation + model.eval() + val_loss = 0.0 + val_correct = 0 + val_total = 0 + + with torch.no_grad(): + for images, labels in val_loader: + images, labels = images.to(device), labels.to(device) + outputs = model(images) + loss = criterion(outputs, labels) + + val_loss += loss.item() + _, predicted = torch.max(outputs.data, 1) + val_total += labels.size(0) + val_correct += (predicted == labels).sum().item() + + val_loss /= len(val_loader) + val_acc = val_correct / val_total + history['val_loss'].append(val_loss) + history['val_acc'].append(val_acc) + + if (epoch + 1) % 2 == 0: + print(f"Adversarial Epoch [{epoch+1}/{epochs}], Train Loss: {train_loss:.4f}, " + f"Train Acc: {train_acc:.4f}, Val Acc: {val_acc:.4f}") + + return model, history + + +def evaluate_robustness(model, data_loader, device, attack_epsilon=0.3): + """ + Evaluate model robustness against FGSM attacks. + + Args: + model: Model to evaluate + data_loader: Data loader + device: Device + attack_epsilon: FGSM epsilon + + Returns: + metrics dict with clean and adversarial accuracy + """ + model.eval() + attacker = FGSM(model, epsilon=attack_epsilon, device=device) + criterion = nn.CrossEntropyLoss() + + clean_correct = 0 + adv_correct = 0 + total = 0 + all_perturbations = [] + + for images, labels in data_loader: + images, labels = images.to(device), labels.to(device) + + # Clean accuracy + with torch.no_grad(): + outputs_clean = model(images) + _, pred_clean = torch.max(outputs_clean, 1) + clean_correct += (pred_clean == labels).sum().item() + + # Adversarial accuracy (FGSM needs input gradients) + images_adv, perturbation = attacker.generate(images, labels) + with torch.no_grad(): + outputs_adv = model(images_adv) + _, pred_adv = torch.max(outputs_adv, 1) + adv_correct += (pred_adv == labels).sum().item() + + total += labels.size(0) + all_perturbations.append(perturbation) + + clean_acc = clean_correct / total + adv_acc = adv_correct / total + robustness_gap = clean_acc - adv_acc + + return { + 'clean_accuracy': clean_acc, + 'adversarial_accuracy': adv_acc, + 'robustness_gap': robustness_gap + } + + +def train(train_loader, val_loader, device, **kwargs): + """Train wrapper for protocol.""" + model = build_model() + model.to(device) + model, history = train_standard(model, train_loader, val_loader, device, epochs=10, lr=0.001) + return model, history + + +def evaluate(model, data_loader, device, return_dict=True): + """Evaluate model on clean data.""" + model.eval() + criterion = nn.CrossEntropyLoss() + + correct = 0 + total = 0 + loss = 0.0 + + with torch.no_grad(): + for images, labels in data_loader: + images, labels = images.to(device), labels.to(device) + outputs = model(images) + batch_loss = criterion(outputs, labels) + + loss += batch_loss.item() + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + accuracy = correct / total + avg_loss = loss / len(data_loader) + + if not return_dict: + return accuracy + + return { + 'accuracy': accuracy, + 'ce_loss': avg_loss, + 'correct': correct, + 'total': total + } + + +def predict(model, x, device): + """Make predictions.""" + model.eval() + if isinstance(x, np.ndarray): + x = torch.FloatTensor(x) + x = x.to(device) + + with torch.no_grad(): + outputs = model(x) + _, predictions = torch.max(outputs, 1) + + return predictions.cpu().numpy() + + +def visualize_adversarial_examples(model, data_loader, device, attack_epsilon=0.3, + num_samples=5, output_dir=OUTPUT_DIR): + """ + Visualize adversarial examples and perturbations. + + Args: + model: Model + data_loader: Data loader + device: Device + attack_epsilon: FGSM epsilon + num_samples: Number of examples to visualize + output_dir: Output directory + """ + model.eval() + attacker = FGSM(model, epsilon=attack_epsilon, device=device) + + images_list = [] + labels_list = [] + + # Collect samples + count = 0 + with torch.no_grad(): + for images, labels in data_loader: + images_list.append(images) + labels_list.append(labels) + count += len(images) + if count >= num_samples: + break + + images = torch.cat(images_list)[:num_samples] + labels = torch.cat(labels_list)[:num_samples] + images = images.to(device) + labels = labels.to(device) + + # Generate adversarial examples + images_adv, perturbations = attacker.generate(images, labels) + + # Denormalize for visualization + mean = 0.1307 + std = 0.3081 + images_clean_viz = images * std + mean + images_adv_viz = images_adv * std + mean + perturbations_viz = perturbations * std + + # Create visualization + fig, axes = plt.subplots(num_samples, 3, figsize=(12, 3*num_samples)) + if num_samples == 1: + axes = axes.reshape(1, -1) + + for i in range(num_samples): + # Clean image + axes[i, 0].imshow(images_clean_viz[i, 0].cpu().numpy(), cmap='gray') + axes[i, 0].set_title(f'Clean') + axes[i, 0].axis('off') + + # Perturbation (amplified for visibility) + pert_viz = perturbations_viz[i, 0].cpu().numpy() + pert_abs_max = np.abs(pert_viz).max() + if pert_abs_max > 0: + pert_viz = pert_viz / pert_abs_max + axes[i, 1].imshow(pert_viz, cmap='RdBu', vmin=-1, vmax=1) + axes[i, 1].set_title(f'Perturbation (×{1/attack_epsilon:.1f})') + axes[i, 1].axis('off') + + # Adversarial image + axes[i, 2].imshow(images_adv_viz[i, 0].cpu().numpy(), cmap='gray') + axes[i, 2].set_title(f'Adversarial') + axes[i, 2].axis('off') + + plt.tight_layout() + save_path = os.path.join(output_dir, 'adversarial_examples.png') + plt.savefig(save_path, dpi=100, bbox_inches='tight') + print(f"Saved adversarial examples visualization to {save_path}") + plt.close() + + +def save_artifacts(baseline_model, robust_model, histories, metrics, output_dir=OUTPUT_DIR): + """Save models and metrics.""" + os.makedirs(output_dir, exist_ok=True) + + # Save models + baseline_path = os.path.join(output_dir, 'baseline_model.pt') + torch.save(baseline_model.state_dict(), baseline_path) + print(f"Saved baseline model to {baseline_path}") + + robust_path = os.path.join(output_dir, 'robust_model.pt') + torch.save(robust_model.state_dict(), robust_path) + print(f"Saved robust model to {robust_path}") + + # Save metrics + metrics_path = os.path.join(output_dir, 'metrics.json') + with open(metrics_path, 'w') as f: + json.dump(metrics, f, indent=2) + print(f"Saved metrics to {metrics_path}") + + # Save histories + history_path = os.path.join(output_dir, 'histories.json') + with open(history_path, 'w') as f: + json.dump(histories, f, indent=2) + print(f"Saved histories to {history_path}") + + +if __name__ == '__main__': + """ + Main pipeline: + 1. Load MNIST data + 2. Train baseline model without adversarial training + 3. Evaluate baseline on clean and adversarial examples + 4. Train robust model with adversarial training + 5. Evaluate robust model + 6. Compare robustness and visualize examples + 7. Assert quality thresholds + """ + + try: + device = get_device() + print(f"Using device: {device}") + + # Load data + print("\nLoading MNIST data...") + train_loader, val_loader = make_dataloaders(batch_size=64, download=True) + + # Train baseline model + print("\n" + "="*60) + print("Training Baseline Model (No Adversarial Training)") + print("="*60) + baseline_model = build_model() + baseline_model.to(device) + baseline_model, baseline_history = train_standard( + baseline_model, train_loader, val_loader, device, epochs=10, lr=0.001 + ) + + # Evaluate baseline + print("\nEvaluating Baseline Model...") + baseline_clean_metrics = evaluate(baseline_model, val_loader, device, return_dict=True) + baseline_robustness = evaluate_robustness(baseline_model, val_loader, device, attack_epsilon=0.3) + + print(f"Baseline - Clean Acc: {baseline_robustness['clean_accuracy']:.4f}, " + f"Adversarial Acc: {baseline_robustness['adversarial_accuracy']:.4f}, " + f"Gap: {baseline_robustness['robustness_gap']:.4f}") + + # Train robust model + print("\n" + "="*60) + print("Training Robust Model (With Adversarial Training)") + print("="*60) + robust_model = build_model() + robust_model.to(device) + robust_model, robust_history = train_adversarial( + robust_model, train_loader, val_loader, device, attack_epsilon=0.3, + lambda_adv=0.5, epochs=10, lr=0.001 + ) + + # Evaluate robust model + print("\nEvaluating Robust Model...") + robust_clean_metrics = evaluate(robust_model, val_loader, device, return_dict=True) + robust_robustness = evaluate_robustness(robust_model, val_loader, device, attack_epsilon=0.3) + + print(f"Robust - Clean Acc: {robust_robustness['clean_accuracy']:.4f}, " + f"Adversarial Acc: {robust_robustness['adversarial_accuracy']:.4f}, " + f"Gap: {robust_robustness['robustness_gap']:.4f}") + + # Compare improvements + print("\n" + "="*60) + print("Robustness Comparison") + print("="*60) + + adv_acc_improvement = robust_robustness['adversarial_accuracy'] - baseline_robustness['adversarial_accuracy'] + clean_acc_gap = baseline_robustness['clean_accuracy'] - robust_robustness['clean_accuracy'] + + print(f"\nAdversarial accuracy improvement: {adv_acc_improvement:.4f} " + f"({adv_acc_improvement / baseline_robustness['adversarial_accuracy'] * 100:.2f}% relative)") + print(f"Clean accuracy trade-off: {clean_acc_gap:.4f}") + + # Visualize adversarial examples + print("\nGenerating adversarial examples visualization...") + visualize_adversarial_examples(robust_model, val_loader, device, attack_epsilon=0.3, + num_samples=5, output_dir=OUTPUT_DIR) + + # Collect all metrics + all_metrics = { + 'baseline': { + 'clean_accuracy': baseline_robustness['clean_accuracy'], + 'adversarial_accuracy': baseline_robustness['adversarial_accuracy'], + 'robustness_gap': baseline_robustness['robustness_gap'] + }, + 'robust': { + 'clean_accuracy': robust_robustness['clean_accuracy'], + 'adversarial_accuracy': robust_robustness['adversarial_accuracy'], + 'robustness_gap': robust_robustness['robustness_gap'] + }, + 'improvements': { + 'adversarial_accuracy_gain': adv_acc_improvement, + 'clean_accuracy_trade_off': clean_acc_gap, + 'gap_reduction': baseline_robustness['robustness_gap'] - robust_robustness['robustness_gap'] + } + } + + all_histories = { + 'baseline': baseline_history, + 'robust': robust_history + } + + save_artifacts(baseline_model, robust_model, all_histories, all_metrics) + + # Assertions for quality + print("\n" + "="*60) + print("Quality Assertions") + print("="*60) + + assert robust_robustness['clean_accuracy'] > 0.95, \ + f"Robust model clean accuracy {robust_robustness['clean_accuracy']:.4f} must be > 0.95" + print("✓ Robust model maintains > 95% clean accuracy") + + assert robust_robustness['adversarial_accuracy'] > baseline_robustness['adversarial_accuracy'], \ + f"Robust model should improve adversarial accuracy" + print("✓ Robust model improves adversarial robustness") + + assert adv_acc_improvement > 0.02, \ + f"Adversarial accuracy improvement {adv_acc_improvement:.4f} should be > 0.02" + print(f"✓ Significant robustness improvement ({adv_acc_improvement:.4f})") + + assert clean_acc_gap < 0.05, \ + f"Clean accuracy trade-off {clean_acc_gap:.4f} should be < 0.05" + print(f"✓ Clean accuracy trade-off is acceptable ({clean_acc_gap:.4f})") + + print("\n" + "="*60) + print("SUCCESS: All assertions passed!") + print("="*60) + sys.exit(0) + + except Exception as e: + print(f"\nERROR: {str(e)}") + import traceback + traceback.print_exc() + sys.exit(1) + + +def evaluate_model(model, data_loader, device): + """Alias for evaluate function.""" + return evaluate(model, data_loader, device, return_dict=True) diff --git a/data/MNIST/raw/t10k-images-idx3-ubyte b/data/MNIST/raw/t10k-images-idx3-ubyte new file mode 100644 index 0000000..1170b2c Binary files /dev/null and b/data/MNIST/raw/t10k-images-idx3-ubyte differ diff --git a/data/MNIST/raw/t10k-images-idx3-ubyte.gz b/data/MNIST/raw/t10k-images-idx3-ubyte.gz new file mode 100644 index 0000000..5ace8ea Binary files /dev/null and b/data/MNIST/raw/t10k-images-idx3-ubyte.gz differ diff --git a/data/MNIST/raw/t10k-labels-idx1-ubyte b/data/MNIST/raw/t10k-labels-idx1-ubyte new file mode 100644 index 0000000..d1c3a97 Binary files /dev/null and b/data/MNIST/raw/t10k-labels-idx1-ubyte differ diff --git a/data/MNIST/raw/t10k-labels-idx1-ubyte.gz b/data/MNIST/raw/t10k-labels-idx1-ubyte.gz new file mode 100644 index 0000000..a7e1415 Binary files /dev/null and b/data/MNIST/raw/t10k-labels-idx1-ubyte.gz differ diff --git a/data/MNIST/raw/train-images-idx3-ubyte b/data/MNIST/raw/train-images-idx3-ubyte new file mode 100644 index 0000000..bbce276 Binary files /dev/null and b/data/MNIST/raw/train-images-idx3-ubyte differ diff --git a/data/MNIST/raw/train-images-idx3-ubyte.gz b/data/MNIST/raw/train-images-idx3-ubyte.gz new file mode 100644 index 0000000..b50e4b6 Binary files /dev/null and b/data/MNIST/raw/train-images-idx3-ubyte.gz differ diff --git a/data/MNIST/raw/train-labels-idx1-ubyte b/data/MNIST/raw/train-labels-idx1-ubyte new file mode 100644 index 0000000..d6b4c5d Binary files /dev/null and b/data/MNIST/raw/train-labels-idx1-ubyte differ diff --git a/data/MNIST/raw/train-labels-idx1-ubyte.gz b/data/MNIST/raw/train-labels-idx1-ubyte.gz new file mode 100644 index 0000000..707a576 Binary files /dev/null and b/data/MNIST/raw/train-labels-idx1-ubyte.gz differ diff --git a/fix_hash.py b/fix_hash.py new file mode 100644 index 0000000..0a6ab3c --- /dev/null +++ b/fix_hash.py @@ -0,0 +1,40 @@ +import hashlib + +# Read the file +with open('MLtasks/tasks/nlp_lvl2_bigframe_embeddings/task.py', 'r') as f: + content = f.read() + +# Replace hash() calls with deterministic hash +if 'import hashlib' not in content: + content = content.replace('import random', 'import random\nimport hashlib') + +# Add deterministic hash method after imports +if '_deterministic_hash' not in content: + # Find the EmbeddingGenerator class and add the method + class_start = content.find('class EmbeddingGenerator(nn.Module):') + init_start = content.find('def __init__(self', class_start) + indent_pos = init_start + + method_code = ''' + def _deterministic_hash(self, text: str, mod: int = 2**31) -> int: + """Create deterministic hash from text.""" + return int(hashlib.md5(text.encode()).hexdigest(), 16) % mod + + ''' + + # Insert before __init__ + content = content[:init_start] + method_code + content[init_start:] + +# Replace hash() calls with deterministic hash +content = content.replace('feature_idx = (hash(trigram) % self.embedding_dim)', + 'feature_idx = (self._deterministic_hash(trigram, self.embedding_dim))') +content = content.replace('feature_idx = (hash(word) % (self.embedding_dim // 2)) + (self.embedding_dim // 2)', + 'feature_idx = (self._deterministic_hash(word, self.embedding_dim // 2)) + (self.embedding_dim // 2)') +content = content.replace('np.random.seed(hash(text) % (2**31))', + 'np.random.seed(self._deterministic_hash(text, 2**31))') + +# Write back +with open('MLtasks/tasks/nlp_lvl2_bigframe_embeddings/task.py', 'w') as f: + f.write(content) + +print('Updated to use deterministic hashing') diff --git a/fix_hash2.py b/fix_hash2.py new file mode 100644 index 0000000..9fb347a --- /dev/null +++ b/fix_hash2.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# Fix non-deterministic hash calls in Task 1 + +with open('MLtasks/tasks/nlp_lvl2_bigframe_embeddings/task.py', 'r') as f: + lines = f.readlines() + +# Write back with hash fixes +with open('MLtasks/tasks/nlp_lvl2_bigframe_embeddings/task.py', 'w') as f: + for i, line in enumerate(lines): + # Replace self._deterministic_hash with simple ord-based hash + if 'feature_idx = (self._deterministic_hash(trigram,' in line: + line = ' feature_idx = (sum(ord(c) for c in trigram) * 7919) % self.embedding_dim\n' + elif 'feature_idx = (self._deterministic_hash(word,' in line: + line = ' feature_idx = ((sum(ord(c) for c in word) * 7919) % (self.embedding_dim // 2)) + (self.embedding_dim // 2)\n' + elif 'np.random.seed(self._deterministic_hash(text,' in line: + line = ' np.random.seed((sum(ord(c) for c in text) * 7919) % (2**31))\n' + # Remove broken _deterministic_hash method + elif 'def _deterministic_hash(self' in line: + # Skip this line and the next 3 lines + lines[i] = '# REMOVED: _deterministic_hash method\n' + if i + 3 < len(lines): + for j in range(1, 4): + lines[i + j] = '' + + f.write(line) + +print("Fixed hash functions in Task 1") diff --git a/output/cnn_lvl5_attention_mechanisms/attention_model.pt b/output/cnn_lvl5_attention_mechanisms/attention_model.pt new file mode 100644 index 0000000..ea18953 Binary files /dev/null and b/output/cnn_lvl5_attention_mechanisms/attention_model.pt differ diff --git a/output/cnn_lvl5_attention_mechanisms/baseline_model.pt b/output/cnn_lvl5_attention_mechanisms/baseline_model.pt new file mode 100644 index 0000000..f112fae Binary files /dev/null and b/output/cnn_lvl5_attention_mechanisms/baseline_model.pt differ diff --git a/output/cnn_lvl5_attention_mechanisms/histories.json b/output/cnn_lvl5_attention_mechanisms/histories.json new file mode 100644 index 0000000..0d1d393 --- /dev/null +++ b/output/cnn_lvl5_attention_mechanisms/histories.json @@ -0,0 +1,142 @@ +{ + "baseline": { + "train_loss": [ + 0.10944708847394213, + 0.04406837517116219, + 0.03607772962267821, + 0.02749645783076994, + 0.023638193830595507, + 0.020910559896020763, + 0.020035679228788163, + 0.01571369153111785, + 0.0154063645596907, + 0.013608231714819946, + 0.010893110613481744, + 0.010959611512552024, + 0.009622129402716988, + 0.006918792315219131, + 0.009771746546650925 + ], + "train_acc": [ + 0.9664583333333333, + 0.9857083333333333, + 0.9886875, + 0.9918958333333333, + 0.9929791666666666, + 0.9934791666666667, + 0.993375, + 0.994875, + 0.9950625, + 0.9955208333333333, + 0.9965833333333334, + 0.9966666666666667, + 0.9973541666666667, + 0.9979375, + 0.997125 + ], + "val_loss": [ + 0.05038964498399242, + 0.034991722021430256, + 0.039150330034007835, + 0.032434606269971675, + 0.0356802054972851, + 0.08517433609937299, + 0.04488459521002988, + 0.03670523029541073, + 0.032923617885821745, + 0.029801824872284922, + 0.03444839667931677, + 0.02829220462483639, + 0.030740520008313323, + 0.04093641194935543, + 0.04580788777904208 + ], + "val_acc": [ + 0.9845833333333334, + 0.9889166666666667, + 0.9875833333333334, + 0.9915, + 0.9906666666666667, + 0.97475, + 0.9875833333333334, + 0.9903333333333333, + 0.9915, + 0.9919166666666667, + 0.992, + 0.9929166666666667, + 0.9923333333333333, + 0.9896666666666667, + 0.9901666666666666 + ] + }, + "attention": { + "train_loss": [ + 0.11474037680712838, + 0.043710119268624115, + 0.033287675210429975, + 0.027907959828638317, + 0.021432925867964515, + 0.021019603672679903, + 0.014274312352460886, + 0.016296407099609497, + 0.012828751149931728, + 0.009895872821786422, + 0.010319322989786694, + 0.010615768888334666, + 0.008557220307314614, + 0.006922988358022243, + 0.005864194547411292 + ], + "train_acc": [ + 0.9652083333333333, + 0.9865416666666667, + 0.9897708333333334, + 0.9912708333333333, + 0.9932083333333334, + 0.9934166666666666, + 0.995875, + 0.9947916666666666, + 0.9960208333333334, + 0.9970625, + 0.9966041666666666, + 0.9964583333333333, + 0.9972916666666667, + 0.9978125, + 0.998375 + ], + "val_loss": [ + 0.07896503069477671, + 0.0929551854360591, + 0.03769002913712881, + 0.04154696216233362, + 0.03722109455323315, + 0.031313602028797294, + 0.03437889040647628, + 0.029758671455714106, + 0.03769907946379464, + 0.03040043964470084, + 0.02889214757262012, + 0.04723456082273457, + 0.031172871388797773, + 0.02941834801228961, + 0.03245923362172281 + ], + "val_acc": [ + 0.9775, + 0.9725833333333334, + 0.98875, + 0.9875833333333334, + 0.9891666666666666, + 0.9915833333333334, + 0.9905, + 0.99175, + 0.9893333333333333, + 0.9928333333333333, + 0.9935833333333334, + 0.9891666666666666, + 0.9919166666666667, + 0.9935, + 0.9926666666666667 + ] + } +} \ No newline at end of file diff --git a/output/cnn_lvl5_attention_mechanisms/metrics.json b/output/cnn_lvl5_attention_mechanisms/metrics.json new file mode 100644 index 0000000..2b5f09b --- /dev/null +++ b/output/cnn_lvl5_attention_mechanisms/metrics.json @@ -0,0 +1,37 @@ +{ + "baseline": { + "train": { + "loss": 0.010462590872205814, + "accuracy": 0.9967083333333333, + "correct": 47842, + "total": 48000 + }, + "val": { + "loss": 0.04580788777904208, + "accuracy": 0.9901666666666666, + "correct": 11882, + "total": 12000 + }, + "params": 11172810 + }, + "attention": { + "train": { + "loss": 0.0039065008122391495, + "accuracy": 0.9987916666666666, + "correct": 47942, + "total": 48000 + }, + "val": { + "loss": 0.03245923362172281, + "accuracy": 0.9926666666666667, + "correct": 11912, + "total": 12000 + }, + "params": 11261890 + }, + "comparison": { + "param_increase_percent": 0.7972927132923588, + "accuracy_gain": 0.0025000000000000577, + "efficiency_ratio": 3.1891708531693617 + } +} \ No newline at end of file diff --git a/output/linreg_lvl3_fit.png b/output/linreg_lvl3_fit.png new file mode 100644 index 0000000..c1d9ece Binary files /dev/null and b/output/linreg_lvl3_fit.png differ diff --git a/output/linreg_lvl3_history.json b/output/linreg_lvl3_history.json new file mode 100644 index 0000000..a322a03 --- /dev/null +++ b/output/linreg_lvl3_history.json @@ -0,0 +1,206 @@ +{ + "train_losses": [ + 2.782244725823402, + 0.7814754831790924, + 0.3234826326370239, + 0.26689534246921537, + 0.3006064921617508, + 0.25884998083114624, + 0.2602108043432236, + 0.2542596447467804, + 0.2533987349271774, + 0.2837154543399811, + 0.25970075726509095, + 0.25504467129707337, + 0.24783639818429948, + 0.2825534307956696, + 0.2901459294557571, + 0.26338808953762055, + 0.26114507019519806, + 0.2538106799125671, + 0.26013572096824644, + 0.247129362821579, + 0.2561055475473404, + 0.2619620478153229, + 0.24857771575450896, + 0.26032410860061644, + 0.30803109765052794, + 0.2606967431306839, + 0.252079696059227, + 0.2557601338624954, + 0.2490994155406952, + 0.25839682459831237, + 0.2647661817073822, + 0.2548650163412094, + 0.2463023406267166, + 0.25219542503356934, + 0.2534993642568588, + 0.25404793679714205, + 0.26291375279426576, + 0.2585968428850174, + 0.24777377605438233, + 0.2559710431098938, + 0.25835608303546903, + 0.24876139283180237, + 0.24630393266677855, + 0.25958851754665374, + 0.2611415255069733, + 0.24776701927185057, + 0.24697202146053315, + 0.2470162183046341, + 0.24964770853519438, + 0.2614587587118149, + 0.25185371160507203, + 0.2504545617103577, + 0.24271693348884582, + 0.26374206483364104, + 0.2678121888637543, + 0.2797450989484787, + 0.2729140639305115, + 0.25157166600227354, + 0.2543622475862503, + 0.2732303833961487, + 0.27231643795967103, + 0.25549046099185946, + 0.258145107626915, + 0.2545686489343643, + 0.2522685533761978, + 0.24925511598587036, + 0.24939465165138244, + 0.26009415209293363, + 0.25962310791015625, + 0.2490253460407257, + 0.2909778505563736, + 0.2773927301168442, + 0.3034765100479126, + 0.2667998814582825, + 0.26012414157390595, + 0.2466878429055214, + 0.25324820101261136, + 0.25777539134025573, + 0.2533584266901016, + 0.25134375274181364, + 0.25780499041080474, + 0.26197738528251646, + 0.25487955391407013, + 0.26637407422065734, + 0.25805389404296875, + 0.25007459700107576, + 0.2521152657270431, + 0.252663391828537, + 0.2531278479099274, + 0.26285885393619535, + 0.2588582581281662, + 0.2606839174032211, + 0.25072758793830874, + 0.252754944562912, + 0.2535406345129013, + 0.25422093749046326, + 0.2572420221567154, + 0.2632756894826889, + 0.2550387328863144, + 0.2521221458911896 + ], + "val_losses": [ + 2.6025374787194386, + 0.584973326751164, + 0.2526684935603823, + 0.24318827688694, + 0.22818391025066376, + 0.23487589827605657, + 0.24540992081165314, + 0.2361621856689453, + 0.24184484354087285, + 0.297220664364951, + 0.2279862697635378, + 0.2243365900857108, + 0.22268867705549514, + 0.36399505180971964, + 0.22802100437028067, + 0.2755655752761023, + 0.28203841192381723, + 0.2322091289928981, + 0.22560147089617594, + 0.2718021422624588, + 0.26348780734198435, + 0.24990418340478623, + 0.23025193171841757, + 0.2500296937567847, + 0.22748543747833797, + 0.22240871829645975, + 0.252707947577749, + 0.23490517692906515, + 0.2868259847164154, + 0.2357464794601713, + 0.24592024513653346, + 0.22248669820172445, + 0.23955045427594865, + 0.23030300438404083, + 0.29621736279555727, + 0.224794545343944, + 0.25429069783006397, + 0.23449548653193883, + 0.22811431969915116, + 0.29303258444581715, + 0.2481350941317422, + 0.22886000786508834, + 0.24048281141689845, + 0.24219935919557298, + 0.2420959153345653, + 0.24650209929261888, + 0.2276011279651097, + 0.2339490737233843, + 0.23711047853742326, + 0.2768063502652304, + 0.23361457884311676, + 0.2437661247594016, + 0.23241619978632247, + 0.22933210858276912, + 0.23631148040294647, + 0.2630397102662495, + 0.230550046477999, + 0.23318930396011897, + 0.2280324399471283, + 0.30969364302498953, + 0.26002174402986256, + 0.23405546375683375, + 0.23069471546581813, + 0.3398279824427196, + 0.2275352797337941, + 0.22281217575073242, + 0.22737286346299307, + 0.26636134726660593, + 0.22817839894975936, + 0.2525726684502193, + 0.23625945619174413, + 0.41123523882457186, + 0.28980412227766855, + 0.22274160598005568, + 0.23652855839048112, + 0.23127549248082296, + 0.22906516066619328, + 0.2359237947634288, + 0.23774940839835576, + 0.23148191826684134, + 0.2505004129239491, + 0.22836399929864065, + 0.2576232190643038, + 0.29791433470589773, + 0.2275526842900685, + 0.22624462417193822, + 0.2275578954390117, + 0.23494348568575724, + 0.23735398266996657, + 0.27302949130535126, + 0.22899719434125082, + 0.2513778401272638, + 0.23795249419552938, + 0.2460523886340005, + 0.22743771970272064, + 0.2302820874111993, + 0.2535078525543213, + 0.23169354668685369, + 0.2435615552323205, + 0.26732408148901804 + ] +} \ No newline at end of file diff --git a/output/linreg_lvl3_model.pth b/output/linreg_lvl3_model.pth new file mode 100644 index 0000000..a23ab9d Binary files /dev/null and b/output/linreg_lvl3_model.pth differ diff --git a/output/logreg_lvl2_multiclass_boundary.png b/output/logreg_lvl2_multiclass_boundary.png new file mode 100644 index 0000000..16adf71 Binary files /dev/null and b/output/logreg_lvl2_multiclass_boundary.png differ diff --git a/output/logreg_lvl2_multiclass_metrics.json b/output/logreg_lvl2_multiclass_metrics.json new file mode 100644 index 0000000..c45984e --- /dev/null +++ b/output/logreg_lvl2_multiclass_metrics.json @@ -0,0 +1,19 @@ +{ + "train": { + "loss": 0.0006037681174348109, + "accuracy": 1.0, + "f1_macro": 1.0 + }, + "validation": { + "loss": 0.001843012892225358, + "accuracy": 1.0, + "f1_macro": 1.0 + }, + "metadata": { + "task_name": "softmax_regression_multiclass", + "task_type": "classification", + "num_classes": 3, + "input_dim": 2, + "description": "Multiclass softmax regression using torch.nn.Module + CrossEntropyLoss" + } +} \ No newline at end of file diff --git a/output/logreg_lvl2_multiclass_model.pt b/output/logreg_lvl2_multiclass_model.pt new file mode 100644 index 0000000..542f1fb Binary files /dev/null and b/output/logreg_lvl2_multiclass_model.pt differ diff --git a/output/mlp_lvl5_ensemble_distillation/histories.json b/output/mlp_lvl5_ensemble_distillation/histories.json new file mode 100644 index 0000000..d4d51e6 --- /dev/null +++ b/output/mlp_lvl5_ensemble_distillation/histories.json @@ -0,0 +1,244 @@ +{ + "teachers": { + "teacher_0": { + "train_loss": [ + 0.30643721254666645, + 0.1405091902439793, + 0.10515355076827108, + 0.08824015836045146, + 0.07414756383995215, + 0.06580860247183591, + 0.058448180354665966, + 0.05475985555490479, + 0.05008242286074286, + 0.04809125070335964 + ], + "train_acc": [ + 0.9074166666666666, + 0.9570833333333333, + 0.967625, + 0.9724375, + 0.9764791666666667, + 0.9786458333333333, + 0.9815625, + 0.9821458333333334, + 0.9836041666666666, + 0.9840416666666667 + ], + "val_loss": [ + 0.13994414119565107, + 0.10683212701251056, + 0.09938871223935263, + 0.0926622179748015, + 0.08298208773572077, + 0.0877118494517685, + 0.08438536849311315, + 0.08569249532561987, + 0.09360808002585397, + 0.10132567092087096 + ], + "val_acc": [ + 0.9574166666666667, + 0.96875, + 0.9721666666666666, + 0.973, + 0.9749166666666667, + 0.9751666666666666, + 0.9763333333333334, + 0.976, + 0.9764166666666667, + 0.97325 + ] + }, + "teacher_1": { + "train_loss": [ + 0.3046378974169493, + 0.14114437504361074, + 0.10233240941104789, + 0.09046705976314842, + 0.07618218874558806, + 0.06885411054889361, + 0.06270504728782302, + 0.05491849934216589, + 0.05240748234558851, + 0.04797875055973418 + ], + "train_acc": [ + 0.9078541666666666, + 0.9570208333333333, + 0.9680833333333333, + 0.97225, + 0.9762708333333333, + 0.9780625, + 0.9798333333333333, + 0.982125, + 0.9831041666666667, + 0.9845625 + ], + "val_loss": [ + 0.15195544686247694, + 0.10437384403825915, + 0.09219382327307571, + 0.0991917492901074, + 0.09593994600916321, + 0.09233225922179507, + 0.07997044591243042, + 0.08984628368411808, + 0.08863662358155434, + 0.08584917902678965 + ], + "val_acc": [ + 0.95275, + 0.968, + 0.9728333333333333, + 0.9714166666666667, + 0.9721666666666666, + 0.97375, + 0.9766666666666667, + 0.9748333333333333, + 0.9765833333333334, + 0.9791666666666666 + ] + }, + "teacher_2": { + "train_loss": [ + 0.3040618342198432, + 0.14062167304381729, + 0.10804808153460423, + 0.09061471881655356, + 0.07547720752625416, + 0.06910906935265909, + 0.06278906111853819, + 0.05721899927717944, + 0.05213819264725316, + 0.05118575161548021 + ], + "train_acc": [ + 0.9089583333333333, + 0.957375, + 0.9667083333333333, + 0.97125, + 0.9757291666666666, + 0.978, + 0.9805416666666666, + 0.9815416666666666, + 0.9826875, + 0.983625 + ], + "val_loss": [ + 0.14865453743395654, + 0.11027546420178198, + 0.10140314670477776, + 0.09525124077744623, + 0.09330859005778155, + 0.09557575379508211, + 0.08263755143877674, + 0.09405867101753408, + 0.08478734690660086, + 0.0946680541472891 + ], + "val_acc": [ + 0.9550833333333333, + 0.9669166666666666, + 0.9679166666666666, + 0.97275, + 0.97225, + 0.9729166666666667, + 0.9761666666666666, + 0.97475, + 0.9776666666666667, + 0.9749166666666667 + ] + } + }, + "distillation": { + "train_loss": [ + 0.38715402569373447, + 0.17272705280284087, + 0.13112674575050673, + 0.10766219371557235, + 0.0930194324478507, + 0.08596505692104499, + 0.07976283518970012, + 0.07276120748867591, + 0.06973133137822152, + 0.06637857400874297, + 0.06484495962659519, + 0.0631252696911494, + 0.059794749781489374, + 0.05754093719770511, + 0.05805832609285911, + 0.05514080911378066, + 0.05410984020431837, + 0.05207223269715905, + 0.05545447801053524, + 0.05139500498771667 + ], + "train_acc": [ + 0.8789791666666666, + 0.944375, + 0.9570625, + 0.9640833333333333, + 0.9693125, + 0.9715833333333334, + 0.9738125, + 0.9769375, + 0.9780416666666667, + 0.97975, + 0.9801666666666666, + 0.9814583333333333, + 0.9822916666666667, + 0.98325, + 0.9832291666666667, + 0.9841041666666667, + 0.984375, + 0.9850416666666667, + 0.9842291666666667, + 0.985875 + ], + "val_loss": [ + 0.5633522603224884, + 0.3325822701023773, + 0.2657974609421527, + 0.19846518908409364, + 0.1516602603703777, + 0.13386096125666766, + 0.11674324512243905, + 0.1072641553659428, + 0.09979860836075262, + 0.10095378895478442, + 0.10742011778997852, + 0.08423146581217805, + 0.08085440320428461, + 0.08538507920085511, + 0.06631870083312721, + 0.0753459043161785, + 0.0672402447783091, + 0.06867082633494222, + 0.061718691970182066, + 0.0772400183969752 + ], + "val_acc": [ + 0.9519166666666666, + 0.9697083333333333, + 0.9746666666666667, + 0.9816666666666667, + 0.9848333333333333, + 0.9870416666666667, + 0.9886458333333333, + 0.9898333333333333, + 0.9908541666666667, + 0.9908333333333333, + 0.9908333333333333, + 0.9931666666666666, + 0.9940833333333333, + 0.9932291666666667, + 0.9953958333333334, + 0.9943125, + 0.9957083333333333, + 0.9953541666666667, + 0.9957708333333334, + 0.9948958333333333 + ] + } +} \ No newline at end of file diff --git a/output/mlp_lvl5_ensemble_distillation/metrics.json b/output/mlp_lvl5_ensemble_distillation/metrics.json new file mode 100644 index 0000000..7eecaf4 --- /dev/null +++ b/output/mlp_lvl5_ensemble_distillation/metrics.json @@ -0,0 +1,32 @@ +{ + "teachers": { + "teacher_0": { + "accuracy": 0.97325, + "ce_loss": 0.10132567092087096, + "correct": 11679, + "total": 12000 + }, + "teacher_1": { + "accuracy": 0.9791666666666666, + "ce_loss": 0.08584917902678965, + "correct": 11750, + "total": 12000 + }, + "teacher_2": { + "accuracy": 0.9749166666666667, + "ce_loss": 0.0946680541472891, + "correct": 11699, + "total": 12000 + } + }, + "ensemble_accuracy": 0.9813333333333333, + "student": { + "accuracy": 0.97525, + "ce_loss": 0.09860549226992744, + "correct": 11703, + "total": 12000, + "ensemble_accuracy": 0.9813333333333333, + "accuracy_ratio": 0.9938009510869565 + }, + "compression_ratio": 0.1550611109693552 +} \ No newline at end of file diff --git a/output/mlp_lvl5_ensemble_distillation/student.pt b/output/mlp_lvl5_ensemble_distillation/student.pt new file mode 100644 index 0000000..fbf3c78 Binary files /dev/null and b/output/mlp_lvl5_ensemble_distillation/student.pt differ diff --git a/output/mlp_lvl5_ensemble_distillation/teacher_0.pt b/output/mlp_lvl5_ensemble_distillation/teacher_0.pt new file mode 100644 index 0000000..7c4015e Binary files /dev/null and b/output/mlp_lvl5_ensemble_distillation/teacher_0.pt differ diff --git a/output/mlp_lvl5_ensemble_distillation/teacher_1.pt b/output/mlp_lvl5_ensemble_distillation/teacher_1.pt new file mode 100644 index 0000000..5129901 Binary files /dev/null and b/output/mlp_lvl5_ensemble_distillation/teacher_1.pt differ diff --git a/output/mlp_lvl5_ensemble_distillation/teacher_2.pt b/output/mlp_lvl5_ensemble_distillation/teacher_2.pt new file mode 100644 index 0000000..213332b Binary files /dev/null and b/output/mlp_lvl5_ensemble_distillation/teacher_2.pt differ diff --git a/output/nlp_lvl1_text_embedding/embedding_model.pt b/output/nlp_lvl1_text_embedding/embedding_model.pt new file mode 100644 index 0000000..140f6a3 Binary files /dev/null and b/output/nlp_lvl1_text_embedding/embedding_model.pt differ diff --git a/output/nlp_lvl1_text_embedding/embeddings.pt b/output/nlp_lvl1_text_embedding/embeddings.pt new file mode 100644 index 0000000..8927845 Binary files /dev/null and b/output/nlp_lvl1_text_embedding/embeddings.pt differ diff --git a/output/nlp_lvl1_text_embedding/metrics.json b/output/nlp_lvl1_text_embedding/metrics.json new file mode 100644 index 0000000..bcd58b9 --- /dev/null +++ b/output/nlp_lvl1_text_embedding/metrics.json @@ -0,0 +1,7 @@ +{ + "embedding_training_loss": 2.9972058225561073, + "avg_word_pair_similarity": 0.3477613503734271, + "embedding_norm": 19.597349166870117, + "sentiment_accuracy": 0.8500000238418579, + "vocab_size": 45 +} \ No newline at end of file diff --git a/output/nlp_lvl1_text_embedding/vocab.json b/output/nlp_lvl1_text_embedding/vocab.json new file mode 100644 index 0000000..9366b5d --- /dev/null +++ b/output/nlp_lvl1_text_embedding/vocab.json @@ -0,0 +1,96 @@ +{ + "word2id": { + "": 0, + "": 1, + "this": 2, + "and": 3, + "film": 4, + "movie": 5, + "i": 6, + "performance": 7, + "actors": 8, + "terrible": 9, + "is": 10, + "great": 11, + "amazing": 12, + "loved": 13, + "very": 14, + "much": 15, + "by": 16, + "direction": 17, + "cinematography": 18, + "absolutely": 19, + "experience": 20, + "have": 21, + "seen": 22, + "year": 23, + "story": 24, + "every": 25, + "minute": 26, + "of": 27, + "plot": 28, + "ending": 29, + "throughout": 30, + "boring": 31, + "hated": 32, + "awful": 33, + "fantastic": 34, + "excellent": 35, + "wonderful": 36, + "best": 37, + "incredible": 38, + "perfect": 39, + "outstanding": 40, + "horrible": 41, + "worst": 42, + "bad": 43, + "disappointing": 44 + }, + "id2word": { + "0": "", + "1": "", + "2": "this", + "3": "and", + "4": "film", + "5": "movie", + "6": "i", + "7": "performance", + "8": "actors", + "9": "terrible", + "10": "is", + "11": "great", + "12": "amazing", + "13": "loved", + "14": "very", + "15": "much", + "16": "by", + "17": "direction", + "18": "cinematography", + "19": "absolutely", + "20": "experience", + "21": "have", + "22": "seen", + "23": "year", + "24": "story", + "25": "every", + "26": "minute", + "27": "of", + "28": "plot", + "29": "ending", + "30": "throughout", + "31": "boring", + "32": "hated", + "33": "awful", + "34": "fantastic", + "35": "excellent", + "36": "wonderful", + "37": "best", + "38": "incredible", + "39": "perfect", + "40": "outstanding", + "41": "horrible", + "42": "worst", + "43": "bad", + "44": "disappointing" + } +} \ No newline at end of file diff --git a/output/nlp_lvl2_bigframe_embeddings/embedding_cache_summary.json b/output/nlp_lvl2_bigframe_embeddings/embedding_cache_summary.json new file mode 100644 index 0000000..81065f6 --- /dev/null +++ b/output/nlp_lvl2_bigframe_embeddings/embedding_cache_summary.json @@ -0,0 +1,4 @@ +{ + "cached_texts": 199, + "embedding_dim": 768 +} \ No newline at end of file diff --git a/output/nlp_lvl2_bigframe_embeddings/metrics.json b/output/nlp_lvl2_bigframe_embeddings/metrics.json new file mode 100644 index 0000000..adae845 --- /dev/null +++ b/output/nlp_lvl2_bigframe_embeddings/metrics.json @@ -0,0 +1,50 @@ +{ + "validation_accuracy": 0.8, + "validation_loss": 0.72280053794384, + "training_history": { + "loss": [ + 0.6954035878181457, + 0.687330961227417, + 0.676107656955719, + 0.6565568685531616, + 0.5996217489242553, + 0.4803223431110382, + 0.313069549202919, + 0.12548963576555253, + 0.029864443466067314, + 0.007506624702364207, + 0.003791995969368145, + 0.006337589502800256, + 0.004684542290488025, + 0.0005286922940285876, + 0.0007058505389068159, + 0.0014939721582777564, + 0.001253157469909638, + 0.00014555209199897944, + 0.0001912849062591704, + 0.0010467915060871746 + ], + "accuracy": [ + 0.4625, + 0.5375, + 0.625, + 0.6125, + 0.7125, + 0.9125, + 0.975, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ] + } +} \ No newline at end of file diff --git a/output/nlp_lvl2_bigframe_embeddings/semantic_similarity_model.pt b/output/nlp_lvl2_bigframe_embeddings/semantic_similarity_model.pt new file mode 100644 index 0000000..4fa9127 Binary files /dev/null and b/output/nlp_lvl2_bigframe_embeddings/semantic_similarity_model.pt differ diff --git a/output/nlp_lvl3_bigframe_llm_classification/feature_cache_summary.json b/output/nlp_lvl3_bigframe_llm_classification/feature_cache_summary.json new file mode 100644 index 0000000..b7b7c14 --- /dev/null +++ b/output/nlp_lvl3_bigframe_llm_classification/feature_cache_summary.json @@ -0,0 +1,5 @@ +{ + "feature_cache_size": 128, + "augmentation_cache_size": 38, + "feature_dim": 512 +} \ No newline at end of file diff --git a/output/nlp_lvl3_bigframe_llm_classification/llm_classifier_model.pt b/output/nlp_lvl3_bigframe_llm_classification/llm_classifier_model.pt new file mode 100644 index 0000000..2e45008 Binary files /dev/null and b/output/nlp_lvl3_bigframe_llm_classification/llm_classifier_model.pt differ diff --git a/output/nlp_lvl3_bigframe_llm_classification/metrics.json b/output/nlp_lvl3_bigframe_llm_classification/metrics.json new file mode 100644 index 0000000..271d784 --- /dev/null +++ b/output/nlp_lvl3_bigframe_llm_classification/metrics.json @@ -0,0 +1,65 @@ +{ + "validation_accuracy": 1.0, + "validation_loss": 0.01234927971381694, + "class_accuracies": [ + 1.0, + 1.0, + 1.0 + ], + "training_history": { + "loss": [ + 0.9518333435058594, + 0.5567063212394714, + 0.4196665287017822, + 0.30208355486392974, + 0.24634159505367278, + 0.2555822253227234, + 0.18381600305438042, + 0.35089140981435774, + 0.41420138329267503, + 0.31168401464819906, + 0.15566000007092953, + 0.244389671087265, + 0.1348598152399063, + 0.1620915826410055, + 0.12848771288990973, + 0.12583621516823768, + 0.16192929968237876, + 0.20077648274600507, + 0.09774699062108994, + 0.133341483771801, + 0.1464177794754505, + 0.15628004670143128, + 0.09522949419915676, + 0.17320457063615322, + 0.13843653295189143 + ], + "accuracy": [ + 0.5555555555555556, + 0.9027777777777778, + 0.9166666666666666, + 0.9166666666666666, + 0.9166666666666666, + 0.9305555555555556, + 0.9444444444444444, + 0.8888888888888888, + 0.9166666666666666, + 0.9444444444444444, + 0.9583333333333334, + 0.9027777777777778, + 0.9583333333333334, + 0.9444444444444444, + 0.9722222222222222, + 0.9583333333333334, + 0.9444444444444444, + 0.9722222222222222, + 0.9861111111111112, + 0.9583333333333334, + 0.9444444444444444, + 0.9444444444444444, + 0.9583333333333334, + 0.9583333333333334, + 0.9722222222222222 + ] + } +} \ No newline at end of file diff --git a/output/robust_lvl1_adversarial_training/adversarial_examples.png b/output/robust_lvl1_adversarial_training/adversarial_examples.png new file mode 100644 index 0000000..3c4c815 Binary files /dev/null and b/output/robust_lvl1_adversarial_training/adversarial_examples.png differ diff --git a/output/robust_lvl1_adversarial_training/baseline_model.pt b/output/robust_lvl1_adversarial_training/baseline_model.pt new file mode 100644 index 0000000..0dfdb54 Binary files /dev/null and b/output/robust_lvl1_adversarial_training/baseline_model.pt differ diff --git a/output/robust_lvl1_adversarial_training/histories.json b/output/robust_lvl1_adversarial_training/histories.json new file mode 100644 index 0000000..c9029ad --- /dev/null +++ b/output/robust_lvl1_adversarial_training/histories.json @@ -0,0 +1,102 @@ +{ + "baseline": { + "train_loss": [ + 0.21351166077082356, + 0.07777359422047933, + 0.05770759176866462, + 0.042820376604329795, + 0.040344835867872464, + 0.0288427608764614, + 0.02874447090411559, + 0.024721622006696028, + 0.02325397492015327, + 0.020041238437554662 + ], + "train_acc": [ + 0.9358333333333333, + 0.9781666666666666, + 0.983, + 0.9870625, + 0.9882291666666667, + 0.9910416666666667, + 0.991375, + 0.9921458333333333, + 0.993, + 0.9940833333333333 + ], + "val_loss": [ + 0.07000401534729618, + 0.04691888943767829, + 0.04072503822278529, + 0.03820486335500894, + 0.035187122512186454, + 0.03445676990457364, + 0.03504796417883939, + 0.0359689194490571, + 0.033156672717713595, + 0.0369511972475111 + ], + "val_acc": [ + 0.979, + 0.9866666666666667, + 0.9883333333333333, + 0.9880833333333333, + 0.99, + 0.9910833333333333, + 0.9914166666666666, + 0.9913333333333333, + 0.9914166666666666, + 0.99175 + ] + }, + "robust": { + "train_loss": [ + 0.3505865542590618, + 0.1578897270311912, + 0.12151816497246425, + 0.10370911592245102, + 0.09282413524885973, + 0.08192929725597302, + 0.07515056945569813, + 0.06845140532590449, + 0.06429263439153632, + 0.05806334437119464 + ], + "train_acc": [ + 0.935125, + 0.9798958333333333, + 0.9854375, + 0.987875, + 0.989625, + 0.9914375, + 0.9922291666666667, + 0.9929791666666666, + 0.9940208333333334, + 0.9941041666666667 + ], + "val_loss": [ + 0.0545171835073369, + 0.043468562521701774, + 0.033864628809982256, + 0.0336176204227547, + 0.033112780223281665, + 0.030363963315934637, + 0.028758808036671208, + 0.031411080578576965, + 0.029561343373329014, + 0.031159754610869523 + ], + "val_acc": [ + 0.9834166666666667, + 0.9870833333333333, + 0.991, + 0.991, + 0.9900833333333333, + 0.9911666666666666, + 0.9920833333333333, + 0.9924166666666666, + 0.9931666666666666, + 0.993 + ] + } +} \ No newline at end of file diff --git a/output/robust_lvl1_adversarial_training/metrics.json b/output/robust_lvl1_adversarial_training/metrics.json new file mode 100644 index 0000000..e71fe7f --- /dev/null +++ b/output/robust_lvl1_adversarial_training/metrics.json @@ -0,0 +1,17 @@ +{ + "baseline": { + "clean_accuracy": 0.99175, + "adversarial_accuracy": 0.7293333333333333, + "robustness_gap": 0.26241666666666674 + }, + "robust": { + "clean_accuracy": 0.993, + "adversarial_accuracy": 0.9555, + "robustness_gap": 0.03749999999999998 + }, + "improvements": { + "adversarial_accuracy_gain": 0.22616666666666674, + "clean_accuracy_trade_off": -0.0012499999999999734, + "gap_reduction": 0.22491666666666676 + } +} \ No newline at end of file diff --git a/output/robust_lvl1_adversarial_training/robust_model.pt b/output/robust_lvl1_adversarial_training/robust_model.pt new file mode 100644 index 0000000..046bc6b Binary files /dev/null and b/output/robust_lvl1_adversarial_training/robust_model.pt differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0898d3b --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +-r MLtasks/requirements.txt \ No newline at end of file diff --git a/run2_task1.txt b/run2_task1.txt new file mode 100644 index 0000000..f4ee5c3 Binary files /dev/null and b/run2_task1.txt differ diff --git a/run2_task2.txt b/run2_task2.txt new file mode 100644 index 0000000..02c25c3 Binary files /dev/null and b/run2_task2.txt differ diff --git a/temp_task1.txt b/temp_task1.txt new file mode 100644 index 0000000..6022752 Binary files /dev/null and b/temp_task1.txt differ diff --git a/verify_new_tasks.py b/verify_new_tasks.py new file mode 100644 index 0000000..2f48150 --- /dev/null +++ b/verify_new_tasks.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +""" +Verification script for new CoderGym neural network tasks. + +This script verifies that all 4 new tasks are properly set up and follow +the pytorch_task_v1 protocol. +""" + +import json +import os +import re +import sys +from pathlib import Path + + +def check_json_validity(json_path): + """Verify ml_tasks.json is valid JSON and contains new tasks.""" + print("\n" + "="*70) + print("1. Checking ml_tasks.json") + print("="*70) + + try: + with open(json_path, 'r') as f: + data = json.load(f) + print("✓ JSON is valid") + except json.JSONDecodeError as e: + print(f"✗ JSON is invalid: {e}") + return False + + # Check for new tasks + required_tasks = [ + 'mlp_lvl5_ensemble_distillation', + 'cnn_lvl5_attention_mechanisms', + 'nlp_lvl1_text_embedding', + 'robust_lvl1_adversarial_training' + ] + + existing_tasks = {task['id'] for task in data.get('tasks', [])} + + all_present = True + for task_id in required_tasks: + if task_id in existing_tasks: + print(f"✓ {task_id} found in JSON") + else: + print(f"✗ {task_id} NOT found in JSON") + all_present = False + + print(f"\nTotal tasks in JSON: {len(existing_tasks)}") + return all_present + + +def check_task_files(base_path): + """Verify that all task.py files exist and are substantial.""" + print("\n" + "="*70) + print("2. Checking task.py files") + print("="*70) + + tasks = [ + 'mlp_lvl5_ensemble_distillation', + 'cnn_lvl5_attention_mechanisms', + 'nlp_lvl1_text_embedding', + 'robust_lvl1_adversarial_training' + ] + + all_exist = True + for task in tasks: + task_path = os.path.join(base_path, task, 'task.py') + + if os.path.exists(task_path): + size_kb = os.path.getsize(task_path) / 1024 + lines = len(open(task_path).readlines()) + print(f"✓ {task}") + print(f" Size: {size_kb:.1f} KB ({lines} lines)") + + if size_kb < 10: + print(f" ⚠ Warning: File size seems small") + else: + print(f"✗ {task} - task.py NOT FOUND") + all_exist = False + + return all_exist + + +def check_protocol_compliance(base_path): + """Verify pytorch_task_v1 protocol compliance.""" + print("\n" + "="*70) + print("3. Checking pytorch_task_v1 Protocol Compliance") + print("="*70) + + tasks = [ + 'mlp_lvl5_ensemble_distillation', + 'cnn_lvl5_attention_mechanisms', + 'nlp_lvl1_text_embedding', + 'robust_lvl1_adversarial_training' + ] + + required_functions = [ + 'get_task_metadata', + 'set_seed', + 'get_device', + 'make_dataloaders', + 'build_model', + 'train', + 'evaluate', + 'predict', + 'save_artifacts' + ] + + all_compliant = True + + for task in tasks: + task_path = os.path.join(base_path, task, 'task.py') + + if not os.path.exists(task_path): + print(f"✗ {task}: task.py not found") + all_compliant = False + continue + + with open(task_path, 'r') as f: + content = f.read() + + print(f"\n{task}:") + task_compliant = True + + # Check required functions + for func in required_functions: + pattern = rf'^def {func}\(' + if re.search(pattern, content, re.MULTILINE): + print(f" ✓ {func}") + else: + print(f" ✗ {func} MISSING") + task_compliant = False + + # Check main block + if 'if __name__ == ' in content and '__main__' in content: + print(f" ✓ __main__ block") + else: + print(f" ✗ __main__ block MISSING") + task_compliant = False + + # Check for sys.exit + if 'sys.exit' in content: + print(f" ✓ sys.exit calls") + else: + print(f" ⚠ No sys.exit calls found") + + if task_compliant: + print(f" STATUS: ✓ COMPLIANT") + else: + print(f" STATUS: ✗ NEEDS REVIEW") + all_compliant = False + + return all_compliant + + +def check_documentation(base_path): + """Check for documentation files.""" + print("\n" + "="*70) + print("4. Checking Documentation") + print("="*70) + + docs = [ + ('NEW_TASKS_SUMMARY.md', 'Task documentation'), + ('GITHUB_SUBMISSION_GUIDE.md', 'Submission guide'), + ] + + all_exist = True + for doc_name, description in docs: + doc_path = os.path.join(base_path, doc_name) + if os.path.exists(doc_path): + size_kb = os.path.getsize(doc_path) / 1024 + print(f"✓ {doc_name} ({size_kb:.1f} KB)") + print(f" - {description}") + else: + print(f"✗ {doc_name} NOT FOUND") + all_exist = False + + return all_exist + + +def check_imports(base_path): + """Try importing key functions from tasks (without execution).""" + print("\n" + "="*70) + print("5. Checking Imports") + print("="*70) + + task_imports = { + 'mlp_lvl5_ensemble_distillation.task': ['get_task_metadata', 'build_model'], + 'cnn_lvl5_attention_mechanisms.task': ['SEBlock', 'SimpleCNN'], + 'nlp_lvl1_text_embedding.task': ['SkipGramModel', 'make_dataloaders'], + 'robust_lvl1_adversarial_training.task': ['FGSM', 'SimpleCNN'], + } + + # Add tasks directory to path + sys.path.insert(0, os.path.join(base_path, '..', '..')) + + all_importable = True + for module_path, items in task_imports.items(): + task_dir = module_path.split('.')[0] + print(f"\n{task_dir}:") + try: + module = __import__(module_path, fromlist=items) + for item in items: + if hasattr(module, item): + print(f" ✓ {item}") + else: + print(f" ⚠ {item} not found (may be defined in function)") + except ImportError as e: + print(f" ✗ Import failed: {e}") + all_importable = False + except Exception as e: + print(f" ⚠ Error: {e}") + + return all_importable + + +def generate_summary(base_path): + """Generate summary of verification results.""" + print("\n" + "="*70) + print("SUMMARY") + print("="*70) + + # Count lines of code + tasks = [ + 'mlp_lvl5_ensemble_distillation', + 'cnn_lvl5_attention_mechanisms', + 'nlp_lvl1_text_embedding', + 'robust_lvl1_adversarial_training' + ] + + total_lines = 0 + total_size_kb = 0 + + for task in tasks: + task_path = os.path.join(base_path, task, 'task.py') + if os.path.exists(task_path): + lines = len(open(task_path).readlines()) + size_kb = os.path.getsize(task_path) / 1024 + total_lines += lines + total_size_kb += size_kb + + print(f"\nFiles Created:") + print(f" - 4 task.py implementations") + print(f" - Total code: {total_lines:,} lines, {total_size_kb:.1f} KB") + print(f" - 2 documentation files") + print(f" - 1 ml_tasks.json updated") + + print(f"\nTasks Added:") + for task in tasks: + print(f" ✓ {task}") + + print(f"\nNext Steps:") + print(f" 1. See NEW_TASKS_SUMMARY.md for task descriptions") + print(f" 2. See GITHUB_SUBMISSION_GUIDE.md for submission instructions") + print(f" 3. Run individual tasks to verify execution") + print(f" 4. Create PR or new repository as described in guide") + + +def main(base_path=None): + """Run all verification checks.""" + + if base_path is None: + # Find MLtasks directory + current = Path(__file__).parent.absolute() + base_path = current / 'MLtasks' / 'tasks' + + if not base_path.exists(): + print(f"Error: Could not find tasks directory at {base_path}") + print(f"Please provide the path to MLtasks/tasks as argument") + return False + + base_path = str(base_path) + json_path = os.path.join(base_path, '..', 'ml_tasks.json') + + print("\n" + "="*70) + print("CoderGym Neural Network Tasks Verification") + print("="*70) + + results = [] + + # Run all checks + results.append(("JSON Validity", check_json_validity(json_path))) + results.append(("Task Files", check_task_files(base_path))) + results.append(("Protocol Compliance", check_protocol_compliance(base_path))) + results.append(("Documentation", check_documentation(os.path.dirname(base_path)))) + results.append(("Imports", check_imports(base_path))) + + # Summary + generate_summary(base_path) + + # Final status + print("\n" + "="*70) + print("VERIFICATION RESULTS") + print("="*70) + + all_passed = True + for check_name, result in results: + status = "✓ PASS" if result else "✗ FAIL" + print(f"{check_name:.<50} {status}") + if not result: + all_passed = False + + print("="*70) + + if all_passed: + print("\n✓ All verification checks passed!") + print("Tasks are ready for GitHub submission.") + return True + else: + print("\n✗ Some verification checks failed.") + print("Please review the output above for details.") + return False + + +if __name__ == '__main__': + if len(sys.argv) > 1: + success = main(sys.argv[1]) + else: + success = main() + + sys.exit(0 if success else 1)