From 23700f4d124993d56d2b883969b810828b53fc6d Mon Sep 17 00:00:00 2001 From: komaksym Date: Sun, 6 Jul 2025 22:39:36 +0200 Subject: [PATCH 01/14] create a new problem --- .../description.md | 1 + .../123_mixed_precision_training/example.json | 5 ++ .../123_mixed_precision_training/learn.md | 55 +++++++++++++++ .../123_mixed_precision_training/meta.json | 15 +++++ .../123_mixed_precision_training/solution.py | 67 +++++++++++++++++++ .../starter_code.py | 14 ++++ .../123_mixed_precision_training/tests.json | 22 ++++++ 7 files changed, 179 insertions(+) create mode 100644 questions/123_mixed_precision_training/description.md create mode 100644 questions/123_mixed_precision_training/example.json create mode 100644 questions/123_mixed_precision_training/learn.md create mode 100644 questions/123_mixed_precision_training/meta.json create mode 100644 questions/123_mixed_precision_training/solution.py create mode 100644 questions/123_mixed_precision_training/starter_code.py create mode 100644 questions/123_mixed_precision_training/tests.json diff --git a/questions/123_mixed_precision_training/description.md b/questions/123_mixed_precision_training/description.md new file mode 100644 index 00000000..b2d93313 --- /dev/null +++ b/questions/123_mixed_precision_training/description.md @@ -0,0 +1 @@ +Write a Python class to implement Mixed Precision Training that uses both float32 and float16 data types to optimize memory usage and speed. Your class should have an `__init__(self, loss_scale=1024.0)` method to initialize with loss scaling factor. Implement `forward(self, weights, inputs, targets)` to perform forward pass with float16 computation and return loss (scaled) in float32, and `backward(self, gradients)` to unscale gradients and check for overflow. Use float16 for computations but float32 for gradient accumulation. Return gradients as float32 and set them to zero if overflow is detected. The forward method must return loss as float32 dtype. Only use NumPy. \ No newline at end of file diff --git a/questions/123_mixed_precision_training/example.json b/questions/123_mixed_precision_training/example.json new file mode 100644 index 00000000..5a618f33 --- /dev/null +++ b/questions/123_mixed_precision_training/example.json @@ -0,0 +1,5 @@ +{ + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 1638.4000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32", + "reasoning": "Forward pass converts inputs to float16, computes loss, then scales and returns as Python float (float32). Backward converts gradients to float32 and unscales. Final gradients must be float32 type." +} \ No newline at end of file diff --git a/questions/123_mixed_precision_training/learn.md b/questions/123_mixed_precision_training/learn.md new file mode 100644 index 00000000..45c6e225 --- /dev/null +++ b/questions/123_mixed_precision_training/learn.md @@ -0,0 +1,55 @@ +# **Mixed Precision Training** + +## **1. Definition** + +Mixed Precision Training is a **deep learning optimization technique** that uses both **float16** (half precision) and **float32** (single precision) data types during training to reduce memory usage and increase training speed while maintaining model accuracy. + +The technique works by: +- **Using float16 for forward pass computations** to save memory and increase speed +- **Using float32 for gradient accumulation** to maintain numerical precision +- **Applying loss scaling** to prevent gradient underflow in float16 + +--- + +## **2. Key Components** + +### **Loss Scaling** +To prevent gradient underflow in float16, gradients are scaled up during the forward pass: + +$$ +\text{scaled\_loss} = \text{loss} \times \text{scale\_factor} +$$ + +Then unscaled during backward pass: + +$$ +\text{gradient} = \frac{\text{scaled\_gradient}}{\text{scale\_factor}} +$$ + +### **Overflow Detection** +Check for invalid gradients (NaN or Inf) that indicate numerical overflow: + +$$ +\text{overflow} = \text{any}(\text{isnan}(\text{gradients}) \text{ or } \text{isinf}(\text{gradients})) +$$ + +--- + +## **3. Precision Usage** + +- **float16**: Forward pass computations, activations, temporary calculations +- **float32**: Gradient accumulation, parameter updates, loss scaling +- **Automatic casting**: Convert between precisions as needed + +--- + +## **4. Benefits and Applications** + +- **Memory Efficiency**: Reduces memory usage by ~50% for activations +- **Speed Improvement**: Faster computation on modern GPUs with Tensor Cores +- **Training Stability**: Loss scaling prevents gradient underflow +- **Model Accuracy**: Maintains comparable accuracy to full precision training + +Common in training large neural networks where memory is a constraint and speed is critical. + +--- \ No newline at end of file diff --git a/questions/123_mixed_precision_training/meta.json b/questions/123_mixed_precision_training/meta.json new file mode 100644 index 00000000..37acfb6d --- /dev/null +++ b/questions/123_mixed_precision_training/meta.json @@ -0,0 +1,15 @@ +{ + "id": "142", + "title": "Mixed Precision Training", + "difficulty": "medium", + "category": "Machine Learning", + "video": "", + "likes": "0", + "dislikes": "0", + "contributor": [ + { + "profile_link": "https://github.com/contributor", + "name": "contributor" + } + ] +} \ No newline at end of file diff --git a/questions/123_mixed_precision_training/solution.py b/questions/123_mixed_precision_training/solution.py new file mode 100644 index 00000000..99e04be4 --- /dev/null +++ b/questions/123_mixed_precision_training/solution.py @@ -0,0 +1,67 @@ +import numpy as np +import json + +class MixedPrecision: + def __init__(self, loss_scale=1024.0): + self.loss_scale = loss_scale + + def forward(self, weights, inputs, targets): + # Convert ALL inputs to float16 for computation (regardless of input dtype) + weights_fp16 = weights.astype(np.float16) + inputs_fp16 = inputs.astype(np.float16) + targets_fp16 = targets.astype(np.float16) + + # Simple forward pass: linear model + MSE loss + predictions = np.dot(inputs_fp16, weights_fp16) + loss = np.mean((predictions - targets_fp16) ** 2) + + # Scale loss and convert back to float32 (Python float) + scaled_loss = float(loss) * self.loss_scale + return scaled_loss + + def backward(self, gradients): + # Convert gradients to float32 for precision (regardless of input dtype) + gradients_fp32 = gradients.astype(np.float32) + + # Check for overflow (NaN or Inf) + overflow = np.any(np.isnan(gradients_fp32)) or np.any(np.isinf(gradients_fp32)) + + if overflow: + # Return zero gradients if overflow detected (must be float32) + return np.zeros_like(gradients_fp32, dtype=np.float32) + + # Unscale gradients (ensure result is float32) + unscaled_gradients = gradients_fp32 / self.loss_scale + return unscaled_gradients.astype(np.float32) + + +tests = [ + { + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 1638.4000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32" + }, + { + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\nweights = np.array([1.0, 0.5], dtype=np.float64)\ninputs = np.array([[2.0, 1.0]], dtype=np.float64)\ntargets = np.array([3.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([1024.0, 512.0], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 256.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" + }, + { + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\nweights = np.array([0.1, 0.2], dtype=np.float32)\ninputs = np.array([[1.0, 1.0]], dtype=np.float32)\ntargets = np.array([0.5], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([200.0, 100.0], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 4.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" + }, + { + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=2048.0)\nweights = np.array([0.25], dtype=np.float64)\ninputs = np.array([[4.0]], dtype=np.float64)\ntargets = np.array([2.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.nan], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 2048.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" + }, + { + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=256.0)\nweights = np.array([1.0], dtype=np.float16)\ninputs = np.array([[2.0]], dtype=np.float16)\ntargets = np.array([3.0], dtype=np.float16)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.inf], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 256.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" + } +] + +for i, test in enumerate(tests): + print(f"Test #{i+1}.") + print("Got:") + exec(test['input']) + print(f"\nExpected: {test['output']}") + print("--------" * 3) + diff --git a/questions/123_mixed_precision_training/starter_code.py b/questions/123_mixed_precision_training/starter_code.py new file mode 100644 index 00000000..7a5e42a0 --- /dev/null +++ b/questions/123_mixed_precision_training/starter_code.py @@ -0,0 +1,14 @@ +import numpy as np + +class MixedPrecision: + def __init__(self, loss_scale=1024.0): + # Initialize loss scaling factor + pass + + def forward(self, weights, inputs, targets): + # Perform forward pass with float16, return scaled loss as float32 + pass + + def backward(self, gradients): + # Unscale gradients and check for overflow, return as float32 + pass \ No newline at end of file diff --git a/questions/123_mixed_precision_training/tests.json b/questions/123_mixed_precision_training/tests.json new file mode 100644 index 00000000..2a6e3144 --- /dev/null +++ b/questions/123_mixed_precision_training/tests.json @@ -0,0 +1,22 @@ +[ + { + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 1638.4000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32" + }, + { + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\nweights = np.array([1.0, 0.5], dtype=np.float64)\ninputs = np.array([[2.0, 1.0]], dtype=np.float64)\ntargets = np.array([3.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([1024.0, 512.0], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 256.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" + }, + { + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\nweights = np.array([0.1, 0.2], dtype=np.float32)\ninputs = np.array([[1.0, 1.0]], dtype=np.float32)\ntargets = np.array([0.5], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([200.0, 100.0], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 4.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" + }, + { + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=2048.0)\nweights = np.array([0.25], dtype=np.float64)\ninputs = np.array([[4.0]], dtype=np.float64)\ntargets = np.array([2.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.nan], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 2048.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" + }, + { + "input": "import numpy as np\nmp = MixedPrecision(loss_scale=256.0)\nweights = np.array([1.0], dtype=np.float16)\ninputs = np.array([[2.0]], dtype=np.float16)\ntargets = np.array([3.0], dtype=np.float16)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.inf], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "output": "Loss: 256.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" + } +] \ No newline at end of file From defbfb454f35985aeaf57c377b5f2a636116a24b Mon Sep 17 00:00:00 2001 From: komaksym Date: Sun, 6 Jul 2025 23:21:57 +0200 Subject: [PATCH 02/14] fix test cases --- questions/123_mixed_precision_training/tests.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/questions/123_mixed_precision_training/tests.json b/questions/123_mixed_precision_training/tests.json index 2a6e3144..fab5395c 100644 --- a/questions/123_mixed_precision_training/tests.json +++ b/questions/123_mixed_precision_training/tests.json @@ -1,11 +1,11 @@ [ { "input": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 1638.4000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32" + "output": "Loss: 665.0000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32" }, { "input": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\nweights = np.array([1.0, 0.5], dtype=np.float64)\ninputs = np.array([[2.0, 1.0]], dtype=np.float64)\ntargets = np.array([3.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([1024.0, 512.0], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 256.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" + "output": "Loss: 128.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" }, { "input": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\nweights = np.array([0.1, 0.2], dtype=np.float32)\ninputs = np.array([[1.0, 1.0]], dtype=np.float32)\ntargets = np.array([0.5], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([200.0, 100.0], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", From a434de21ae4d60803e9bda40e57bd5f4c40b6ec9 Mon Sep 17 00:00:00 2001 From: komaksym Date: Sun, 6 Jul 2025 23:22:27 +0200 Subject: [PATCH 03/14] remove sandbox code --- .../123_mixed_precision_training/solution.py | 34 +------------------ 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/questions/123_mixed_precision_training/solution.py b/questions/123_mixed_precision_training/solution.py index 99e04be4..1eae144b 100644 --- a/questions/123_mixed_precision_training/solution.py +++ b/questions/123_mixed_precision_training/solution.py @@ -32,36 +32,4 @@ def backward(self, gradients): # Unscale gradients (ensure result is float32) unscaled_gradients = gradients_fp32 / self.loss_scale - return unscaled_gradients.astype(np.float32) - - -tests = [ - { - "input": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 1638.4000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32" - }, - { - "input": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\nweights = np.array([1.0, 0.5], dtype=np.float64)\ninputs = np.array([[2.0, 1.0]], dtype=np.float64)\ntargets = np.array([3.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([1024.0, 512.0], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 256.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" - }, - { - "input": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\nweights = np.array([0.1, 0.2], dtype=np.float32)\ninputs = np.array([[1.0, 1.0]], dtype=np.float32)\ntargets = np.array([0.5], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([200.0, 100.0], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 4.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" - }, - { - "input": "import numpy as np\nmp = MixedPrecision(loss_scale=2048.0)\nweights = np.array([0.25], dtype=np.float64)\ninputs = np.array([[4.0]], dtype=np.float64)\ntargets = np.array([2.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.nan], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 2048.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" - }, - { - "input": "import numpy as np\nmp = MixedPrecision(loss_scale=256.0)\nweights = np.array([1.0], dtype=np.float16)\ninputs = np.array([[2.0]], dtype=np.float16)\ntargets = np.array([3.0], dtype=np.float16)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.inf], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 256.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" - } -] - -for i, test in enumerate(tests): - print(f"Test #{i+1}.") - print("Got:") - exec(test['input']) - print(f"\nExpected: {test['output']}") - print("--------" * 3) - + return unscaled_gradients.astype(np.float32) \ No newline at end of file From 579dc3041108b1c8c296a8d42d3b6729e417f057 Mon Sep 17 00:00:00 2001 From: komaksym Date: Sun, 6 Jul 2025 23:26:39 +0200 Subject: [PATCH 04/14] fix expected loss --- questions/123_mixed_precision_training/example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questions/123_mixed_precision_training/example.json b/questions/123_mixed_precision_training/example.json index 5a618f33..ec739c21 100644 --- a/questions/123_mixed_precision_training/example.json +++ b/questions/123_mixed_precision_training/example.json @@ -1,5 +1,5 @@ { "input": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 1638.4000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32", + "output": "Loss: 665.0000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32", "reasoning": "Forward pass converts inputs to float16, computes loss, then scales and returns as Python float (float32). Backward converts gradients to float32 and unscales. Final gradients must be float32 type." } \ No newline at end of file From 2b66fac4d92b363d0e5520a34c1d9beb1022cc2a Mon Sep 17 00:00:00 2001 From: komaksym Date: Sun, 6 Jul 2025 23:35:45 +0200 Subject: [PATCH 05/14] minor cleanup --- questions/123_mixed_precision_training/solution.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/questions/123_mixed_precision_training/solution.py b/questions/123_mixed_precision_training/solution.py index 1eae144b..442472d9 100644 --- a/questions/123_mixed_precision_training/solution.py +++ b/questions/123_mixed_precision_training/solution.py @@ -1,5 +1,4 @@ import numpy as np -import json class MixedPrecision: def __init__(self, loss_scale=1024.0): @@ -13,7 +12,7 @@ def forward(self, weights, inputs, targets): # Simple forward pass: linear model + MSE loss predictions = np.dot(inputs_fp16, weights_fp16) - loss = np.mean((predictions - targets_fp16) ** 2) + loss = np.mean((targets_fp16 - predictions) ** 2) # Scale loss and convert back to float32 (Python float) scaled_loss = float(loss) * self.loss_scale From 52b1d5e64841a603a5f3aeec1ea48d99e3290730 Mon Sep 17 00:00:00 2001 From: komaksym Date: Sun, 6 Jul 2025 23:39:49 +0200 Subject: [PATCH 06/14] cleared up the fact that the loss should be computed as MSE --- .../description.md | 2 +- .../123_mixed_precision_training/learn.md | 27 ++++++------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/questions/123_mixed_precision_training/description.md b/questions/123_mixed_precision_training/description.md index b2d93313..2bc349d1 100644 --- a/questions/123_mixed_precision_training/description.md +++ b/questions/123_mixed_precision_training/description.md @@ -1 +1 @@ -Write a Python class to implement Mixed Precision Training that uses both float32 and float16 data types to optimize memory usage and speed. Your class should have an `__init__(self, loss_scale=1024.0)` method to initialize with loss scaling factor. Implement `forward(self, weights, inputs, targets)` to perform forward pass with float16 computation and return loss (scaled) in float32, and `backward(self, gradients)` to unscale gradients and check for overflow. Use float16 for computations but float32 for gradient accumulation. Return gradients as float32 and set them to zero if overflow is detected. The forward method must return loss as float32 dtype. Only use NumPy. \ No newline at end of file +Write a Python class to implement Mixed Precision Training that uses both float32 and float16 data types to optimize memory usage and speed. Your class should have an `__init__(self, loss_scale=1024.0)` method to initialize with loss scaling factor. Implement `forward(self, weights, inputs, targets)` to perform forward pass with float16 computation and return Mean Squared Error (MSE) loss (scaled) in float32, and `backward(self, gradients)` to unscale gradients and check for overflow. Use float16 for computations but float32 for gradient accumulation. Return gradients as float32 and set them to zero if overflow is detected. Only use NumPy. \ No newline at end of file diff --git a/questions/123_mixed_precision_training/learn.md b/questions/123_mixed_precision_training/learn.md index 45c6e225..c3828c9f 100644 --- a/questions/123_mixed_precision_training/learn.md +++ b/questions/123_mixed_precision_training/learn.md @@ -1,55 +1,44 @@ # **Mixed Precision Training** - ## **1. Definition** - Mixed Precision Training is a **deep learning optimization technique** that uses both **float16** (half precision) and **float32** (single precision) data types during training to reduce memory usage and increase training speed while maintaining model accuracy. - The technique works by: - **Using float16 for forward pass computations** to save memory and increase speed - **Using float32 for gradient accumulation** to maintain numerical precision - **Applying loss scaling** to prevent gradient underflow in float16 - --- - ## **2. Key Components** +### **Mean Squared Error (MSE) Loss** +The loss function must be computed as Mean Squared Error: +$$ +\text{MSE} = \frac{1}{n} \sum_{i=1}^{n} (y_i - \hat{y}_i)^2 +$$ +where $y_i$ is the target and $\hat{y}_i$ is the prediction for sample $i$. ### **Loss Scaling** To prevent gradient underflow in float16, gradients are scaled up during the forward pass: - $$ -\text{scaled\_loss} = \text{loss} \times \text{scale\_factor} +\text{scaled\_loss} = \text{MSE} \times \text{scale\_factor} $$ - Then unscaled during backward pass: - $$ \text{gradient} = \frac{\text{scaled\_gradient}}{\text{scale\_factor}} $$ - ### **Overflow Detection** Check for invalid gradients (NaN or Inf) that indicate numerical overflow: - $$ \text{overflow} = \text{any}(\text{isnan}(\text{gradients}) \text{ or } \text{isinf}(\text{gradients})) $$ - --- - ## **3. Precision Usage** - - **float16**: Forward pass computations, activations, temporary calculations - **float32**: Gradient accumulation, parameter updates, loss scaling - **Automatic casting**: Convert between precisions as needed - +- **Loss computation**: Use MSE as the loss function before scaling --- - ## **4. Benefits and Applications** - - **Memory Efficiency**: Reduces memory usage by ~50% for activations - **Speed Improvement**: Faster computation on modern GPUs with Tensor Cores - **Training Stability**: Loss scaling prevents gradient underflow - **Model Accuracy**: Maintains comparable accuracy to full precision training - Common in training large neural networks where memory is a constraint and speed is critical. - --- \ No newline at end of file From 3431f5082237d45cd57aa43c40da171231b8e71e Mon Sep 17 00:00:00 2001 From: komaksym Date: Sun, 6 Jul 2025 23:43:00 +0200 Subject: [PATCH 07/14] fix meta --- questions/123_mixed_precision_training/meta.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/questions/123_mixed_precision_training/meta.json b/questions/123_mixed_precision_training/meta.json index 37acfb6d..a8a12cdf 100644 --- a/questions/123_mixed_precision_training/meta.json +++ b/questions/123_mixed_precision_training/meta.json @@ -1,5 +1,5 @@ { - "id": "142", + "id": "161", "title": "Mixed Precision Training", "difficulty": "medium", "category": "Machine Learning", @@ -8,8 +8,8 @@ "dislikes": "0", "contributor": [ { - "profile_link": "https://github.com/contributor", - "name": "contributor" + "profile_link": "https://github.com/komaksym", + "name": "komaksym" } ] } \ No newline at end of file From f8cd69b6c4e18e0dad53e4524f3674a426c0f23d Mon Sep 17 00:00:00 2001 From: komaksym Date: Sun, 6 Jul 2025 23:44:45 +0200 Subject: [PATCH 08/14] rename folder --- .../description.md | 0 .../example.json | 0 .../learn.md | 0 .../meta.json | 0 .../solution.py | 0 .../starter_code.py | 0 .../tests.json | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename questions/{123_mixed_precision_training => 161_mixed_precision_training}/description.md (100%) rename questions/{123_mixed_precision_training => 161_mixed_precision_training}/example.json (100%) rename questions/{123_mixed_precision_training => 161_mixed_precision_training}/learn.md (100%) rename questions/{123_mixed_precision_training => 161_mixed_precision_training}/meta.json (100%) rename questions/{123_mixed_precision_training => 161_mixed_precision_training}/solution.py (100%) rename questions/{123_mixed_precision_training => 161_mixed_precision_training}/starter_code.py (100%) rename questions/{123_mixed_precision_training => 161_mixed_precision_training}/tests.json (100%) diff --git a/questions/123_mixed_precision_training/description.md b/questions/161_mixed_precision_training/description.md similarity index 100% rename from questions/123_mixed_precision_training/description.md rename to questions/161_mixed_precision_training/description.md diff --git a/questions/123_mixed_precision_training/example.json b/questions/161_mixed_precision_training/example.json similarity index 100% rename from questions/123_mixed_precision_training/example.json rename to questions/161_mixed_precision_training/example.json diff --git a/questions/123_mixed_precision_training/learn.md b/questions/161_mixed_precision_training/learn.md similarity index 100% rename from questions/123_mixed_precision_training/learn.md rename to questions/161_mixed_precision_training/learn.md diff --git a/questions/123_mixed_precision_training/meta.json b/questions/161_mixed_precision_training/meta.json similarity index 100% rename from questions/123_mixed_precision_training/meta.json rename to questions/161_mixed_precision_training/meta.json diff --git a/questions/123_mixed_precision_training/solution.py b/questions/161_mixed_precision_training/solution.py similarity index 100% rename from questions/123_mixed_precision_training/solution.py rename to questions/161_mixed_precision_training/solution.py diff --git a/questions/123_mixed_precision_training/starter_code.py b/questions/161_mixed_precision_training/starter_code.py similarity index 100% rename from questions/123_mixed_precision_training/starter_code.py rename to questions/161_mixed_precision_training/starter_code.py diff --git a/questions/123_mixed_precision_training/tests.json b/questions/161_mixed_precision_training/tests.json similarity index 100% rename from questions/123_mixed_precision_training/tests.json rename to questions/161_mixed_precision_training/tests.json From 06adf2616328d243d296989b6d1e0993d1e4a484 Mon Sep 17 00:00:00 2001 From: komaksym Date: Wed, 9 Jul 2025 23:58:37 +0200 Subject: [PATCH 09/14] fix dict key names --- .../161_mixed_precision_training/tests.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/questions/161_mixed_precision_training/tests.json b/questions/161_mixed_precision_training/tests.json index fab5395c..6e9470e0 100644 --- a/questions/161_mixed_precision_training/tests.json +++ b/questions/161_mixed_precision_training/tests.json @@ -1,22 +1,22 @@ [ { - "input": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 665.0000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32" + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", + "expected_output": "Loss: 665.0000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32" }, { - "input": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\nweights = np.array([1.0, 0.5], dtype=np.float64)\ninputs = np.array([[2.0, 1.0]], dtype=np.float64)\ntargets = np.array([3.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([1024.0, 512.0], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 128.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\nweights = np.array([1.0, 0.5], dtype=np.float64)\ninputs = np.array([[2.0, 1.0]], dtype=np.float64)\ntargets = np.array([3.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([1024.0, 512.0], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "expected_output": "Loss: 128.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" }, { - "input": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\nweights = np.array([0.1, 0.2], dtype=np.float32)\ninputs = np.array([[1.0, 1.0]], dtype=np.float32)\ntargets = np.array([0.5], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([200.0, 100.0], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 4.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\nweights = np.array([0.1, 0.2], dtype=np.float32)\ninputs = np.array([[1.0, 1.0]], dtype=np.float32)\ntargets = np.array([0.5], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([200.0, 100.0], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "expected_output": "Loss: 4.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" }, { - "input": "import numpy as np\nmp = MixedPrecision(loss_scale=2048.0)\nweights = np.array([0.25], dtype=np.float64)\ninputs = np.array([[4.0]], dtype=np.float64)\ntargets = np.array([2.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.nan], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 2048.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=2048.0)\nweights = np.array([0.25], dtype=np.float64)\ninputs = np.array([[4.0]], dtype=np.float64)\ntargets = np.array([2.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.nan], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "expected_output": "Loss: 2048.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" }, { - "input": "import numpy as np\nmp = MixedPrecision(loss_scale=256.0)\nweights = np.array([1.0], dtype=np.float16)\ninputs = np.array([[2.0]], dtype=np.float16)\ntargets = np.array([3.0], dtype=np.float16)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.inf], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "output": "Loss: 256.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=256.0)\nweights = np.array([1.0], dtype=np.float16)\ninputs = np.array([[2.0]], dtype=np.float16)\ntargets = np.array([3.0], dtype=np.float16)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.inf], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "expected_output": "Loss: 256.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" } ] \ No newline at end of file From e02f1eb7d5c7fe39a6e269612336790c4b033539 Mon Sep 17 00:00:00 2001 From: komaksym Date: Thu, 10 Jul 2025 00:12:58 +0200 Subject: [PATCH 10/14] change to using only numpy-based dtypes --- .../161_mixed_precision_training/solution.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/questions/161_mixed_precision_training/solution.py b/questions/161_mixed_precision_training/solution.py index 442472d9..d4b97ad3 100644 --- a/questions/161_mixed_precision_training/solution.py +++ b/questions/161_mixed_precision_training/solution.py @@ -1,34 +1,36 @@ import numpy as np + class MixedPrecision: def __init__(self, loss_scale=1024.0): self.loss_scale = loss_scale - + def forward(self, weights, inputs, targets): # Convert ALL inputs to float16 for computation (regardless of input dtype) weights_fp16 = weights.astype(np.float16) inputs_fp16 = inputs.astype(np.float16) targets_fp16 = targets.astype(np.float16) - + # Simple forward pass: linear model + MSE loss predictions = np.dot(inputs_fp16, weights_fp16) loss = np.mean((targets_fp16 - predictions) ** 2) - - # Scale loss and convert back to float32 (Python float) - scaled_loss = float(loss) * self.loss_scale + + # Scale loss and convert back to float32 + scaled_loss = loss.astype(np.float32) * self.loss_scale + return scaled_loss - + def backward(self, gradients): # Convert gradients to float32 for precision (regardless of input dtype) gradients_fp32 = gradients.astype(np.float32) - + # Check for overflow (NaN or Inf) overflow = np.any(np.isnan(gradients_fp32)) or np.any(np.isinf(gradients_fp32)) - + if overflow: # Return zero gradients if overflow detected (must be float32) return np.zeros_like(gradients_fp32, dtype=np.float32) - + # Unscale gradients (ensure result is float32) unscaled_gradients = gradients_fp32 / self.loss_scale - return unscaled_gradients.astype(np.float32) \ No newline at end of file + return unscaled_gradients.astype(np.float32) From fe7c3483058c51bde6893652b532756fb9b5a890 Mon Sep 17 00:00:00 2001 From: komaksym Date: Thu, 10 Jul 2025 00:44:04 +0200 Subject: [PATCH 11/14] break tests down into separate forward & backward pass tests --- .../161_mixed_precision_training/tests.json | 60 ++++++++++++------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/questions/161_mixed_precision_training/tests.json b/questions/161_mixed_precision_training/tests.json index 6e9470e0..00dcb1ca 100644 --- a/questions/161_mixed_precision_training/tests.json +++ b/questions/161_mixed_precision_training/tests.json @@ -1,22 +1,42 @@ [ - { - "test": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", - "expected_output": "Loss: 665.0000\nLoss dtype: float\nGradients: [0.5 -0.25]\nGrad dtype: float32" - }, - { - "test": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\nweights = np.array([1.0, 0.5], dtype=np.float64)\ninputs = np.array([[2.0, 1.0]], dtype=np.float64)\ntargets = np.array([3.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([1024.0, 512.0], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "expected_output": "Loss: 128.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" - }, - { - "test": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\nweights = np.array([0.1, 0.2], dtype=np.float32)\ninputs = np.array([[1.0, 1.0]], dtype=np.float32)\ntargets = np.array([0.5], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([200.0, 100.0], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "expected_output": "Loss: 4.0\nLoss dtype: float\nGradients: [2 1]\nGrad dtype: float32" - }, - { - "test": "import numpy as np\nmp = MixedPrecision(loss_scale=2048.0)\nweights = np.array([0.25], dtype=np.float64)\ninputs = np.array([[4.0]], dtype=np.float64)\ntargets = np.array([2.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.nan], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "expected_output": "Loss: 2048.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" - }, - { - "test": "import numpy as np\nmp = MixedPrecision(loss_scale=256.0)\nweights = np.array([1.0], dtype=np.float16)\ninputs = np.array([[2.0]], dtype=np.float16)\ntargets = np.array([3.0], dtype=np.float16)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")\ngrads = np.array([np.inf], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", - "expected_output": "Loss: 256.0\nLoss dtype: float\nGradients: [0]\nGrad dtype: float32" - } + { + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")", + "expected_output": "Loss: 665.0000\nLoss dtype: float32" + }, + { + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", + "expected_output": "Gradients: [ 0.5 -0.25]\nGrad dtype: float32" + }, + { + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\nweights = np.array([1.0, 0.5], dtype=np.float64)\ninputs = np.array([[2.0, 1.0]], dtype=np.float64)\ntargets = np.array([3.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")", + "expected_output": "Loss: 128.0\nLoss dtype: float32" + }, + { + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\ngrads = np.array([1024.0, 512.0], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "expected_output": "Gradients: [2 1]\nGrad dtype: float32" + }, + { + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\nweights = np.array([0.1, 0.2], dtype=np.float32)\ninputs = np.array([[1.0, 1.0]], dtype=np.float32)\ntargets = np.array([0.5], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")", + "expected_output": "Loss: 4.0\nLoss dtype: float32" + }, + { + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\ngrads = np.array([200.0, 100.0], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "expected_output": "Gradients: [2 1]\nGrad dtype: float32" + }, + { + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=2048.0)\nweights = np.array([0.25], dtype=np.float64)\ninputs = np.array([[4.0]], dtype=np.float64)\ntargets = np.array([2.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")", + "expected_output": "Loss: 2048.0\nLoss dtype: float32" + }, + { + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=2048.0)\ngrads = np.array([np.nan], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "expected_output": "Gradients: [0]\nGrad dtype: float32" + }, + { + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=256.0)\nweights = np.array([1.0], dtype=np.float16)\ninputs = np.array([[2.0]], dtype=np.float16)\ntargets = np.array([3.0], dtype=np.float16)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")", + "expected_output": "Loss: 256.0\nLoss dtype: float32" + }, + { + "test": "import numpy as np\nmp = MixedPrecision(loss_scale=256.0)\ngrads = np.array([np.inf], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", + "expected_output": "Gradients: [0]\nGrad dtype: float32" + } ] \ No newline at end of file From 4fb614bc18117398d8b4d6033c13a1955b4e0fbd Mon Sep 17 00:00:00 2001 From: komaksym Date: Thu, 10 Jul 2025 00:54:59 +0200 Subject: [PATCH 12/14] rename folder --- .../description.md | 0 .../example.json | 0 .../learn.md | 0 .../meta.json | 0 .../solution.py | 0 .../starter_code.py | 0 .../tests.json | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename questions/{161_mixed_precision_training => 160_mixed_precision_training}/description.md (100%) rename questions/{161_mixed_precision_training => 160_mixed_precision_training}/example.json (100%) rename questions/{161_mixed_precision_training => 160_mixed_precision_training}/learn.md (100%) rename questions/{161_mixed_precision_training => 160_mixed_precision_training}/meta.json (100%) rename questions/{161_mixed_precision_training => 160_mixed_precision_training}/solution.py (100%) rename questions/{161_mixed_precision_training => 160_mixed_precision_training}/starter_code.py (100%) rename questions/{161_mixed_precision_training => 160_mixed_precision_training}/tests.json (100%) diff --git a/questions/161_mixed_precision_training/description.md b/questions/160_mixed_precision_training/description.md similarity index 100% rename from questions/161_mixed_precision_training/description.md rename to questions/160_mixed_precision_training/description.md diff --git a/questions/161_mixed_precision_training/example.json b/questions/160_mixed_precision_training/example.json similarity index 100% rename from questions/161_mixed_precision_training/example.json rename to questions/160_mixed_precision_training/example.json diff --git a/questions/161_mixed_precision_training/learn.md b/questions/160_mixed_precision_training/learn.md similarity index 100% rename from questions/161_mixed_precision_training/learn.md rename to questions/160_mixed_precision_training/learn.md diff --git a/questions/161_mixed_precision_training/meta.json b/questions/160_mixed_precision_training/meta.json similarity index 100% rename from questions/161_mixed_precision_training/meta.json rename to questions/160_mixed_precision_training/meta.json diff --git a/questions/161_mixed_precision_training/solution.py b/questions/160_mixed_precision_training/solution.py similarity index 100% rename from questions/161_mixed_precision_training/solution.py rename to questions/160_mixed_precision_training/solution.py diff --git a/questions/161_mixed_precision_training/starter_code.py b/questions/160_mixed_precision_training/starter_code.py similarity index 100% rename from questions/161_mixed_precision_training/starter_code.py rename to questions/160_mixed_precision_training/starter_code.py diff --git a/questions/161_mixed_precision_training/tests.json b/questions/160_mixed_precision_training/tests.json similarity index 100% rename from questions/161_mixed_precision_training/tests.json rename to questions/160_mixed_precision_training/tests.json From cdf5dd120d925d0e3e63a56210d5b36b4d57d1dc Mon Sep 17 00:00:00 2001 From: komaksym Date: Thu, 10 Jul 2025 00:55:45 +0200 Subject: [PATCH 13/14] change question number --- questions/160_mixed_precision_training/meta.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questions/160_mixed_precision_training/meta.json b/questions/160_mixed_precision_training/meta.json index a8a12cdf..87ab5133 100644 --- a/questions/160_mixed_precision_training/meta.json +++ b/questions/160_mixed_precision_training/meta.json @@ -1,5 +1,5 @@ { - "id": "161", + "id": "160", "title": "Mixed Precision Training", "difficulty": "medium", "category": "Machine Learning", From f13d2a369b399c8ef12fb4bdc50d8f21c0bdbad6 Mon Sep 17 00:00:00 2001 From: komaksym Date: Sat, 16 Aug 2025 17:46:08 +0200 Subject: [PATCH 14/14] fix expected loss dtypes --- questions/160_mixed_precision_training/tests.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/questions/160_mixed_precision_training/tests.json b/questions/160_mixed_precision_training/tests.json index 00dcb1ca..522b3043 100644 --- a/questions/160_mixed_precision_training/tests.json +++ b/questions/160_mixed_precision_training/tests.json @@ -1,7 +1,7 @@ [ { "test": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\nweights = np.array([0.5, -0.3], dtype=np.float32)\ninputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\ntargets = np.array([1.0, 0.0], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.4f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")", - "expected_output": "Loss: 665.0000\nLoss dtype: float32" + "expected_output": "Loss: 665.0000\nLoss dtype: float" }, { "test": "import numpy as np\nmp = MixedPrecision(loss_scale=1024.0)\ngrads = np.array([512.0, -256.0], dtype=np.float32)\nresult = mp.backward(grads)\nprint(f\"Gradients: {result}\")\nprint(f\"Grad dtype: {result.dtype}\")", @@ -9,7 +9,7 @@ }, { "test": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\nweights = np.array([1.0, 0.5], dtype=np.float64)\ninputs = np.array([[2.0, 1.0]], dtype=np.float64)\ntargets = np.array([3.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")", - "expected_output": "Loss: 128.0\nLoss dtype: float32" + "expected_output": "Loss: 128.0\nLoss dtype: float" }, { "test": "import numpy as np\nmp = MixedPrecision(loss_scale=512.0)\ngrads = np.array([1024.0, 512.0], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", @@ -17,7 +17,7 @@ }, { "test": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\nweights = np.array([0.1, 0.2], dtype=np.float32)\ninputs = np.array([[1.0, 1.0]], dtype=np.float32)\ntargets = np.array([0.5], dtype=np.float32)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")", - "expected_output": "Loss: 4.0\nLoss dtype: float32" + "expected_output": "Loss: 4.0\nLoss dtype: float" }, { "test": "import numpy as np\nmp = MixedPrecision(loss_scale=100.0)\ngrads = np.array([200.0, 100.0], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f} {result[1]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", @@ -25,7 +25,7 @@ }, { "test": "import numpy as np\nmp = MixedPrecision(loss_scale=2048.0)\nweights = np.array([0.25], dtype=np.float64)\ninputs = np.array([[4.0]], dtype=np.float64)\ntargets = np.array([2.0], dtype=np.float64)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")", - "expected_output": "Loss: 2048.0\nLoss dtype: float32" + "expected_output": "Loss: 2048.0\nLoss dtype: float" }, { "test": "import numpy as np\nmp = MixedPrecision(loss_scale=2048.0)\ngrads = np.array([np.nan], dtype=np.float16)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")", @@ -33,7 +33,7 @@ }, { "test": "import numpy as np\nmp = MixedPrecision(loss_scale=256.0)\nweights = np.array([1.0], dtype=np.float16)\ninputs = np.array([[2.0]], dtype=np.float16)\ntargets = np.array([3.0], dtype=np.float16)\nloss = mp.forward(weights, inputs, targets)\nprint(f\"Loss: {loss:.1f}\")\nprint(f\"Loss dtype: {type(loss).__name__}\")", - "expected_output": "Loss: 256.0\nLoss dtype: float32" + "expected_output": "Loss: 256.0\nLoss dtype: float" }, { "test": "import numpy as np\nmp = MixedPrecision(loss_scale=256.0)\ngrads = np.array([np.inf], dtype=np.float64)\nresult = mp.backward(grads)\nprint(f\"Gradients: [{result[0]:.0f}]\")\nprint(f\"Grad dtype: {result.dtype}\")",