From d7e2c4691526ac1d30eac44d15121a6b900c938c Mon Sep 17 00:00:00 2001
From: Andrej Karpathy <andrej.karpathy@gmail.com>
Date: Mon, 24 Jul 2023 02:02:12 +0000
Subject: [PATCH] slight tweaks to softmax

---
 run.c | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/run.c b/run.c
index 5bd7e4b..dc5cd9c 100644
--- a/run.c
+++ b/run.c
@@ -196,10 +196,6 @@ void rmsnorm(float* o, float* x, float* weight, int size) {
 }
 
 void softmax(float* x, int size) {
-    if(size == 1) {
-        x[0] = 1.0f;
-        return;
-    }
     // find max value (for numerical stability)
     float max_val = x[0];
     for (int i = 1; i < size; i++) {
@@ -207,14 +203,13 @@ void softmax(float* x, int size) {
             max_val = x[i];
         }
     }
-
-    // normalize
+    // exp and sum
     float sum = 0.0f;
     for (int i = 0; i < size; i++) {
         x[i] = exp(x[i] - max_val);
         sum += x[i];
     }
-
+    // normalize
     for (int i = 0; i < size; i++) {
         x[i] /= sum;
     }