schihei/llama2.c
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #43 from krzysztof-jusiak/rmsnorm

Browse Source 
Speed up rmsnorm by using sqrtf/expf
This commit is contained in:
Andrej
2023-07-24 14:13:49 -07:00
committed by GitHub
parent 687473c009 c9b1f10124
commit 669b75ddc8
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
run.c
+2 -2
View File
@@ -184,7 +184,7 @@ void rmsnorm(float* o, float* x, float* weight, int size) {
} }
ss /= size; ss /= size;
ss += 1e-5f; ss += 1e-5f;
ss = 1.0f / sqrt(ss); ss = 1.0f / sqrtf(ss);
// normalize and scale // normalize and scale
for (int j = 0; j < size; j++) { for (int j = 0; j < size; j++) {
o[j] = weight[j] * (ss * x[j]); o[j] = weight[j] * (ss * x[j]);
@@ -202,7 +202,7 @@ void softmax(float* x, int size) {
// exp and sum // exp and sum
float sum = 0.0f; float sum = 0.0f;
for (int i = 0; i < size; i++) { for (int i = 0; i < size; i++) {
x[i] = exp(x[i] - max_val); x[i] = expf(x[i] - max_val);
sum += x[i]; sum += x[i];
} }
// normalize // normalize