the accum function has been bothering me, there is no real need to add a function here, it does something trivial and is only used twice, scrap
This commit is contained in:
@@ -159,12 +159,6 @@ void checkpoint_init_weights(TransformerWeights *w, Config* p, float* f, int sha
|
|||||||
// ----------------------------------------------------------------------------
|
// ----------------------------------------------------------------------------
|
||||||
// neural net blocks
|
// neural net blocks
|
||||||
|
|
||||||
void accum(float *a, float *b, int size) {
|
|
||||||
for (int i = 0; i < size; i++) {
|
|
||||||
a[i] += b[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void rmsnorm(float* o, float* x, float* weight, int size) {
|
void rmsnorm(float* o, float* x, float* weight, int size) {
|
||||||
// calculate sum of squares
|
// calculate sum of squares
|
||||||
float ss = 0.0f;
|
float ss = 0.0f;
|
||||||
@@ -312,7 +306,9 @@ void transformer(int token, int pos, Config* p, RunState* s, TransformerWeights*
|
|||||||
matmul(s->xb2, s->xb, w->wo + l*dim*dim, dim, dim);
|
matmul(s->xb2, s->xb, w->wo + l*dim*dim, dim, dim);
|
||||||
|
|
||||||
// residual connection back into x
|
// residual connection back into x
|
||||||
accum(x, s->xb2, dim);
|
for (int i = 0; i < dim; i++) {
|
||||||
|
x[i] += s->xb2[i];
|
||||||
|
}
|
||||||
|
|
||||||
// ffn rmsnorm
|
// ffn rmsnorm
|
||||||
rmsnorm(s->xb, x, w->rms_ffn_weight + l*dim, dim);
|
rmsnorm(s->xb, x, w->rms_ffn_weight + l*dim, dim);
|
||||||
@@ -336,7 +332,9 @@ void transformer(int token, int pos, Config* p, RunState* s, TransformerWeights*
|
|||||||
matmul(s->xb, s->hb, w->w2 + l*dim*hidden_dim, hidden_dim, dim);
|
matmul(s->xb, s->hb, w->w2 + l*dim*hidden_dim, hidden_dim, dim);
|
||||||
|
|
||||||
// residual connection
|
// residual connection
|
||||||
accum(x, s->xb, dim);
|
for (int i = 0; i < dim; i++) {
|
||||||
|
x[i] += s->xb[i];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// final rmsnorm
|
// final rmsnorm
|
||||||
|
|||||||
Reference in New Issue
Block a user