schihei/llama2.c
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove freq_cis fields as they are not used anymore

Browse Source
This commit is contained in:
Andrej Karpathy
2023-08-20 17:26:43 +00:00
parent c0511de617
commit 1e335a41cf
1 changed files with 2 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
run.c
+2 -7
View File
@@ -43,9 +43,6 @@ typedef struct {
float* w3; // (layer, hidden_dim, dim) float* w3; // (layer, hidden_dim, dim)
// final rmsnorm // final rmsnorm
float* rms_final_weight; // (dim,) float* rms_final_weight; // (dim,)
// freq_cis for RoPE relatively positional embeddings (not used anymore)
float* freq_cis_real; // (seq_len, head_size/2)
float* freq_cis_imag; // (seq_len, head_size/2)
// (optional) classifier weights for the logits, on the last layer // (optional) classifier weights for the logits, on the last layer
float* wcls; float* wcls;
} TransformerWeights; } TransformerWeights;
@@ -133,10 +130,8 @@ void checkpoint_init_weights(TransformerWeights *w, Config* p, float* ptr, int s
ptr += p->n_layers * p->dim * p->hidden_dim; ptr += p->n_layers * p->dim * p->hidden_dim;
w->rms_final_weight = ptr; w->rms_final_weight = ptr;
ptr += p->dim; ptr += p->dim;
w->freq_cis_real = ptr; ptr += p->seq_len * head_size / 2; // skip what used to be freq_cis_real (for RoPE)
ptr += p->seq_len * head_size / 2; ptr += p->seq_len * head_size / 2; // skip what used to be freq_cis_imag (for RoPE)
w->freq_cis_imag = ptr;
ptr += p->seq_len * head_size / 2;
w->wcls = shared_weights ? w->token_embedding_table : ptr; w->wcls = shared_weights ? w->token_embedding_table : ptr;
} }