Small changes to ROPE & comments

2023-08-03 20:13:50 +05:30
parent af8708d87b
commit 02cf3c7311
2 changed files with 3 additions and 3 deletions
@@ -51,8 +51,8 @@ typedef struct {
    // final rmsnorm
    float* rms_final_weight; // (dim,)
    // freq_cis for RoPE relatively positional embeddings
-    float* freq_cis_real; // (seq_len, dim/2)
-    float* freq_cis_imag; // (seq_len, dim/2)
+    float* freq_cis_real; // (seq_len, head_size/2)
+    float* freq_cis_imag; // (seq_len, head_size/2)
    // (optional) classifier weights for the logits, on the last layer
    float* wcls;
 } TransformerWeights;