schihei/llama2.c
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

get rid of unneeded comment now

Browse Source
This commit is contained in:
Andrej Karpathy
2023-08-05 16:19:27 +00:00
parent db4ad580f3
commit 837796e0b7
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model.py
+1 -1
View File
@@ -215,7 +215,7 @@ class Transformer(nn.Module):
# share the unembedding parameters with the embedding parameters # share the unembedding parameters with the embedding parameters
self.tok_embeddings.weight = self.output.weight # https://paperswithcode.com/method/weight-tying self.tok_embeddings.weight = self.output.weight # https://paperswithcode.com/method/weight-tying
# some useful precompute for the RoPE relative positional embeddings. TODO why * 2 here? confuse # some useful precompute for the RoPE relative positional embeddings
freqs_cos, freqs_sin = precompute_freqs_cis(self.params.dim // self.params.n_heads, self.params.max_seq_len) freqs_cos, freqs_sin = precompute_freqs_cis(self.params.dim // self.params.n_heads, self.params.max_seq_len)
self.register_buffer("freqs_cos", freqs_cos, persistent=False) self.register_buffer("freqs_cos", freqs_cos, persistent=False)
self.register_buffer("freqs_sin", freqs_sin, persistent=False) self.register_buffer("freqs_sin", freqs_sin, persistent=False)