schihei/llama2.c
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix bug, have to use raw_model not model to access the loss

Browse Source
This commit is contained in:
Andrej Karpathy
2023-08-06 07:55:46 +00:00
parent 65b0846637
commit 623894f5da
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
train.py
+2 -2
View File
@@ -212,7 +212,7 @@ def estimate_loss():
X, Y = next(batch_iter) X, Y = next(batch_iter)
with ctx: with ctx:
logits = model(X, Y) logits = model(X, Y)
loss = model.last_loss loss = raw_model.last_loss
losses[k] = loss.item() losses[k] = loss.item()
out[split] = losses.mean() out[split] = losses.mean()
model.train() model.train()
@@ -296,7 +296,7 @@ while True:
model.require_backward_grad_sync = micro_step == gradient_accumulation_steps - 1 model.require_backward_grad_sync = micro_step == gradient_accumulation_steps - 1
with ctx: with ctx:
logits = model(X, Y) logits = model(X, Y)
loss = model.last_loss loss = raw_model.last_loss
loss = loss / gradient_accumulation_steps loss = loss / gradient_accumulation_steps
# immediately async prefetch next batch while model is doing the forward pass on the GPU # immediately async prefetch next batch while model is doing the forward pass on the GPU
X, Y = next(train_batch_iter) X, Y = next(train_batch_iter)