schihei/llama2.c
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

extra line

Browse Source
This commit is contained in:
Michael Cusack
2023-08-04 16:52:04 +07:00
parent f2e34e6b0a
commit 11a8348dfc
1 changed files with 0 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model.py
-1
View File
@@ -108,7 +108,6 @@ class Attention(nn.Module):
# use flash attention or a manual implementation? # use flash attention or a manual implementation?
self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention') self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention')
if not self.flash: if not self.flash:
print("WARNING: using slow attention. Flash Attention requires PyTorch >= 2.0") print("WARNING: using slow attention. Flash Attention requires PyTorch >= 2.0")
mask = torch.full((1, 1, args.max_seq_len, args.max_seq_len), float("-inf")) mask = torch.full((1, 1, args.max_seq_len, args.max_seq_len), float("-inf"))