From fbefeec1b1e215206060079e3abe7b9bf95ea548 Mon Sep 17 00:00:00 2001 From: rahulschand Date: Sat, 19 Aug 2023 13:05:26 +0530 Subject: [PATCH] add assert message to give better warning --- tinystories.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tinystories.py b/tinystories.py index 690cb02..90d576b 100644 --- a/tinystories.py +++ b/tinystories.py @@ -196,6 +196,7 @@ class PretokDataset(torch.utils.data.IterableDataset): shard_filenames = sorted(glob.glob(os.path.join(bin_dir, "*.bin"))) # train/test split. let's use only shard 0 for test split, rest train shard_filenames = shard_filenames[1:] if self.split == "train" else shard_filenames[:1] + assert len(shard_filenames)>0, f"No bin files found in {bin_dir}" while True: rng.shuffle(shard_filenames) for shard in shard_filenames: