Merge pull request #282 from mihainadas/master-1

Fixes https://github.com/karpathy/llama2.c/issues/280
2023-08-13 10:13:08 -07:00
parent 8b472ded1f 570789aa04
commit 12dec61fbf
1 changed files with 1 additions and 1 deletions
@@ -120,7 +120,7 @@ def train_vocab(vocab_size):

 def process_shard(args, vocab_size):
    shard_id, shard = args
-    tokenizer_model = get_tokenizer_model_path()
+    tokenizer_model = get_tokenizer_model_path(vocab_size)
    enc = Tokenizer(tokenizer_model)
    with open(shard, "r") as f:
        data = json.load(f)