schihei/llama2.c
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix tokenizer reading on Windows

Browse Source
This commit is contained in:
Henri Vasserman
2023-07-24 11:08:29 +03:00
committed by GitHub
parent d548245321
commit 4d637983ad
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
run.c
+1 -1
View File
@@ -417,7 +417,7 @@ int main(int argc, char *argv[]) {
// read in the tokenizer.bin file // read in the tokenizer.bin file
char** vocab = (char**)malloc(config.vocab_size * sizeof(char*)); char** vocab = (char**)malloc(config.vocab_size * sizeof(char*));
{ {
FILE *file = fopen("tokenizer.bin", "r"); FILE *file = fopen("tokenizer.bin", "rb");
if (!file) { if (!file) {
printf("Unable to open the tokenizer file tokenizer.bin! Run " printf("Unable to open the tokenizer file tokenizer.bin! Run "
"python tokenizer.py to convert tokenizer.model -> tokenizer.bin\n"); "python tokenizer.py to convert tokenizer.model -> tokenizer.bin\n");