Fix tokenizer reading on Windows

2023-07-24 11:08:29 +03:00
parent d548245321
commit 4d637983ad
1 changed files with 1 additions and 1 deletions
@@ -417,7 +417,7 @@ int main(int argc, char *argv[]) {
    // read in the tokenizer.bin file
    char** vocab = (char**)malloc(config.vocab_size * sizeof(char*));
    {
-        FILE *file = fopen("tokenizer.bin", "r");
+        FILE *file = fopen("tokenizer.bin", "rb");
        if (!file) {
            printf("Unable to open the tokenizer file tokenizer.bin! Run "
            "python tokenizer.py to convert tokenizer.model -> tokenizer.bin\n");