schihei/llama2.c
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #306 from rdentato/patch-utf8-no-validation

Browse Source 
minimal protection against invalid UTF8 encoding.
This commit is contained in:
Andrej
2023-08-16 09:51:11 -07:00
committed by GitHub
parent df6557a10d 55e60740f5
commit 57bf0e9ee4
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
run.c
+4 -2
View File
@@ -362,7 +362,7 @@ void bpe_encode(char *text, char **vocab, float *vocab_scores, int vocab_size, u
qsort(sorted_vocab, vocab_size, sizeof(TokenIndex), compare_tokens); qsort(sorted_vocab, vocab_size, sizeof(TokenIndex), compare_tokens);
// create a temporary buffer that will store merge candidates of always two consecutive tokens // create a temporary buffer that will store merge candidates of always two consecutive tokens
char* str_buffer = malloc((max_token_length*2+1) * sizeof(char)); // *2 for concat, +1 for null terminator char* str_buffer = malloc((max_token_length*2 +1 +2) * sizeof(char)); // *2 for concat, +1 for null terminator +2 for UTF8 (in case max_token_lenght is 1)
size_t str_len = 0; size_t str_len = 0;
// add_dummy_prefix is true by default // add_dummy_prefix is true by default
@@ -396,7 +396,8 @@ void bpe_encode(char *text, char **vocab, float *vocab_scores, int vocab_size, u
str_buffer[str_len] = '\0'; str_buffer[str_len] = '\0';
// while the next character is a continuation byte, continue appending // while the next character is a continuation byte, continue appending
if ((*(c+1) & 0xC0) == 0x80) { // but if there are too many of them, just stop to avoid overruning str_buffer size.
if ((*(c+1) & 0xC0) == 0x80 && str_len < 4) {
continue; continue;
} }
@@ -414,6 +415,7 @@ void bpe_encode(char *text, char **vocab, float *vocab_scores, int vocab_size, u
tokens[(*n_tokens)++] = (unsigned char)str_buffer[i] + 3; tokens[(*n_tokens)++] = (unsigned char)str_buffer[i] + 3;
} }
} }
str_len = 0; // protect against a sequence of stray UTF8 continuation bytes
} }
// merge the best consecutive pair each iteration, according the scores in vocab_scores // merge the best consecutive pair each iteration, according the scores in vocab_scores