schihei/llama2.c
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update run_wrap.py

Browse Source
This commit is contained in:
Artem Yatsenko
2023-07-23 09:28:49 -07:00
committed by GitHub
parent 00727ba1c0
commit 0bddcd94c1
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
run_wrap.py
+5 -4
View File
@@ -17,15 +17,16 @@ enc = Tokenizer()
t0 = time.time() t0 = time.time()
tokens = [] tokens = []
last = ''
for line in proc.stdout: for line in proc.stdout:
token = int(line.decode('utf-8').strip()) token = int(line.decode('utf-8').strip())
dec = enc.decode([token]) dec = enc.decode(tokens + [token])
print(dec, end=" ", flush=True) chunk = dec[len(last):]
print(chunk, end='',flush=True)
tokens.append(token) tokens.append(token)
last = dec
t1 = time.time() t1 = time.time()
print('\n---\n')
print("Sorry I'm not sure why sentencepiece can't stream tokens properly, I'll solve it later. Here is the whole thing properly:")
print('\n---\n') print('\n---\n')
print(enc.decode(tokens)) print(enc.decode(tokens))