From 87b11edf270feefd9606662e73dd2a202c5b4b7a Mon Sep 17 00:00:00 2001 From: Ruhollah Majdoddin Date: Tue, 15 Aug 2023 16:01:53 +0000 Subject: [PATCH] modifiying test_all so it can safely run on windows --- test_all.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/test_all.py b/test_all.py index 625af44..4423cb7 100644 --- a/test_all.py +++ b/test_all.py @@ -30,7 +30,7 @@ def attempt_download_files(): root_url = "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K" need = ["stories260K.bin", "stories260K.pt", "tok512.bin", "tok512.model"] for file in need: - url = os.path.join(root_url, file) + url = root_url + '/' + file #os.path.join inserts \\ on windows filename = os.path.join(test_ckpt_dir, file) if not os.path.exists(filename): download_file(url, filename) @@ -46,13 +46,17 @@ def test_runc(): model_path = os.path.join(test_ckpt_dir, "stories260K.bin") tokenizer_path = os.path.join(test_ckpt_dir, "tok512.bin") - command = ["./run", model_path, "-z", tokenizer_path, "-t", "0.0", "-n", "200"] - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - stdout, stderr = proc.communicate() - + command = ["./run", model_path, "-z", tokenizer_path, "-t", "0.0", "-n", "200"] + with open('err.txt', mode='wb') as fe: + with open('stdout.txt', mode='wb') as fo: + proc = subprocess.Popen(command, stdout=fo, stderr=fe) #pipe in windows terminal does funny things like replacing \n with \r\n + proc.wait() + + with open('stdout.txt', mode='r') as f: + stdout = f.read() # strip the very last \n that is added by run.c for aesthetic reasons - stdout = stdout[:-1] + stdout = stdout[:-1].encode('ascii') + assert stdout == expected_stdout def test_python(): @@ -82,4 +86,6 @@ def test_python(): text = enc.decode(pt_tokens) text = text.encode('ascii') # turn into bytes - assert text == expected_stdout \ No newline at end of file + assert text == expected_stdout + +test_runc() \ No newline at end of file