diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 13b5be4..f8b216b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -4,10 +4,10 @@ on:
   push:
     branches:
       - master
-    paths: ['.github/workflows/**', '**/Makefile', '**/*.c', '**/*.h']
+    paths: ['.github/workflows/**', '**/Makefile', '**/*.c', '**/*.h', '**/*.py']
   pull_request:
     types: [opened, synchronize, reopened]
-    paths: ['**/Makefile', '**/*.c', '**/*.h']
+    paths: ['**/Makefile', '**/*.c', '**/*.h', '**/*.py']
   # for manual triggering
   workflow_dispatch:
 
@@ -17,7 +17,7 @@ env:
 jobs:
   # check basic builds to avoid breaking changes
   ubuntu-focal-make:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Clone
@@ -30,6 +30,16 @@ jobs:
           sudo apt-get update
           sudo apt-get install build-essential -y
 
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+
+      - name: Pip setup
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
       - name: Build
         id: make_build
         run: |
@@ -40,6 +50,10 @@ jobs:
         run: |
           make runfast
 
+      - name: Test with pytest
+        run: |
+          pytest
+
   macOS-latest-make:
     runs-on: macos-latest
 
diff --git a/test_all.py b/test_all.py
index e8590ea..625af44 100644
--- a/test_all.py
+++ b/test_all.py
@@ -42,6 +42,7 @@ expected_stdout = b'Once upon a time, there was a little girl named Lily. She lo
 
 def test_runc():
     """ Forwards a model against a known-good desired outcome in run.c for 200 steps"""
+    attempt_download_files()
 
     model_path = os.path.join(test_ckpt_dir, "stories260K.bin")
     tokenizer_path = os.path.join(test_ckpt_dir, "tok512.bin")
@@ -56,6 +57,7 @@ def test_runc():
 
 def test_python():
     """ Forwards a model against a known-good desired outcome in sample.py for 200 steps"""
+    attempt_download_files()
 
     device = "cpu" # stories260K is small enough to just breeze through it on CPU
     checkpoint = os.path.join(test_ckpt_dir, "stories260K.pt")