diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml
index 50e81799e..543804b9f 100644
--- a/.github/azure-steps.yml
+++ b/.github/azure-steps.yml
@@ -100,3 +100,8 @@ steps:
       python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113
     displayName: 'Test assemble CLI vectors warning'
     condition: eq(variables['python_version'], '3.8')
+
+  - script: |
+      python .github/validate_universe_json.py website/meta/universe.json
+    displayName: 'Test website/meta/universe.json'
+    condition: eq(variables['python_version'], '3.8')
diff --git a/spacy/tests/universe/test_universe_json.py b/.github/validate_universe_json.py
similarity index 72%
rename from spacy/tests/universe/test_universe_json.py
rename to .github/validate_universe_json.py
index 295889186..b96b7b347 100644
--- a/spacy/tests/universe/test_universe_json.py
+++ b/.github/validate_universe_json.py
@@ -1,13 +1,11 @@
 import json
 import re
+import sys
 from pathlib import Path
 
 
-def test_universe_json():
-
-    root_dir = Path(__file__).parent
-    universe_file = root_dir / "universe.json"
-
+def validate_json(document):
+    universe_file = Path(document)
     with universe_file.open() as f:
         universe_data = json.load(f)
         for entry in universe_data["resources"]:
@@ -15,3 +13,7 @@ def test_universe_json():
                 assert not re.match(
                     r"^(http:)|^(https:)", entry["github"]
                 ), "Github field should be user/repo, not a url"
+
+
+if __name__ == "__main__":
+    validate_json(str(sys.argv[1]))
diff --git a/.gitignore b/.gitignore
index 60036a475..ac72f2bbf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,7 +9,6 @@ keys/
 spacy/tests/package/setup.cfg
 spacy/tests/package/pyproject.toml
 spacy/tests/package/requirements.txt
-spacy/tests/universe/universe.json
 
 # Website
 website/.cache/
diff --git a/setup.py b/setup.py
index 03a1e01dd..fcc124a43 100755
--- a/setup.py
+++ b/setup.py
@@ -81,7 +81,6 @@ COPY_FILES = {
     ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package",
     ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package",
     ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package",
-    ROOT / "website" / "meta" / "universe.json": PACKAGE_ROOT / "tests" / "universe",
 }