diff --git a/spacy/tests/test_cli_app.py b/spacy/tests/test_cli_app.py
index 7ee48b6ff..708e909a3 100644
--- a/spacy/tests/test_cli_app.py
+++ b/spacy/tests/test_cli_app.py
@@ -1,11 +1,8 @@
 import os
 from pathlib import Path
 import pytest
-import shutil
 import srsly
-import stat
 from typer.testing import CliRunner
-import tempfile
 
 from spacy.cli._util import app
 from .util import make_tempdir
@@ -104,23 +101,12 @@ def test_project_run(project_dir):
 
 
 def test_project_clone():
-    # Git clones create some readonly/hidden files. On Windows, trying to
-    # delete these gives an error, so we have to handle that specially.
-
-    workspace = Path(tempfile.mkdtemp())
-
-    def _fix_perms(func, path, exc_info):
-        os.chmod(path, stat.S_IWUSR)
-        func(path)
-
-    try:
+    with make_tempdir() as workspace:
         out = workspace / "project"
         target = "benchmarks/ner_conll03"
         result = CliRunner().invoke(app, ["project", "clone", target, str(out)])
         assert result.exit_code == 0
         assert (out / "README.md").is_file()
-    finally:
-        shutil.rmtree(str(workspace), ignore_errors=False, onerror=_fix_perms)
 
 
 def test_project_push_pull(project_dir):
diff --git a/spacy/util.py b/spacy/util.py
index 1dec50872..55de96093 100644
--- a/spacy/util.py
+++ b/spacy/util.py
@@ -32,6 +32,7 @@ import inspect
 import pkgutil
 import logging
 import socket
+import stat
 
 try:
     import cupy.random
@@ -1032,9 +1033,18 @@ def make_tempdir() -> Generator[Path, None, None]:
     YIELDS (Path): The path of the temp directory.
     """
     d = Path(tempfile.mkdtemp())
+
+    # On Windows, attempts to delete a git directory fail with permissions
+    # errors due to read only / hidden files. This mainly addresses that issue.
+    def _fix_perms(func, path, exc_info):
+        # IWUSR = give user write permission
+        os.chmod(path, stat.S_IWUSR)
+        # func is the function that gave the error, so it behaves like rmtree.
+        func(path)
+
     yield d
     try:
-        shutil.rmtree(str(d))
+        shutil.rmtree(str(d), ignore_errors=False, onerror=_fix_perms)
     except PermissionError as e:
         warnings.warn(Warnings.W091.format(dir=d, msg=e))