diff --git a/spacy/util.py b/spacy/util.py
index 3e4db141a..3956d27d7 100644
--- a/spacy/util.py
+++ b/spacy/util.py
@@ -1032,12 +1032,12 @@ def make_tempdir() -> Generator[Path, None, None]:
 
     YIELDS (Path): The path of the temp directory.
     """
-
+    d = Path(tempfile.mkdtemp())
+    yield d
     try:
-        with tempfile.TemporaryDirectory() as td:
-            yield Path(td)
+        shutil.rmtree(str(d))
     except PermissionError as e:
-        warnings.warn(Warnings.W091.format(dir=td, msg=e))
+        warnings.warn(Warnings.W091.format(dir=d, msg=e))
 
 
 def is_cwd(path: Union[Path, str]) -> bool: