diff --git a/.ci/test.sh b/.ci/test.sh
index 7507c83ff..516581ff0 100755
--- a/.ci/test.sh
+++ b/.ci/test.sh
@@ -2,7 +2,7 @@
 
 set -e
 
-python -m pytest -v -x --cov PIL --cov Tests --cov-report term Tests
+python -m pytest -v -x -W always --cov PIL --cov Tests --cov-report term Tests
 
 # Docs
 if [ "$TRAVIS_PYTHON_VERSION" == "3.8" ] && [ "$TRAVIS_CPU_ARCH" == "amd64" ]; then
diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml
index 164461f03..9853d425e 100644
--- a/.github/workflows/test-windows.yml
+++ b/.github/workflows/test-windows.yml
@@ -335,7 +335,7 @@ jobs:
         rem Add libraqm.dll (copied to INCLIB) to PATH.
         path %INCLIB%;%PATH%
         cd /D %GITHUB_WORKSPACE%
-        %PYTHON%\python.exe -m pytest -vx --cov PIL --cov Tests --cov-report term --cov-report xml Tests
+        %PYTHON%\python.exe -m pytest -vx -W always --cov PIL --cov Tests --cov-report term --cov-report xml Tests
       shell: cmd
 
     - name: Prepare to upload errors
diff --git a/Tests/test_file_jpeg.py b/Tests/test_file_jpeg.py
index eb623b408..59eb7c641 100644
--- a/Tests/test_file_jpeg.py
+++ b/Tests/test_file_jpeg.py
@@ -644,11 +644,11 @@ class TestFileJpeg(PillowTestCase):
 
     def test_invalid_exif_x_resolution(self):
         # When no x or y resolution is defined in EXIF
-        im = Image.open("Tests/images/invalid-exif-without-x-resolution.jpg")
+        with Image.open("Tests/images/invalid-exif-without-x-resolution.jpg") as im:
 
-        # This should return the default, and not a ValueError or
-        # OSError for an unidentified image.
-        self.assertEqual(im.info.get("dpi"), (72, 72))
+            # This should return the default, and not a ValueError or
+            # OSError for an unidentified image.
+            self.assertEqual(im.info.get("dpi"), (72, 72))
 
     def test_ifd_offset_exif(self):
         # Arrange
diff --git a/Tests/test_file_tiff_metadata.py b/Tests/test_file_tiff_metadata.py
index e46e94668..5a9ca9d63 100644
--- a/Tests/test_file_tiff_metadata.py
+++ b/Tests/test_file_tiff_metadata.py
@@ -232,9 +232,9 @@ class TestFileTiffMetadata(PillowTestCase):
         out = self.tempfile("temp.tiff")
         im.save(out, tiffinfo=info, compression="raw")
 
-        reloaded = Image.open(out)
-        self.assertEqual(max_long, reloaded.tag_v2[41493].numerator)
-        self.assertEqual(1, reloaded.tag_v2[41493].denominator)
+        with Image.open(out) as reloaded:
+            self.assertEqual(max_long, reloaded.tag_v2[41493].numerator)
+            self.assertEqual(1, reloaded.tag_v2[41493].denominator)
 
         # out of bounds of 4 byte unsigned long
         numerator = max_long + 1
@@ -244,9 +244,9 @@ class TestFileTiffMetadata(PillowTestCase):
         out = self.tempfile("temp.tiff")
         im.save(out, tiffinfo=info, compression="raw")
 
-        reloaded = Image.open(out)
-        self.assertEqual(max_long, reloaded.tag_v2[41493].numerator)
-        self.assertEqual(1, reloaded.tag_v2[41493].denominator)
+        with Image.open(out) as reloaded:
+            self.assertEqual(max_long, reloaded.tag_v2[41493].numerator)
+            self.assertEqual(1, reloaded.tag_v2[41493].denominator)
 
     def test_ifd_signed_rational(self):
         im = hopper()
@@ -261,9 +261,9 @@ class TestFileTiffMetadata(PillowTestCase):
         out = self.tempfile("temp.tiff")
         im.save(out, tiffinfo=info, compression="raw")
 
-        reloaded = Image.open(out)
-        self.assertEqual(numerator, reloaded.tag_v2[37380].numerator)
-        self.assertEqual(denominator, reloaded.tag_v2[37380].denominator)
+        with Image.open(out) as reloaded:
+            self.assertEqual(numerator, reloaded.tag_v2[37380].numerator)
+            self.assertEqual(denominator, reloaded.tag_v2[37380].denominator)
 
         numerator = -(2 ** 31)
         denominator = 2 ** 31 - 1
@@ -273,9 +273,9 @@ class TestFileTiffMetadata(PillowTestCase):
         out = self.tempfile("temp.tiff")
         im.save(out, tiffinfo=info, compression="raw")
 
-        reloaded = Image.open(out)
-        self.assertEqual(numerator, reloaded.tag_v2[37380].numerator)
-        self.assertEqual(denominator, reloaded.tag_v2[37380].denominator)
+        with Image.open(out) as reloaded:
+            self.assertEqual(numerator, reloaded.tag_v2[37380].numerator)
+            self.assertEqual(denominator, reloaded.tag_v2[37380].denominator)
 
         # out of bounds of 4 byte signed long
         numerator = -(2 ** 31) - 1
@@ -286,9 +286,9 @@ class TestFileTiffMetadata(PillowTestCase):
         out = self.tempfile("temp.tiff")
         im.save(out, tiffinfo=info, compression="raw")
 
-        reloaded = Image.open(out)
-        self.assertEqual(2 ** 31 - 1, reloaded.tag_v2[37380].numerator)
-        self.assertEqual(-1, reloaded.tag_v2[37380].denominator)
+        with Image.open(out) as reloaded:
+            self.assertEqual(2 ** 31 - 1, reloaded.tag_v2[37380].numerator)
+            self.assertEqual(-1, reloaded.tag_v2[37380].denominator)
 
     def test_ifd_signed_long(self):
         im = hopper()
@@ -299,8 +299,8 @@ class TestFileTiffMetadata(PillowTestCase):
         out = self.tempfile("temp.tiff")
         im.save(out, tiffinfo=info, compression="raw")
 
-        reloaded = Image.open(out)
-        self.assertEqual(reloaded.tag_v2[37000], -60000)
+        with Image.open(out) as reloaded:
+            self.assertEqual(reloaded.tag_v2[37000], -60000)
 
     def test_empty_values(self):
         data = io.BytesIO(
diff --git a/tox.ini b/tox.ini
index 07d75be64..cdfd05dde 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,7 +14,7 @@ commands =
     {envpython} setup.py clean
     {envpython} setup.py build_ext --inplace
     {envpython} selftest.py
-    {envpython} -m pytest {posargs}
+    {envpython} -m pytest -W always {posargs}
 deps =
     cffi
     numpy