diff --git a/PIL/Image.py b/PIL/Image.py
index 9ee8fa66a..92cac7c9a 100644
--- a/PIL/Image.py
+++ b/PIL/Image.py
@@ -871,18 +871,16 @@ class Image(object):
         :returns: An :py:class:`~PIL.Image.Image` object.
         """
 
-        if not mode:
+        self.load()
+
+        if not mode and self.mode == "P":
             # determine default mode
-            if self.mode == "P":
-                self.load()
-                if self.palette:
-                    mode = self.palette.mode
-                else:
-                    mode = "RGB"
+            if self.palette:
+                mode = self.palette.mode
             else:
-                return self.copy()
-        else:
-            self.load()
+                mode = "RGB"
+        if not mode or (mode == self.mode and not matrix):
+            return self.copy()
 
         if matrix:
             # matrix conversion