diff --git a/PIL/Image.py b/PIL/Image.py
index e1d88fe59..ec5ff548d 100644
--- a/PIL/Image.py
+++ b/PIL/Image.py
@@ -475,7 +475,7 @@ class Image:
         new.mode = im.mode
         new.size = im.size
         new.palette = self.palette
-        if im.mode == "P":
+        if im.mode == "P" and not new.palette:
             from PIL import ImagePalette
             new.palette = ImagePalette.ImagePalette()
         try: