diff --git a/Tests/test_imagepalette.py b/Tests/test_imagepalette.py
index 0ea2472a9..17d32cbf3 100644
--- a/Tests/test_imagepalette.py
+++ b/Tests/test_imagepalette.py
@@ -7,7 +7,9 @@ from .helper import assert_image_equal_tofile
 
 def test_sanity():
 
-    ImagePalette.ImagePalette("RGB", list(range(256)) * 3)
+    palette = ImagePalette.ImagePalette("RGB", list(range(256)) * 3)
+    assert len(palette.colors) == 256
+
     with pytest.raises(ValueError):
         ImagePalette.ImagePalette("RGB", list(range(256)) * 2)
 
diff --git a/src/PIL/ImagePalette.py b/src/PIL/ImagePalette.py
index d0604112f..4d223cc06 100644
--- a/src/PIL/ImagePalette.py
+++ b/src/PIL/ImagePalette.py
@@ -40,13 +40,26 @@ class ImagePalette:
         self.mode = mode
         self.rawmode = None  # if set, palette contains raw data
         self.palette = palette or bytearray(range(256)) * len(self.mode)
-        self.colors = {}
         self.dirty = None
         if (size == 0 and len(self.mode) * 256 != len(self.palette)) or (
             size != 0 and size != len(self.palette)
         ):
             raise ValueError("wrong palette size")
 
+    @property
+    def palette(self):
+        return self._palette
+
+    @palette.setter
+    def palette(self, palette):
+        self._palette = palette
+
+        mode_len = len(self.mode)
+        self.colors = {
+            tuple(self.palette[i : i + mode_len]): i // mode_len
+            for i in range(0, len(self.palette), mode_len)
+        }
+
     def copy(self):
         new = ImagePalette()
 
@@ -54,7 +67,6 @@ class ImagePalette:
         new.rawmode = self.rawmode
         if self.palette is not None:
             new.palette = self.palette[:]
-        new.colors = self.colors.copy()
         new.dirty = self.dirty
 
         return new
@@ -100,14 +112,14 @@ class ImagePalette:
             except KeyError as e:
                 # allocate new color slot
                 if isinstance(self.palette, bytes):
-                    self.palette = bytearray(self.palette)
+                    self._palette = bytearray(self.palette)
                 index = len(self.colors)
                 if index >= 256:
                     raise ValueError("cannot allocate more than 256 colors") from e
                 self.colors[color] = index
-                self.palette[index] = color[0]
-                self.palette[index + 256] = color[1]
-                self.palette[index + 512] = color[2]
+                self._palette[index] = color[0]
+                self._palette[index + 256] = color[1]
+                self._palette[index + 512] = color[2]
                 self.dirty = 1
                 return index
         else: