diff --git a/Tests/test_file_gif.py b/Tests/test_file_gif.py
index c261cfb97..2bfaef4ee 100644
--- a/Tests/test_file_gif.py
+++ b/Tests/test_file_gif.py
@@ -180,6 +180,21 @@ def test_optimize_full_l():
     assert im.mode == "L"
 
 
+def test_optimize_if_palette_can_be_reduced_by_half():
+    with Image.open("Tests/images/test.colors.gif") as im:
+        # Reduce because original is too big for _get_optimize()
+        im = im.resize((591, 443))
+        imrgb = im.convert("RGB")
+        out = BytesIO()
+        imrgb.save(out, "GIF", optimize=False)
+        with Image.open(out) as reloaded:
+            assert len(reloaded.palette.palette) // 3 == 256
+        out = BytesIO()
+        imrgb.save(out, "GIF", optimize=True)
+        with Image.open(out) as reloaded:
+            assert len(reloaded.palette.palette) // 3 == 8
+
+
 def test_roundtrip(tmp_path):
     out = str(tmp_path / "temp.gif")
     im = hopper()
diff --git a/src/PIL/GifImagePlugin.py b/src/PIL/GifImagePlugin.py
index 3469199ca..f58146c7e 100644
--- a/src/PIL/GifImagePlugin.py
+++ b/src/PIL/GifImagePlugin.py
@@ -824,9 +824,14 @@ def _get_optimize(im, info):
                 if count:
                     used_palette_colors.append(i)
 
+            num_palette_colors = len(im.palette.palette) // 4 if im.palette.mode == 'RGBA' else len(im.palette.palette) // 3
+            # Round up to power of 2 but at least 4
+            num_palette_colors = max(4, 1 << (num_palette_colors - 1).bit_length())
             if optimise or (
+
                 len(used_palette_colors) <= 128
-                and max(used_palette_colors) > len(used_palette_colors)
+                and max(used_palette_colors) >= len(used_palette_colors)
+                or len(used_palette_colors) <= num_palette_colors // 2
             ):
                 return used_palette_colors