diff --git a/PIL/GifImagePlugin.py b/PIL/GifImagePlugin.py
index 0b154f417..35dbc5c68 100644
--- a/PIL/GifImagePlugin.py
+++ b/PIL/GifImagePlugin.py
@@ -433,7 +433,7 @@ def getheader(im, palette=None, info=None):
 
     usedPaletteColors = paletteBytes = None
 
-    if optimize:
+    if im.mode == "P" and optimize:
         usedPaletteColors = []
 
         # check which colors are used