diff --git a/PIL/GifImagePlugin.py b/PIL/GifImagePlugin.py
index 0b154f417..55aece38c 100644
--- a/PIL/GifImagePlugin.py
+++ b/PIL/GifImagePlugin.py
@@ -433,7 +433,7 @@ def getheader(im, palette=None, info=None):
 
     usedPaletteColors = paletteBytes = None
 
-    if optimize:
+    if im.mode in ("P", "L") and optimize:
         usedPaletteColors = []
 
         # check which colors are used
diff --git a/Tests/test_file_gif.py b/Tests/test_file_gif.py
index 200b48372..57cd71ac6 100644
--- a/Tests/test_file_gif.py
+++ b/Tests/test_file_gif.py
@@ -28,13 +28,22 @@ class TestFileGif(PillowTestCase):
     def test_optimize(self):
         from io import BytesIO
 
-        def test(optimize):
+        def test_grayscale(optimize):
             im = Image.new("L", (1, 1), 0)
             file = BytesIO()
             im.save(file, "GIF", optimize=optimize)
             return len(file.getvalue())
-        self.assertEqual(test(0), 800)
-        self.assertEqual(test(1), 38)
+
+        def test_bilevel(optimize):
+            im = Image.new("1", (1, 1), 0)
+            file = BytesIO()
+            im.save(file, "GIF", optimize=optimize)
+            return len(file.getvalue())
+
+        self.assertEqual(test_grayscale(0), 800)
+        self.assertEqual(test_grayscale(1), 38)
+        self.assertEqual(test_bilevel(0), 800)
+        self.assertEqual(test_bilevel(1), 800)
 
     def test_optimize_full_l(self):
         from io import BytesIO