diff --git a/src/_imaging.c b/src/_imaging.c
index 53ab36ccd..64cf62bcb 100644
--- a/src/_imaging.c
+++ b/src/_imaging.c
@@ -1771,7 +1771,13 @@ _putpalette(ImagingObject *self, PyObject *args) {
 
     ImagingPaletteDelete(self->image->palette);
 
-    self->image->mode = strlen(self->image->mode->name) == 2 ? IMAGING_MODE_PA : IMAGING_MODE_P;
+    if (self->image->mode == IMAGING_MODE_LA) {
+        self->image->mode = IMAGING_MODE_PA;
+    } else if (self->image->mode == IMAGING_MODE_L) {
+        self->image->mode = IMAGING_MODE_P;
+    } else {
+        // The image already has a palette mode so we don't need to change it.
+    }
 
     self->image->palette = ImagingPaletteNew(palette_mode);
 
diff --git a/src/libImaging/Matrix.c b/src/libImaging/Matrix.c
index fd5584611..f848b870d 100644
--- a/src/libImaging/Matrix.c
+++ b/src/libImaging/Matrix.c
@@ -46,7 +46,11 @@ ImagingConvertMatrix(Imaging im, const Mode *mode, float m[]) {
             }
         }
         ImagingSectionLeave(&cookie);
-    } else if (strlen(mode->name) == 3) {
+    } else if (
+        mode == IMAGING_MODE_HSV ||
+        mode == IMAGING_MODE_LAB ||
+        mode == IMAGING_MODE_RGB
+    ) {
         imOut = ImagingNewDirty(mode, im->xsize, im->ysize);
         if (!imOut) {
             return NULL;