diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh
index f00311c13..755459ad7 100755
--- a/.github/workflows/build.sh
+++ b/.github/workflows/build.sh
@@ -4,9 +4,8 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
   # libxdmcp causes an issue on macOS < 11
   # curl from brew requires zstd, use system curl
   # if php is installed, brew tries to reinstall these after installing openblas
-  # remove lcms2 to fix building openjpeg on arm64
-  # remove xmlto to skip building giflib docs
-  brew remove --ignore-dependencies webp zstd xz libtiff libxcb libxdmcp curl php lcms2 xmlto ghostscript
+  # remove lcms2 and libpng to fix building openjpeg on arm64
+  brew remove --ignore-dependencies webp zstd xz libpng libtiff libxcb libxdmcp curl php lcms2 ghostscript
 
   if [[ "$PLAT" == "arm64" ]]; then
     export MACOSX_DEPLOYMENT_TARGET="11.0"
diff --git a/config.sh b/config.sh
index 713ab72f5..84df85e2a 100644
--- a/config.sh
+++ b/config.sh
@@ -87,10 +87,6 @@ function pre_build {
         rm /usr/local/lib/libjpeg.dylib
     fi
     build_tiff
-    if [ -n "$IS_MACOS" ]; then
-        # Remove existing libpng
-        rm /usr/local/lib/libpng*
-    fi
     build_libpng
     build_lcms2
     build_openjpeg