diff --git a/setup.py b/setup.py
index 93918debd..50ca985e3 100644
--- a/setup.py
+++ b/setup.py
@@ -205,25 +205,31 @@ class pil_build_ext(build_ext):
             # darwin ports installation directories
             _add_directory(library_dirs, "/opt/local/lib")
             _add_directory(include_dirs, "/opt/local/include")
-            
-            # if homebrew is installed, use its lib and include directories
+
+            # if Homebrew is installed, use its lib and include directories
             import subprocess
             try:
-                prefix = subprocess.check_output(['brew', '--prefix'])
-                if prefix:
-                    prefix = prefix.strip()
-                    _add_directory(library_dirs, os.path.join(prefix, 'lib'))
-                    _add_directory(include_dirs, os.path.join(prefix, 'include'))
-                    
-                    # freetype2 is a key-only brew under opt/
-                    _add_directory(library_dirs, os.path.join(prefix, 'opt', 'freetype', 'lib'))
-                    _add_directory(include_dirs, os.path.join(prefix, 'opt', 'freetype', 'include'))
+                prefix = subprocess.check_output(['brew', '--prefix']).strip()
             except:
-                pass # homebrew not installed
-            
-            # freetype2 ships with X11 (after homebrew, so that homebrew freetype is preferred)
-            _add_directory(library_dirs, "/usr/X11/lib")
-            _add_directory(include_dirs, "/usr/X11/include")
+                # Homebrew not installed
+                prefix = None
+
+            ft_prefix = None
+
+            if prefix:
+                # add Homebrew's include and lib directories
+                _add_directory(library_dirs, os.path.join(prefix, 'lib'))
+                _add_directory(include_dirs, os.path.join(prefix, 'include'))
+                ft_prefix = os.path.join(prefix, 'opt', 'freetype')
+
+            if ft_prefix and os.path.isdir(ft_prefix):
+                # freetype might not be linked into Homebrew's prefix
+                _add_directory(library_dirs, os.path.join(ft_prefix, 'lib'))
+                _add_directory(include_dirs, os.path.join(ft_prefix, 'include'))
+            else:
+                # fall back to freetype from XQuartz if Homebrew's freetype is missing
+                _add_directory(library_dirs, "/usr/X11/lib")
+                _add_directory(include_dirs, "/usr/X11/include")
 
         elif sys.platform.startswith("linux"):
             arch_tp = (plat.processor(), plat.architecture()[0])