diff --git a/setup.py b/setup.py
index 5ecd6b816..6215b76ff 100644
--- a/setup.py
+++ b/setup.py
@@ -14,6 +14,7 @@ import shutil
 import struct
 import subprocess
 import sys
+import sysconfig
 import warnings
 from collections.abc import Iterator
 from typing import Any
@@ -552,8 +553,16 @@ class pil_build_ext(build_ext):
                 for d in os.environ[k].split(os.path.pathsep):
                     _add_directory(library_dirs, d)
 
-        _add_directory(library_dirs, os.path.join(sys.prefix, "lib"))
-        _add_directory(include_dirs, os.path.join(sys.prefix, "include"))
+        _add_directory(
+            library_dirs,
+            (sys.prefix == sys.base_prefix and sysconfig.get_config_var("LIBDIR"))
+            or os.path.join(sys.prefix, "lib"),
+        )
+        _add_directory(
+            include_dirs,
+            (sys.prefix == sys.base_prefix and sysconfig.get_config_var("INCLUDEDIR"))
+            or os.path.join(sys.prefix, "include"),
+        )
 
         #
         # add platform directories