Improve usage of pkg_resources and handling of entry points (#4387)

* Only import pkg_resources where it's needed

Apparently it's really slow

* Use importlib_metadata for entry points

* Revert "Only import pkg_resources where it's needed"

This reverts commit 5ed8c03afa.

* Revert "Revert "Only import pkg_resources where it's needed""

This reverts commit 8b30b57957.

* Revert "Use importlib_metadata for entry points"

This reverts commit 9f071f5c40.

* Revert "Revert "Use importlib_metadata for entry points""

This reverts commit 02e12a17ec.

* Skip test that weirdly hangs

* Fix hanging test by using global

requirements.txt

@@ -11,6 +11,7 @@ numpy>=1.15.0
 requests>=2.13.0,<3.0.0
 plac<1.0.0,>=0.9.6
 pathlib==1.0.1; python_version < "3.4"
+importlib_metadata>=0.23; python_version < "3.8"
 # Optional dependencies
 jsonschema>=2.6.0,<3.1.0
 # Development dependencies

setup.cfg

@@ -50,6 +50,7 @@ install_requires =
     wasabi>=0.2.0,<1.1.0
     srsly>=0.1.0,<1.1.0
     pathlib==1.0.1; python_version < "3.4"
+    importlib_metadata>=0.23; python_version < "3.8"
 
 [options.extras_require]
 lookups =

spacy/cli/download.py

@@ -6,7 +6,6 @@ import requests
 import os
 import subprocess
 import sys
-import pkg_resources
 from wasabi import Printer
 
 from .link import link
@@ -87,6 +86,8 @@ def download(model, direct=False, *pip_args):
 
 def require_package(name):
     try:
+        import pkg_resources
+
         pkg_resources.working_set.require(name)
         return True
     except:  # noqa: E722

spacy/cli/validate.py

@@ -1,7 +1,6 @@
 # coding: utf8
 from __future__ import unicode_literals, print_function
 
-import pkg_resources
 from pathlib import Path
 import sys
 import requests
@@ -109,6 +108,8 @@ def get_model_links(compat):
 
 
 def get_model_pkgs(compat, all_models):
+    import pkg_resources
+
     pkgs = {}
     for pkg_name, pkg_data in pkg_resources.working_set.by_key.items():
         package = pkg_name.replace("-", "_")

spacy/compat.py

@@ -35,6 +35,11 @@ try:
 except ImportError:
     cupy = None
 
+try:  # Python 3.8
+    import importlib.metadata as importlib_metadata
+except ImportError:
+    import importlib_metadata  # noqa: F401
+
 try:
     from thinc.neural.optimizers import Optimizer  # noqa: F401
 except ImportError:

spacy/util.py

@@ -2,7 +2,6 @@
 from __future__ import unicode_literals, print_function
 
 import os
-import pkg_resources
 import importlib
 import re
 from pathlib import Path
@@ -28,7 +27,7 @@ except ImportError:
 
 from .symbols import ORTH
 from .compat import cupy, CudaStream, path2str, basestring_, unicode_
-from .compat import import_file
+from .compat import import_file, importlib_metadata
 from .errors import Errors, Warnings, deprecation_warning
 
 
@@ -37,6 +36,11 @@ _data_path = Path(__file__).parent / "data"
 _PRINT_ENV = False
 
 
+# NB: Ony ever call this once! If called more than ince within the
+# function, test_issue1506 hangs and it's not 100% clear why.
+AVAILABLE_ENTRY_POINTS = importlib_metadata.entry_points()
+
+
 class ENTRY_POINTS(object):
     """Available entry points to register extensions."""
 
@@ -253,6 +257,8 @@ def is_package(name):
     name (unicode): Name of package.
     RETURNS (bool): True if installed package, False if not.
     """
+    import pkg_resources
+
     name = name.lower()  # compare package name against lowercase name
     packages = pkg_resources.working_set.by_key.keys()
     for package in packages:
@@ -282,7 +288,7 @@ def get_entry_points(key):
     RETURNS (dict): Entry points, keyed by name.
     """
     result = {}
-    for entry_point in pkg_resources.iter_entry_points(key):
+    for entry_point in AVAILABLE_ENTRY_POINTS.get(key, []):
         result[entry_point.name] = entry_point.load()
     return result
 
@@ -296,7 +302,7 @@ def get_entry_point(key, value, default=None):
     default: Optional default value to return.
     RETURNS: The loaded entry point or None.
     """
-    for entry_point in pkg_resources.iter_entry_points(key):
+    for entry_point in AVAILABLE_ENTRY_POINTS.get(key, []):
         if entry_point.name == value:
             return entry_point.load()
     return default