Merge pull request #11668 from adrianeboyd/chore/v2.3.8

Updates for python 3.10 and 3.11, set version to v2.3.8

MANIFEST.in

@@ -7,3 +7,4 @@ include pyproject.toml
 recursive-exclude spacy/lang *.json
 recursive-include spacy/lang *.json.gz
 recursive-include licenses *
+recursive-exclude spacy *.cpp

azure-pipelines.yml

@@ -21,7 +21,7 @@ jobs:
 # defined in .flake8 and overwrites the selected codes.
 - job: 'Validate'
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -35,50 +35,37 @@ jobs:
   dependsOn: 'Validate'
   strategy:
     matrix:
-      Python35Linux:
-        imageName: 'ubuntu-18.04'
-        python.version: '3.5'
-        os: linux
-      Python35Windows:
-        imageName: 'vs2017-win2016'
-        python.version: '3.5'
-# Test on one OS per python 3.6/3.7/3.8 to speed up CI
+# Test on one OS per python 3.6/3.7/3.8/3.9 to speed up CI
       Python36Linux:
-        imageName: 'ubuntu-18.04'
+        imageName: 'ubuntu-latest'
         python.version: '3.6'
-#      Python36Windows:
-#        imageName: 'vs2017-win2016'
-#        python.version: '3.6'
-#      Python36Mac:
-#        imageName: 'macos-10.14'
-#        python.version: '3.6'
-#      Python37Linux:
-#        imageName: 'ubuntu-18.04'
-#        python.version: '3.7'
       Python37Windows:
-        imageName: 'vs2017-win2016'
+        imageName: 'windows-latest'
         python.version: '3.7'
-#      Python37Mac:
-#        imageName: 'macos-10.14'
-#        python.version: '3.7'
-#      Python38Linux:
-#        imageName: 'ubuntu-18.04'
-#        python.version: '3.8'
-#      Python38Windows:
-#        imageName: 'vs2017-win2016'
-#        python.version: '3.8'
       Python38Mac:
-        imageName: 'macos-10.14'
+        imageName: 'macos-latest'
         python.version: '3.8'
       Python39Linux:
-        imageName: 'ubuntu-18.04'
-        python.version: '3.9'
-      Python39Windows:
-        imageName: 'vs2017-win2016'
-        python.version: '3.9'
-      Python39Mac:
-        imageName: 'macos-10.14'
+        imageName: 'ubuntu-latest'
         python.version: '3.9'
+      Python310Linux:
+        imageName: 'ubuntu-latest'
+        python.version: '3.10'
+      Python310Windows:
+        imageName: 'windows-latest'
+        python.version: '3.10'
+      Python310Mac:
+        imageName: 'macos-latest'
+        python.version: '3.10'
+      Python311Linux:
+        imageName: 'ubuntu-latest'
+        python.version: '3.11.0-rc.2'
+      Python311Windows:
+        imageName: 'windows-latest'
+        python.version: '3.11.0-rc.2'
+      Python311Mac:
+        imageName: 'macos-latest'
+        python.version: '3.11.0-rc.2'
     maxParallel: 4
   pool:
     vmImage: $(imageName)
@@ -88,17 +75,13 @@ jobs:
     inputs:
       versionSpec: '$(python.version)'
       architecture: 'x64'
+      allowUnstable: true
 
   - script: python -m pip install -U pip setuptools
     displayName: 'Update pip'
 
-  - script: pip install -r requirements.txt --prefer-binary
-    displayName: 'Install dependencies (python 3.5: prefer binary)'
-    condition: eq(variables['python.version'], '3.5')
-
   - script: pip install -r requirements.txt
     displayName: 'Install dependencies'
-    condition: not(eq(variables['python.version'], '3.5'))
 
   - script: |
       python setup.py build_ext --inplace -j 2
@@ -115,20 +98,13 @@ jobs:
       pip uninstall -y -r installed.txt
     displayName: 'Uninstall all packages'
 
-  - bash: |
-      SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1)
-      pip install dist/$SDIST --prefer-binary
-    displayName: 'Install from sdist (python 3.5: prefer binary)'
-    condition: eq(variables['python.version'], '3.5')
-
   - bash: |
       SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1)
       pip install dist/$SDIST
     displayName: 'Install from sdist'
-    condition: not(eq(variables['python.version'], '3.5'))
 
   - script: |
-      pip install -r requirements.txt --prefer-binary
+      pip install -r requirements.txt
       python -m pytest --pyargs spacy
     displayName: 'Run tests'
 
@@ -136,4 +112,4 @@ jobs:
       python -m spacy download en_core_web_sm
       python -c "import spacy; nlp=spacy.load('en_core_web_sm'); doc=nlp('test')"
     displayName: 'Test download CLI'
-    condition: and(eq(variables['python.version'], '3.9'), eq(variables['imageName'], 'ubuntu-18.04'))
+    condition: eq(variables['python.version'], '3.9')

build-constraints.txt

@@ -1,5 +1,9 @@
 # build version constraints for use with wheelwright + multibuild
-numpy==1.15.0; python_version<='3.7'
-numpy==1.17.3; python_version=='3.8'
+numpy==1.15.0; python_version<='3.7' and platform_machine!='aarch64'
+numpy==1.19.2; python_version<='3.7' and platform_machine=='aarch64'
+numpy==1.17.3; python_version=='3.8' and platform_machine!='aarch64'
+numpy==1.19.2; python_version=='3.8' and platform_machine=='aarch64'
 numpy==1.19.3; python_version=='3.9'
-numpy; python_version>='3.10'
+numpy==1.21.3; python_version=='3.10'
+numpy==1.23.2; python_version=='3.11'
+numpy; python_version>='3.12'

pyproject.toml

@@ -1,7 +1,7 @@
 [build-system]
 requires = [
     "setuptools",
-    "cython>=0.25",
+    "cython>=0.25,<3.0",
     "cymem>=2.0.2,<2.1.0",
     "preshed>=3.0.2,<3.1.0",
     "murmurhash>=0.28.0,<1.1.0",

requirements.txt

@@ -17,8 +17,8 @@ tqdm>=4.38.0,<5.0.0
 pyrsistent<0.17.0
 jsonschema>=2.6.0,<3.1.0
 # Development dependencies
-cython>=0.25
+cython>=0.25,<3.0
 pytest>=4.6.5
 pytest-timeout>=1.3.0,<2.0.0
 mock>=2.0.0,<3.0.0
-flake8>=3.5.0,<3.6.0
+flake8>=3.5.0,<6.0.0

setup.cfg

@@ -24,6 +24,8 @@ classifiers =
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
     Topic :: Scientific/Engineering
 
 [options]
@@ -33,7 +35,7 @@ scripts =
     bin/spacy
 python_requires = >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*
 setup_requires =
-    cython>=0.25
+    cython>=0.25,<3.0
     numpy>=1.15.0
     # We also need our Cython packages here to compile against
     cymem>=2.0.2,<2.1.0

setup.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python
 from __future__ import print_function
-import io
 import os
 import subprocess
 import sys
-import contextlib
 import numpy
+from pathlib import Path
 from distutils.command.build_ext import build_ext
 from distutils.sysconfig import get_python_inc
 import distutils.util
 from distutils import ccompiler, msvccompiler
 from setuptools import Extension, setup, find_packages
+from Cython.Build import cythonize
+from Cython.Compiler import Options
 
 
 def is_new_osx():
@@ -28,6 +29,10 @@ def is_new_osx():
         return False
 
 
+# Preserve `__doc__` on functions and classes
+# http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-options
+Options.docstrings = True
+
 PACKAGES = find_packages()
 
 
@@ -74,6 +79,12 @@ COMPILE_OPTIONS = {
 
 LINK_OPTIONS = {"msvc": [], "mingw32": [], "other": []}
 
+COMPILER_DIRECTIVES = {
+    "language_level": -3,
+    "embedsignature": True,
+    "annotation_typing": False,
+}
+
 
 if is_new_osx():
     # On Mac, use libc++ because Apple deprecated use of
@@ -105,20 +116,6 @@ class build_ext_subclass(build_ext, build_ext_options):
         build_ext.build_extensions(self)
 
 
-def generate_cython(root, source):
-    print("Cythonizing sources")
-    p = subprocess.call(
-        [sys.executable, os.path.join(root, "bin", "cythonize.py"), source],
-        env=os.environ,
-    )
-    if p != 0:
-        raise RuntimeError("Running cythonize failed")
-
-
-def is_source_release(path):
-    return os.path.exists(os.path.join(path, "PKG-INFO"))
-
-
 # Include the git version in the build (adapted from NumPy)
 # Copyright (c) 2005-2020, NumPy Developers.
 # BSD 3-Clause license, see licenses/3rd_party_licenses.txt
@@ -169,75 +166,41 @@ GIT_VERSION = "%(git_version)s"
 
 
 def clean(path):
-    for name in MOD_NAMES:
-        name = name.replace(".", "/")
-        for ext in [".so", ".html", ".cpp", ".c"]:
-            file_path = os.path.join(path, name + ext)
-            if os.path.exists(file_path):
-                os.unlink(file_path)
-
-
-@contextlib.contextmanager
-def chdir(new_dir):
-    old_dir = os.getcwd()
-    try:
-        os.chdir(new_dir)
-        sys.path.insert(0, new_dir)
-        yield
-    finally:
-        del sys.path[0]
-        os.chdir(old_dir)
+    for path in path.glob("**/*"):
+        if path.is_file() and path.suffix in (".so", ".cpp"):
+            print(f"Deleting {path.name}")
+            path.unlink()
 
 
 def setup_package():
     write_git_info_py()
 
-    root = os.path.abspath(os.path.dirname(__file__))
+    root = Path(__file__).parent
 
     if hasattr(sys, "argv") and len(sys.argv) > 1 and sys.argv[1] == "clean":
-        return clean(root)
+        return clean(root / "spacy")
 
-    with chdir(root):
-        with io.open(os.path.join(root, "spacy", "about.py"), encoding="utf8") as f:
+    with (root / "spacy" / "about.py").open("r") as f:
         about = {}
         exec(f.read(), about)
 
     include_dirs = [
-            numpy.get_include(),
         get_python_inc(plat_specific=True),
-            os.path.join(root, "include"),
+        numpy.get_include(),
+        str(root / "include"),
     ]
-
     if (
         ccompiler.new_compiler().compiler_type == "msvc"
         and msvccompiler.get_build_version() == 9
     ):
-            include_dirs.append(os.path.join(root, "include", "msvc9"))
-
+        include_dirs.append(str(root / "include" / "msvc9"))
     ext_modules = []
-        for mod_name in MOD_NAMES:
-            mod_path = mod_name.replace(".", "/") + ".cpp"
-            extra_link_args = []
-            # ???
-            # Imported from patch from @mikepb
-            # See Issue #267. Running blind here...
-            if sys.platform == "darwin":
-                dylib_path = [".." for _ in range(mod_name.count("."))]
-                dylib_path = "/".join(dylib_path)
-                dylib_path = "@loader_path/%s/spacy/platform/darwin/lib" % dylib_path
-                extra_link_args.append("-Wl,-rpath,%s" % dylib_path)
-            ext_modules.append(
-                Extension(
-                    mod_name,
-                    [mod_path],
-                    language="c++",
-                    include_dirs=include_dirs,
-                    extra_link_args=extra_link_args,
-                )
-            )
-
-        if not is_source_release(root):
-            generate_cython(root, "spacy")
+    for name in MOD_NAMES:
+        mod_path = name.replace(".", "/") + ".pyx"
+        ext = Extension(name, [mod_path], language="c++")
+        ext_modules.append(ext)
+    print("Cythonizing sources")
+    ext_modules = cythonize(ext_modules, compiler_directives=COMPILER_DIRECTIVES)
 
     setup(
         name="spacy",
@@ -245,6 +208,8 @@ def setup_package():
         version=about["__version__"],
         ext_modules=ext_modules,
         cmdclass={"build_ext": build_ext_subclass},
+        include_dirs=include_dirs,
+        package_data={"": ["*.pyx", "*.pxd", "*.pxi", "*.cpp"]},
     )
 
 

spacy/about.py

@@ -1,6 +1,6 @@
 # fmt: off
 __title__ = "spacy"
-__version__ = "2.3.7"
+__version__ = "2.3.8"
 __release__ = True
 __download_url__ = "https://github.com/explosion/spacy-models/releases/download"
 __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"