From 2e91e0738857ed0547fc7569ce0a5e61d8b6f423 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 18 Oct 2022 16:05:58 +0200 Subject: [PATCH 1/5] Update cythonize --- pyproject.toml | 2 +- setup.py | 131 ++++++++++++++++++------------------------------- 2 files changed, 49 insertions(+), 84 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0d6687986..923f313a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] requires = [ "setuptools", - "cython>=0.25", + "cython>=0.25,<3.0", "cymem>=2.0.2,<2.1.0", "preshed>=3.0.2,<3.1.0", "murmurhash>=0.28.0,<1.1.0", diff --git a/setup.py b/setup.py index 2327a0dea..2dd908d14 100755 --- a/setup.py +++ b/setup.py @@ -1,16 +1,17 @@ #!/usr/bin/env python from __future__ import print_function -import io import os import subprocess import sys -import contextlib import numpy +from pathlib import Path from distutils.command.build_ext import build_ext from distutils.sysconfig import get_python_inc import distutils.util from distutils import ccompiler, msvccompiler from setuptools import Extension, setup, find_packages +from Cython.Build import cythonize +from Cython.Compiler import Options def is_new_osx(): @@ -28,6 +29,10 @@ def is_new_osx(): return False +# Preserve `__doc__` on functions and classes +# http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-options +Options.docstrings = True + PACKAGES = find_packages() @@ -74,6 +79,12 @@ COMPILE_OPTIONS = { LINK_OPTIONS = {"msvc": [], "mingw32": [], "other": []} +COMPILER_DIRECTIVES = { + "language_level": -3, + "embedsignature": True, + "annotation_typing": False, +} + if is_new_osx(): # On Mac, use libc++ because Apple deprecated use of @@ -105,20 +116,6 @@ class build_ext_subclass(build_ext, build_ext_options): build_ext.build_extensions(self) -def generate_cython(root, source): - print("Cythonizing sources") - p = subprocess.call( - [sys.executable, os.path.join(root, "bin", "cythonize.py"), source], - env=os.environ, - ) - if p != 0: - raise RuntimeError("Running cythonize failed") - - -def is_source_release(path): - return os.path.exists(os.path.join(path, "PKG-INFO")) - - # Include the git version in the build (adapted from NumPy) # Copyright (c) 2005-2020, NumPy Developers. # BSD 3-Clause license, see licenses/3rd_party_licenses.txt @@ -169,83 +166,51 @@ GIT_VERSION = "%(git_version)s" def clean(path): - for name in MOD_NAMES: - name = name.replace(".", "/") - for ext in [".so", ".html", ".cpp", ".c"]: - file_path = os.path.join(path, name + ext) - if os.path.exists(file_path): - os.unlink(file_path) - - -@contextlib.contextmanager -def chdir(new_dir): - old_dir = os.getcwd() - try: - os.chdir(new_dir) - sys.path.insert(0, new_dir) - yield - finally: - del sys.path[0] - os.chdir(old_dir) + for path in path.glob("**/*"): + if path.is_file() and path.suffix in (".so", ".cpp"): + print(f"Deleting {path.name}") + path.unlink() def setup_package(): write_git_info_py() - root = os.path.abspath(os.path.dirname(__file__)) + root = Path(__file__).parent if hasattr(sys, "argv") and len(sys.argv) > 1 and sys.argv[1] == "clean": - return clean(root) + return clean(root / "spacy") - with chdir(root): - with io.open(os.path.join(root, "spacy", "about.py"), encoding="utf8") as f: - about = {} - exec(f.read(), about) + with (root / "spacy" / "about.py").open("r") as f: + about = {} + exec(f.read(), about) - include_dirs = [ - numpy.get_include(), - get_python_inc(plat_specific=True), - os.path.join(root, "include"), - ] + include_dirs = [ + get_python_inc(plat_specific=True), + numpy.get_include(), + str(root / "include"), + ] + if ( + ccompiler.new_compiler().compiler_type == "msvc" + and msvccompiler.get_build_version() == 9 + ): + include_dirs.append(str(root / "include" / "msvc9")) + ext_modules = [] + for name in MOD_NAMES: + mod_path = name.replace(".", "/") + ".pyx" + ext = Extension(name, [mod_path], language="c++") + ext_modules.append(ext) + print("Cythonizing sources") + ext_modules = cythonize(ext_modules, compiler_directives=COMPILER_DIRECTIVES) - if ( - ccompiler.new_compiler().compiler_type == "msvc" - and msvccompiler.get_build_version() == 9 - ): - include_dirs.append(os.path.join(root, "include", "msvc9")) - - ext_modules = [] - for mod_name in MOD_NAMES: - mod_path = mod_name.replace(".", "/") + ".cpp" - extra_link_args = [] - # ??? - # Imported from patch from @mikepb - # See Issue #267. Running blind here... - if sys.platform == "darwin": - dylib_path = [".." for _ in range(mod_name.count("."))] - dylib_path = "/".join(dylib_path) - dylib_path = "@loader_path/%s/spacy/platform/darwin/lib" % dylib_path - extra_link_args.append("-Wl,-rpath,%s" % dylib_path) - ext_modules.append( - Extension( - mod_name, - [mod_path], - language="c++", - include_dirs=include_dirs, - extra_link_args=extra_link_args, - ) - ) - - if not is_source_release(root): - generate_cython(root, "spacy") - - setup( - name="spacy", - packages=PACKAGES, - version=about["__version__"], - ext_modules=ext_modules, - cmdclass={"build_ext": build_ext_subclass}, - ) + setup( + name="spacy", + packages=PACKAGES, + version=about["__version__"], + ext_modules=ext_modules, + cmdclass={"build_ext": build_ext_subclass}, + include_dirs=include_dirs, + package_data={"": ["*.pyx", "*.pxd", "*.pxi", "*.cpp"]}, + ) if __name__ == "__main__": From b32e1433262900ee49d30c6925e6cc9638a283a6 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 18 Oct 2022 15:21:51 +0200 Subject: [PATCH 2/5] Update package for python 3.10 and 3.11 --- MANIFEST.in | 1 + build-constraints.txt | 10 +++++++--- requirements.txt | 2 +- setup.cfg | 4 +++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 9819c7b70..b7cb786d6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,3 +7,4 @@ include pyproject.toml recursive-exclude spacy/lang *.json recursive-include spacy/lang *.json.gz recursive-include licenses * +recursive-exclude spacy *.cpp diff --git a/build-constraints.txt b/build-constraints.txt index 23e660096..c1e82f1b0 100644 --- a/build-constraints.txt +++ b/build-constraints.txt @@ -1,5 +1,9 @@ # build version constraints for use with wheelwright + multibuild -numpy==1.15.0; python_version<='3.7' -numpy==1.17.3; python_version=='3.8' +numpy==1.15.0; python_version<='3.7' and platform_machine!='aarch64' +numpy==1.19.2; python_version<='3.7' and platform_machine=='aarch64' +numpy==1.17.3; python_version=='3.8' and platform_machine!='aarch64' +numpy==1.19.2; python_version=='3.8' and platform_machine=='aarch64' numpy==1.19.3; python_version=='3.9' -numpy; python_version>='3.10' +numpy==1.21.3; python_version=='3.10' +numpy==1.23.2; python_version=='3.11' +numpy; python_version>='3.12' diff --git a/requirements.txt b/requirements.txt index 0ab9a7854..ea672cdfc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ tqdm>=4.38.0,<5.0.0 pyrsistent<0.17.0 jsonschema>=2.6.0,<3.1.0 # Development dependencies -cython>=0.25 +cython>=0.25,<3.0 pytest>=4.6.5 pytest-timeout>=1.3.0,<2.0.0 mock>=2.0.0,<3.0.0 diff --git a/setup.cfg b/setup.cfg index 3b38f0b3c..3c4d2a9d8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,8 @@ classifiers = Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Scientific/Engineering [options] @@ -33,7 +35,7 @@ scripts = bin/spacy python_requires = >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.* setup_requires = - cython>=0.25 + cython>=0.25,<3.0 numpy>=1.15.0 # We also need our Cython packages here to compile against cymem>=2.0.2,<2.1.0 From 4297ca43cff9049a7f85fb5b76b3932bcee4bf4b Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 18 Oct 2022 15:25:18 +0200 Subject: [PATCH 3/5] Update CI --- azure-pipelines.yml | 78 ++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e41743ded..2afd06747 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,7 +21,7 @@ jobs: # defined in .flake8 and overwrites the selected codes. - job: 'Validate' pool: - vmImage: 'ubuntu-18.04' + vmImage: 'ubuntu-latest' steps: - task: UsePythonVersion@0 inputs: @@ -35,50 +35,37 @@ jobs: dependsOn: 'Validate' strategy: matrix: - Python35Linux: - imageName: 'ubuntu-18.04' - python.version: '3.5' - os: linux - Python35Windows: - imageName: 'vs2017-win2016' - python.version: '3.5' -# Test on one OS per python 3.6/3.7/3.8 to speed up CI +# Test on one OS per python 3.6/3.7/3.8/3.9 to speed up CI Python36Linux: - imageName: 'ubuntu-18.04' + imageName: 'ubuntu-latest' python.version: '3.6' -# Python36Windows: -# imageName: 'vs2017-win2016' -# python.version: '3.6' -# Python36Mac: -# imageName: 'macos-10.14' -# python.version: '3.6' -# Python37Linux: -# imageName: 'ubuntu-18.04' -# python.version: '3.7' Python37Windows: - imageName: 'vs2017-win2016' + imageName: 'windows-latest' python.version: '3.7' -# Python37Mac: -# imageName: 'macos-10.14' -# python.version: '3.7' -# Python38Linux: -# imageName: 'ubuntu-18.04' -# python.version: '3.8' -# Python38Windows: -# imageName: 'vs2017-win2016' -# python.version: '3.8' Python38Mac: - imageName: 'macos-10.14' + imageName: 'macos-latest' python.version: '3.8' Python39Linux: - imageName: 'ubuntu-18.04' - python.version: '3.9' - Python39Windows: - imageName: 'vs2017-win2016' - python.version: '3.9' - Python39Mac: - imageName: 'macos-10.14' + imageName: 'ubuntu-latest' python.version: '3.9' + Python310Linux: + imageName: 'ubuntu-latest' + python.version: '3.10' + Python310Windows: + imageName: 'windows-latest' + python.version: '3.10' + Python310Mac: + imageName: 'macos-latest' + python.version: '3.10' + Python311Linux: + imageName: 'ubuntu-latest' + python.version: '3.11.0-rc.2' + Python311Windows: + imageName: 'windows-latest' + python.version: '3.11.0-rc.2' + Python311Mac: + imageName: 'macos-latest' + python.version: '3.11.0-rc.2' maxParallel: 4 pool: vmImage: $(imageName) @@ -88,17 +75,13 @@ jobs: inputs: versionSpec: '$(python.version)' architecture: 'x64' + allowUnstable: true - script: python -m pip install -U pip setuptools displayName: 'Update pip' - - script: pip install -r requirements.txt --prefer-binary - displayName: 'Install dependencies (python 3.5: prefer binary)' - condition: eq(variables['python.version'], '3.5') - - script: pip install -r requirements.txt displayName: 'Install dependencies' - condition: not(eq(variables['python.version'], '3.5')) - script: | python setup.py build_ext --inplace -j 2 @@ -115,20 +98,13 @@ jobs: pip uninstall -y -r installed.txt displayName: 'Uninstall all packages' - - bash: | - SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) - pip install dist/$SDIST --prefer-binary - displayName: 'Install from sdist (python 3.5: prefer binary)' - condition: eq(variables['python.version'], '3.5') - - bash: | SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) pip install dist/$SDIST displayName: 'Install from sdist' - condition: not(eq(variables['python.version'], '3.5')) - script: | - pip install -r requirements.txt --prefer-binary + pip install -r requirements.txt python -m pytest --pyargs spacy displayName: 'Run tests' @@ -136,4 +112,4 @@ jobs: python -m spacy download en_core_web_sm python -c "import spacy; nlp=spacy.load('en_core_web_sm'); doc=nlp('test')" displayName: 'Test download CLI' - condition: and(eq(variables['python.version'], '3.9'), eq(variables['imageName'], 'ubuntu-18.04')) + condition: eq(variables['python.version'], '3.9') From 07e630cd03dadbebd2924d71032a87bc314f4445 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 18 Oct 2022 15:26:56 +0200 Subject: [PATCH 4/5] Update dev requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ea672cdfc..ed6f33ad9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,4 +21,4 @@ cython>=0.25,<3.0 pytest>=4.6.5 pytest-timeout>=1.3.0,<2.0.0 mock>=2.0.0,<3.0.0 -flake8>=3.5.0,<3.6.0 +flake8>=3.5.0,<6.0.0 From 19f64ee18ab9f503ff643101cba8ea066f3be3b5 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 18 Oct 2022 15:27:29 +0200 Subject: [PATCH 5/5] Set version to v2.3.8 --- spacy/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/about.py b/spacy/about.py index 4b247868a..ab4f1bf36 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -1,6 +1,6 @@ # fmt: off __title__ = "spacy" -__version__ = "2.3.7" +__version__ = "2.3.8" __release__ = True __download_url__ = "https://github.com/explosion/spacy-models/releases/download" __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"