Merge pull request #5008 from svlandeg/fix/build_dependencies

Re-add pyproject.toml and add tests for dependency version consistency
This commit is contained in:
Ines Montani 2020-02-25 16:52:18 +01:00 committed by GitHub
commit 192b8d45a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 114 additions and 8 deletions

5
.gitignore vendored
View File

@ -5,6 +5,11 @@ corpora/
keys/ keys/
*.json.gz *.json.gz
# Tests
spacy/tests/package/setup.cfg
spacy/tests/package/pyproject.toml
spacy/tests/package/requirements.txt
# Website # Website
website/.cache/ website/.cache/
website/public/ website/public/

12
pyproject.toml Normal file
View File

@ -0,0 +1,12 @@
[build-system]
requires = [
"setuptools",
"wheel",
"cython>=0.25",
"cymem>=2.0.2,<2.1.0",
"preshed>=3.0.2,<3.1.0",
"murmurhash>=0.28.0,<1.1.0",
"thinc==8.0.0a0",
"blis>=0.4.0,<0.5.0"
]
build-backend = "setuptools.build_meta"

View File

@ -15,7 +15,7 @@ plac>=0.9.6,<1.2.0
tqdm>=4.38.0,<5.0.0 tqdm>=4.38.0,<5.0.0
# Optional dependencies # Optional dependencies
jsonschema>=2.6.0,<3.1.0 jsonschema>=2.6.0,<3.1.0
pydantic>=1.0.0,<2.0.0 pydantic>=1.3.0,<2.0.0
# Development dependencies # Development dependencies
cython>=0.25 cython>=0.25
pytest>=4.6.5 pytest>=4.6.5

View File

@ -7,15 +7,19 @@ from distutils import ccompiler, msvccompiler
from setuptools import Extension, setup, find_packages from setuptools import Extension, setup, find_packages
import numpy import numpy
from pathlib import Path from pathlib import Path
import shutil
from Cython.Build import cythonize from Cython.Build import cythonize
from Cython.Compiler import Options from Cython.Compiler import Options
ROOT = Path(__file__).parent
PACKAGE_ROOT = ROOT / "spacy"
# Preserve `__doc__` on functions and classes # Preserve `__doc__` on functions and classes
# http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-options # http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-options
Options.docstrings = True Options.docstrings = True
PACKAGES = find_packages() PACKAGES = find_packages()
MOD_NAMES = [ MOD_NAMES = [
"spacy.parts_of_speech", "spacy.parts_of_speech",
@ -60,6 +64,12 @@ COMPILER_DIRECTIVES = {
"embedsignature": True, "embedsignature": True,
"annotation_typing": False, "annotation_typing": False,
} }
# Files to copy into the package that are otherwise not included
COPY_FILES = {
ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package",
ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package",
ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package",
}
def is_new_osx(): def is_new_osx():
@ -115,25 +125,28 @@ def clean(path):
def setup_package(): def setup_package():
root = Path(__file__).parent
if len(sys.argv) > 1 and sys.argv[1] == "clean": if len(sys.argv) > 1 and sys.argv[1] == "clean":
return clean(root / "spacy") return clean(PACKAGE_ROOT)
with (root / "spacy" / "about.py").open("r") as f: with (PACKAGE_ROOT / "about.py").open("r") as f:
about = {} about = {}
exec(f.read(), about) exec(f.read(), about)
for copy_file, target_dir in COPY_FILES.items():
if copy_file.exists():
shutil.copy(str(copy_file), str(target_dir))
print(f"Copied {copy_file} -> {target_dir}")
include_dirs = [ include_dirs = [
get_python_inc(plat_specific=True), get_python_inc(plat_specific=True),
numpy.get_include(), numpy.get_include(),
str(root / "include"), str(ROOT / "include"),
] ]
if ( if (
ccompiler.new_compiler().compiler_type == "msvc" ccompiler.new_compiler().compiler_type == "msvc"
and msvccompiler.get_build_version() == 9 and msvccompiler.get_build_version() == 9
): ):
include_dirs.append(str(root / "include" / "msvc9")) include_dirs.append(str(ROOT / "include" / "msvc9"))
ext_modules = [] ext_modules = []
for name in MOD_NAMES: for name in MOD_NAMES:
mod_path = name.replace(".", "/") + ".pyx" mod_path = name.replace(".", "/") + ".pyx"

View File

@ -0,0 +1,76 @@
import re
from pathlib import Path
def test_build_dependencies():
# Check that library requirements are pinned exactly the same across different setup files.
libs_ignore_requirements = [
"pytest",
"pytest-timeout",
"mock",
"flake8",
"jsonschema",
]
libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"]
# check requirements.txt
req_dict = {}
root_dir = Path(__file__).parent
req_file = root_dir / "requirements.txt"
with req_file.open() as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if not line.startswith("#"):
lib, v = _parse_req(line)
if lib and lib not in libs_ignore_requirements:
req_dict[lib] = v
# check setup.cfg and compare to requirements.txt
# also fails when there are missing or additional libs
setup_file = root_dir / "setup.cfg"
with setup_file.open() as f:
lines = f.readlines()
setup_keys = set()
for line in lines:
line = line.strip()
if not line.startswith("#"):
lib, v = _parse_req(line)
if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup:
req_v = req_dict.get(lib, None)
assert (
req_v is not None
), "{} in setup.cfg but not in requirements.txt".format(lib)
assert (lib + v) == (lib + req_v), (
"{} has different version in setup.cfg and in requirements.txt: "
"{} and {} respectively".format(lib, v, req_v)
)
setup_keys.add(lib)
assert sorted(setup_keys) == sorted(
req_dict.keys()
) # if fail: requirements.txt contains a lib not in setup.cfg
# check pyproject.toml and compare the versions of the libs to requirements.txt
# does not fail when there are missing or additional libs
toml_file = root_dir / "pyproject.toml"
with toml_file.open() as f:
lines = f.readlines()
for line in lines:
line = line.strip().strip(",").strip('"')
if not line.startswith("#"):
lib, v = _parse_req(line)
if lib:
req_v = req_dict.get(lib, None)
assert (lib + v) == (lib + req_v), (
"{} has different version in pyproject.toml and in requirements.txt: "
"{} and {} respectively".format(lib, v, req_v)
)
def _parse_req(line):
lib = re.match(r"^[a-z0-9\-]*", line).group(0)
v = line.replace(lib, "").strip()
if not re.match(r"^[<>=][<>=].*", v):
return None, None
return lib, v