spaCy/setup.py

#!/usr/bin/env python
import subprocess
from setuptools import setup
from glob import glob

import sys
import os
from os import path
from os.path import splitext


from setuptools import Extension


def clean(ext):
    for src in ext.sources:
        if src.endswith('.c') or src.endswith('cpp'):
            so = src.rsplit('.', 1)[0] + '.so'
            html = src.rsplit('.', 1)[0] + '.html'
            if os.path.exists(so):
                os.unlink(so)
            if os.path.exists(html):
                os.unlink(html)


def name_to_path(mod_name, ext):
    return '%s.%s' % (mod_name.replace('.', '/'), ext)


def c_ext(mod_name, language, includes, compile_args):
    mod_path = name_to_path(mod_name, language)
    return Extension(mod_name, [mod_path], include_dirs=includes,
                     extra_compile_args=compile_args, extra_link_args=compile_args)


def cython_ext(mod_name, language, includes, compile_args):
    import Cython.Distutils
    import Cython.Build
    mod_path = mod_name.replace('.', '/') + '.pyx'
    if language == 'cpp':
        language = 'c++'
    ext = Extension(mod_name, [mod_path], language=language, include_dirs=includes,
                    extra_compile_args=compile_args)
    return Cython.Build.cythonize([ext])[0]


def run_setup(exts):
    
    setup(
        name='spacy',
        packages=['spacy', 'spacy.en', 'spacy.syntax'],
        description="Industrial-strength NLP",
        author='Matthew Honnibal',
        author_email='honnibal@gmail.com',
        version='0.21',
        url="http://honnibal.github.io/spaCy/",
        package_data={"spacy": ["*.pxd"],
                      "spacy.en": ["*.pxd", "data/pos/*",
                                   "data/wordnet/*", "data/tokenizer/*",
                                   "data/vocab/*"],
                      "spacy.syntax": ["*.pxd"]},
        ext_modules=exts,
        license="Dual: Commercial or AGPL",
        install_requires=['murmurhash', 'numpy', 'cymem', 'preshed', 'thinc',
                          "unidecode", "ujson"],
        setup_requires=["headers_workaround"],
    )

    import headers_workaround

    headers_workaround.fix_venv_pypy_include()
    headers_workaround.install_headers('murmurhash')


def main(modules, is_pypy):
    language = "cpp"
    ext_func = cython_ext if use_cython else c_ext
    includes = ['.', path.join(sys.prefix, 'include')]
    compile_args = ['-O3']
    exts = [ext_func(mn, language, includes, compile_args) for mn in modules]
    run_setup(exts)


MOD_NAMES = ['spacy.typedefs', 'spacy.strings', 'spacy.lexeme',
             'spacy.vocab', 'spacy.tokens', 'spacy.morphology',
             'spacy._ml', 'spacy.tokenizer', 'spacy.en.attrs',
             'spacy.en.pos', 'spacy.syntax.parser', 'spacy.syntax._state',
             'spacy.syntax.arc_eager', 'spacy.syntax._parse_features',
             'spacy.orth']


if __name__ == '__main__':
    use_cython = sys.argv[1] == 'build_ext'
    main(MOD_NAMES, use_cython)