explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-02-03 05:04:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

Auto-format

Browse Source
This commit is contained in:
Ines Montani 2019-09-29 17:32:35 +02:00
parent 3d8fd4b461
commit 0a3c825c56
1 changed files with 77 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
fabfile.py vendored
View File

@ -10,113 +10,145 @@ import sys
PWD = path.dirname(__file__) PWD = path.dirname(__file__)
ENV = environ['VENV_DIR'] if 'VENV_DIR' in environ else '.env' ENV = environ["VENV_DIR"] if "VENV_DIR" in environ else ".env"
VENV_DIR = Path(PWD) / ENV VENV_DIR = Path(PWD) / ENV
@contextlib.contextmanager @contextlib.contextmanager
def virtualenv(name, create=False, python='/usr/bin/python3.6'): def virtualenv(name, create=False, python="/usr/bin/python3.6"):
python = Path(python).resolve() python = Path(python).resolve()
env_path = VENV_DIR env_path = VENV_DIR
if create: if create:
if env_path.exists(): if env_path.exists():
shutil.rmtree(str(env_path)) shutil.rmtree(str(env_path))
local('{python} -m venv {env_path}'.format(python=python, env_path=VENV_DIR)) local("{python} -m venv {env_path}".format(python=python, env_path=VENV_DIR))
def wrapped_local(cmd, env_vars=[], capture=False, direct=False): def wrapped_local(cmd, env_vars=[], capture=False, direct=False):
return local('source {}/bin/activate && {}'.format(env_path, cmd), return local(
shell='/bin/bash', capture=False) "source {}/bin/activate && {}".format(env_path, cmd),
shell="/bin/bash",
capture=False,
)
yield wrapped_local yield wrapped_local
def env(lang='python3.6'): def env(lang="python3.6"):
if VENV_DIR.exists(): if VENV_DIR.exists():
local('rm -rf {env}'.format(env=VENV_DIR)) local("rm -rf {env}".format(env=VENV_DIR))
if lang.startswith('python3'): if lang.startswith("python3"):
local('{lang} -m venv {env}'.format(lang=lang, env=VENV_DIR)) local("{lang} -m venv {env}".format(lang=lang, env=VENV_DIR))
else: else:
local('{lang} -m pip install virtualenv --no-cache-dir'.format(lang=lang)) local("{lang} -m pip install virtualenv --no-cache-dir".format(lang=lang))
local('{lang} -m virtualenv {env} --no-cache-dir'.format(lang=lang, env=VENV_DIR)) local(
"{lang} -m virtualenv {env} --no-cache-dir".format(lang=lang, env=VENV_DIR)
)
with virtualenv(VENV_DIR) as venv_local: with virtualenv(VENV_DIR) as venv_local:
print(venv_local('python --version', capture=True)) print(venv_local("python --version", capture=True))
venv_local('pip install --upgrade setuptools --no-cache-dir') venv_local("pip install --upgrade setuptools --no-cache-dir")
venv_local('pip install pytest --no-cache-dir') venv_local("pip install pytest --no-cache-dir")
venv_local('pip install wheel --no-cache-dir') venv_local("pip install wheel --no-cache-dir")
venv_local('pip install -r requirements.txt --no-cache-dir') venv_local("pip install -r requirements.txt --no-cache-dir")
venv_local('pip install pex --no-cache-dir') venv_local("pip install pex --no-cache-dir")
def install(): def install():
with virtualenv(VENV_DIR) as venv_local: with virtualenv(VENV_DIR) as venv_local:
venv_local('pip install dist/*.tar.gz') venv_local("pip install dist/*.tar.gz")
def make(): def make():
with lcd(path.dirname(__file__)): with lcd(path.dirname(__file__)):
local('export PYTHONPATH=`pwd` && source .env/bin/activate && python setup.py build_ext --inplace', local(
shell='/bin/bash') "export PYTHONPATH=`pwd` && source .env/bin/activate && python setup.py build_ext --inplace",
shell="/bin/bash",
)
def sdist(): def sdist():
with virtualenv(VENV_DIR) as venv_local: with virtualenv(VENV_DIR) as venv_local:
with lcd(path.dirname(__file__)): with lcd(path.dirname(__file__)):
local('python -m pip install -U setuptools srsly') local("python -m pip install -U setuptools srsly")
local('python setup.py sdist') local("python setup.py sdist")
def wheel(): def wheel():
with virtualenv(VENV_DIR) as venv_local: with virtualenv(VENV_DIR) as venv_local:
with lcd(path.dirname(__file__)): with lcd(path.dirname(__file__)):
venv_local('python setup.py bdist_wheel') venv_local("python setup.py bdist_wheel")
def pex(): def pex():
with virtualenv(VENV_DIR) as venv_local: with virtualenv(VENV_DIR) as venv_local:
with lcd(path.dirname(__file__)): with lcd(path.dirname(__file__)):
sha = local('git rev-parse --short HEAD', capture=True) sha = local("git rev-parse --short HEAD", capture=True)
venv_local('pex dist/*.whl -e spacy -o dist/spacy-%s.pex' % sha, venv_local(
direct=True) "pex dist/*.whl -e spacy -o dist/spacy-%s.pex" % sha, direct=True
)
def clean(): def clean():
with lcd(path.dirname(__file__)): with lcd(path.dirname(__file__)):
local('rm -f dist/*.whl') local("rm -f dist/*.whl")
local('rm -f dist/*.pex') local("rm -f dist/*.pex")
with virtualenv(VENV_DIR) as venv_local: with virtualenv(VENV_DIR) as venv_local:
venv_local('python setup.py clean --all') venv_local("python setup.py clean --all")
def test(): def test():
with virtualenv(VENV_DIR) as venv_local: with virtualenv(VENV_DIR) as venv_local:
with lcd(path.dirname(__file__)): with lcd(path.dirname(__file__)):
venv_local('pytest -x spacy/tests') venv_local("pytest -x spacy/tests")
def train(): def train():
args = environ.get('SPACY_TRAIN_ARGS', '') args = environ.get("SPACY_TRAIN_ARGS", "")
with virtualenv(VENV_DIR) as venv_local: with virtualenv(VENV_DIR) as venv_local:
venv_local('spacy train {args}'.format(args=args)) venv_local("spacy train {args}".format(args=args))
def conll17(treebank_dir, experiment_dir, vectors_dir, config, corpus=''): def conll17(treebank_dir, experiment_dir, vectors_dir, config, corpus=""):
is_not_clean = local('git status --porcelain', capture=True) is_not_clean = local("git status --porcelain", capture=True)
if is_not_clean: if is_not_clean:
print("Repository is not clean") print("Repository is not clean")
print(is_not_clean) print(is_not_clean)
sys.exit(1) sys.exit(1)
git_sha = local('git rev-parse --short HEAD', capture=True) git_sha = local("git rev-parse --short HEAD", capture=True)
config_checksum = local('sha256sum {config}'.format(config=config), capture=True) config_checksum = local("sha256sum {config}".format(config=config), capture=True)
experiment_dir = Path(experiment_dir) / '{}--{}'.format(config_checksum[:6], git_sha) experiment_dir = Path(experiment_dir) / "{}--{}".format(
config_checksum[:6], git_sha
)
if not experiment_dir.exists(): if not experiment_dir.exists():
experiment_dir.mkdir() experiment_dir.mkdir()
test_data_dir = Path(treebank_dir) / 'ud-test-v2.0-conll2017' test_data_dir = Path(treebank_dir) / "ud-test-v2.0-conll2017"
assert test_data_dir.exists() assert test_data_dir.exists()
assert test_data_dir.is_dir() assert test_data_dir.is_dir()
if corpus: if corpus:
corpora = [corpus] corpora = [corpus]
else: else:
corpora = ['UD_English', 'UD_Chinese', 'UD_Japanese', 'UD_Vietnamese'] corpora = ["UD_English", "UD_Chinese", "UD_Japanese", "UD_Vietnamese"]
local('cp {config} {experiment_dir}/config.json'.format(config=config, experiment_dir=experiment_dir)) local(
"cp {config} {experiment_dir}/config.json".format(
config=config, experiment_dir=experiment_dir
)
)
with virtualenv(VENV_DIR) as venv_local: with virtualenv(VENV_DIR) as venv_local:
for corpus in corpora: for corpus in corpora:
venv_local('spacy ud-train {treebank_dir} {experiment_dir} {config} {corpus} -v {vectors_dir}'.format( venv_local(
treebank_dir=treebank_dir, experiment_dir=experiment_dir, config=config, corpus=corpus, vectors_dir=vectors_dir)) "spacy ud-train {treebank_dir} {experiment_dir} {config} {corpus} -v {vectors_dir}".format(
venv_local('spacy ud-run-test {test_data_dir} {experiment_dir} {corpus}'.format( treebank_dir=treebank_dir,
test_data_dir=test_data_dir, experiment_dir=experiment_dir, config=config, corpus=corpus)) experiment_dir=experiment_dir,
config=config,
corpus=corpus,
vectors_dir=vectors_dir,
)
)
venv_local(
"spacy ud-run-test {test_data_dir} {experiment_dir} {corpus}".format(
test_data_dir=test_data_dir,
experiment_dir=experiment_dir,
config=config,
corpus=corpus,
)
)