spaCy/spacy/en/__init__.py

67 lines
2.1 KiB
Python
Raw Normal View History

2017-03-12 15:07:28 +03:00
# coding: utf8
2015-08-25 16:37:30 +03:00
from __future__ import unicode_literals, print_function
2015-07-07 15:00:07 +03:00
2015-08-25 16:37:30 +03:00
from os import path
2017-03-08 03:37:52 +03:00
from pathlib import Path
2014-12-24 09:42:00 +03:00
from ..util import match_best_version
from ..util import get_data_path
2015-08-25 16:37:30 +03:00
from ..language import Language
from ..lemmatizer import Lemmatizer
from ..vocab import Vocab
from ..tokenizer import Tokenizer
from ..attrs import LANG
2016-12-18 18:54:19 +03:00
from .language_data import *
2017-03-08 03:37:52 +03:00
try:
basestring
except NameError:
basestring = str
2016-12-18 18:58:28 +03:00
2015-08-25 16:37:30 +03:00
class English(Language):
2015-12-28 18:54:03 +03:00
lang = 'en'
2016-09-24 21:26:17 +03:00
class Defaults(Language.Defaults):
lex_attr_getters = dict(Language.Defaults.lex_attr_getters)
lex_attr_getters[LANG] = lambda text: 'en'
2016-09-24 21:26:17 +03:00
tokenizer_exceptions = TOKENIZER_EXCEPTIONS
tag_map = TAG_MAP
stop_words = STOP_WORDS
def __init__(self, **overrides):
# Make a special-case hack for loading the GloVe vectors, to support
# deprecated <1.0 stuff. Phase this out once the data is fixed.
overrides = _fix_deprecated_glove_vectors_loading(overrides)
Language.__init__(self, **overrides)
def _fix_deprecated_glove_vectors_loading(overrides):
if 'data_dir' in overrides and 'path' not in overrides:
raise ValueError("The argument 'data_dir' has been renamed to 'path'")
2016-12-19 01:12:49 +03:00
if overrides.get('path') is False:
return overrides
if overrides.get('path') in (None, True):
data_path = get_data_path()
else:
path = overrides['path']
2017-03-08 03:37:52 +03:00
if isinstance(path, basestring):
path = Path(path)
data_path = path.parent
vec_path = None
if 'add_vectors' not in overrides:
if 'vectors' in overrides:
vec_path = match_best_version(overrides['vectors'], None, data_path)
if vec_path is None:
2017-03-08 03:37:52 +03:00
return overrides
else:
vec_path = match_best_version('en_glove_cc_300_1m_vectors', None, data_path)
if vec_path is not None:
vec_path = vec_path / 'vocab' / 'vec.bin'
2016-12-19 01:06:44 +03:00
if vec_path is not None:
overrides['add_vectors'] = lambda vocab: vocab.load_vectors_from_bin_loc(vec_path)
return overrides