mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-24 17:06:29 +03:00
Replacing regex library with re to increase tokenization speed (#3218)
* replace unicode categories with raw list of code points * simplifying ranges * fixing variable length quotes * removing redundant regular expression * small cleanup of regexp notations * quotes and alpha as ranges instead of alterations * removed most regexp dependencies and features * exponential backtracking - unit tests * rewrote expression with pathological backtracking * disabling double hyphen tests for now * test additional variants of repeating punctuation * remove regex and redundant backslashes from load_reddit script * small typo fixes * disable double punctuation test for russian * clean up old comments * format block code * final cleanup * naming consistency * french strings as unicode for python 2 support * french regular expression case insensitive
This commit is contained in:
Sofie
Matthew Honnibal
parent
66016ac289
commit
46dfe773e1
|
|
@ -55,7 +55,7 @@ even format them as Markdown to copy-paste into GitHub issues:
|
|||
`python -m spacy info --markdown`.
|
||||
|
||||
* **Checking the model compatibility:** If you're having problems with a
|
||||
[statistical model](https://spacy.io/models), it may be because to the
|
||||
[statistical model](https://spacy.io/models), it may be because the
|
||||
model is incompatible with your spaCy installation. In spaCy v2.0+, you can check
|
||||
this on the command line by running `python -m spacy validate`.
|
||||
|
||||
|
|
@ -320,7 +320,7 @@ of other types these names. For instance, don't name a text string `doc` — you
|
|||
should usually call this `text`. Two general code style preferences further help
|
||||
with naming. First, **lean away from introducing temporary variables**, as these
|
||||
clutter your namespace. This is one reason why comprehension expressions are
|
||||
often preferred. Second, **keep your functions shortish**, so that can work in a
|
||||
often preferred. Second, **keep your functions shortish**, so they can work in a
|
||||
smaller scope. Of course, this is a question of trade-offs.
|
||||
|
||||
### Cython conventions
|
||||
|
|
@ -438,7 +438,7 @@ avoid unnecessary imports.
|
|||
Extensive tests that take a long time should be marked with `@pytest.mark.slow`.
|
||||
Tests that require the model to be loaded should be marked with
|
||||
`@pytest.mark.models`. Loading the models is expensive and not necessary if
|
||||
you're not actually testing the model performance. If all you needs ia a `Doc`
|
||||
you're not actually testing the model performance. If all you need is a `Doc`
|
||||
object with annotations like heads, POS tags or the dependency parse, you can
|
||||
use the `get_doc()` utility function to construct it manually.
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
from __future__ import unicode_literals
|
||||
|
||||
import bz2
|
||||
import regex as re
|
||||
import re
|
||||
import srsly
|
||||
import sys
|
||||
import random
|
||||
|
|
@ -16,8 +16,8 @@ _unset = object()
|
|||
class Reddit(object):
|
||||
"""Stream cleaned comments from Reddit."""
|
||||
|
||||
pre_format_re = re.compile(r"^[\`\*\~]")
|
||||
post_format_re = re.compile(r"[\`\*\~]$")
|
||||
pre_format_re = re.compile(r"^[`*~]")
|
||||
post_format_re = re.compile(r"[`*~]$")
|
||||
url_re = re.compile(r"\[([^]]+)\]\(%%URL\)")
|
||||
link_re = re.compile(r"\[([^]]+)\]\(https?://[^\)]+\)")
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ srsly>=0.0.5,<1.1.0
|
|||
numpy>=1.15.0
|
||||
requests>=2.13.0,<3.0.0
|
||||
jsonschema>=2.6.0,<3.0.0
|
||||
regex==2018.01.10
|
||||
plac<1.0.0,>=0.9.6
|
||||
pathlib==1.0.1; python_version < "3.4"
|
||||
# Development dependencies
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@ _suffixes = (
|
|||
+ [
|
||||
r"(?<=[0-9])\+",
|
||||
# Arabic is written from Right-To-Left
|
||||
r"(?<=[0-9])(?:{})".format(CURRENCY),
|
||||
r"(?<=[0-9])(?:{})".format(UNITS),
|
||||
r"(?<=[0-9])(?:{c})".format(c=CURRENCY),
|
||||
r"(?<=[0-9])(?:{u})".format(u=UNITS),
|
||||
r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER),
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
from __future__ import unicode_literals
|
||||
|
||||
from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_ICONS
|
||||
from ..char_classes import ALPHA_LOWER, ALPHA, HYPHENS, QUOTES, UNITS
|
||||
from ..char_classes import ALPHA_LOWER, ALPHA, HYPHENS, CONCAT_QUOTES, UNITS
|
||||
|
||||
|
||||
_currency = r"\$|¢|£|€|¥|฿|৳"
|
||||
_quotes = QUOTES.replace("'", "")
|
||||
_currency = r"\$¢£€¥฿৳"
|
||||
_quotes = CONCAT_QUOTES.replace("'", "")
|
||||
_list_punct = LIST_PUNCT + "। ॥".strip().split()
|
||||
|
||||
|
||||
|
|
@ -20,11 +20,9 @@ _suffixes = (
|
|||
+ [
|
||||
r"(?<=[0-9])\+",
|
||||
r"(?<=°[FfCcKk])\.",
|
||||
r"(?<=[0-9])(?:{})".format(_currency),
|
||||
r"(?<=[0-9])(?:{})".format(UNITS),
|
||||
r"(?<=[{}(?:{})])\.".format(
|
||||
"|".join([ALPHA_LOWER, r"%²\-\)\]\+", QUOTES]), _currency
|
||||
),
|
||||
r"(?<=[0-9])(?:[{c}])".format(c=_currency),
|
||||
r"(?<=[0-9])(?:{u})".format(u=UNITS),
|
||||
r"(?<=[{al}{e}{q}(?:{c})])\.".format(al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, c=_currency),
|
||||
]
|
||||
)
|
||||
|
||||
|
|
@ -36,9 +34,9 @@ _infixes = (
|
|||
zero="০", nine="৯"
|
||||
),
|
||||
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}])[{h}](?={ae})".format(a=ALPHA, h=HYPHENS, ae="এ"),
|
||||
r'(?<=[{a}])[?";:=,.]*(?:{h})(?=[{a}])'.format(a=ALPHA, h=HYPHENS),
|
||||
r'(?<=[{a}"])[:<>=/](?=[{a}])'.format(a=ALPHA),
|
||||
r"(?<=[{a}])({h})(?=[{ae}])".format(a=ALPHA, h=HYPHENS, ae="এ"),
|
||||
r'(?<=[{a}])(?:{h})(?=[{a}])'.format(a=ALPHA, h=HYPHENS),
|
||||
r'(?<=[{a}])[:<>=/](?=[{a}])'.format(a=ALPHA),
|
||||
]
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,34 +1,45 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import regex as re
|
||||
|
||||
re.DEFAULT_VERSION = re.VERSION1
|
||||
merge_char_classes = lambda classes: "[{}]".format("||".join(classes))
|
||||
split_chars = lambda char: list(char.strip().split(" "))
|
||||
merge_chars = lambda char: char.strip().replace(" ", "|")
|
||||
group_chars = lambda char: char.strip().replace(" ", "")
|
||||
|
||||
_bengali = r"[\p{L}&&\p{Bengali}]"
|
||||
_hebrew = r"[\p{L}&&\p{Hebrew}]"
|
||||
_latin_lower = r"[\p{Ll}&&\p{Latin}]"
|
||||
_latin_upper = r"[\p{Lu}&&\p{Latin}]"
|
||||
_latin = r"[[\p{Ll}||\p{Lu}]&&\p{Latin}]"
|
||||
_persian = r"[\p{L}&&\p{Arabic}]"
|
||||
_russian_lower = r"[ёа-я]"
|
||||
_russian_upper = r"[ЁА-Я]"
|
||||
_sinhala = r"[\p{L}&&\p{Sinhala}]"
|
||||
_tatar_lower = r"[әөүҗңһ]"
|
||||
_tatar_upper = r"[ӘӨҮҖҢҺ]"
|
||||
_greek_lower = r"[α-ωάέίόώήύ]"
|
||||
_greek_upper = r"[Α-ΩΆΈΊΌΏΉΎ]"
|
||||
# used https://unicode.org/cldr/utility/list-unicodeset.jsp to convert categories into code points
|
||||
# \p{L}&&\p{Bengali}
|
||||
# https://en.wikipedia.org/wiki/Bengali_(Unicode_block)
|
||||
_bengali = r"\u0980-\u09FF"
|
||||
|
||||
_upper = [_latin_upper, _russian_upper, _tatar_upper, _greek_upper]
|
||||
_lower = [_latin_lower, _russian_lower, _tatar_lower, _greek_lower]
|
||||
_uncased = [_bengali, _hebrew, _persian, _sinhala]
|
||||
# \p{L}&&\p{Hebrew}
|
||||
# https://en.wikipedia.org/wiki/Unicode_and_HTML_for_the_Hebrew_alphabet
|
||||
_hebrew = r"\u0591-\u05F4\uFB1D-\uFB4F"
|
||||
|
||||
ALPHA = merge_char_classes(_upper + _lower + _uncased)
|
||||
ALPHA_LOWER = merge_char_classes(_lower + _uncased)
|
||||
ALPHA_UPPER = merge_char_classes(_upper + _uncased)
|
||||
# \p{Ll}&&\p{Latin}
|
||||
_latin_lower = r"a-z\u00DF-\u00F6\u00F8-\u00FF\u0101\u0103\u0105\u0107\u0109\u010B\u010D\u010F\u0111\u0113\u0115\u0117\u0119\u011B\u011D\u011F\u0121\u0123\u0125\u0127\u0129\u012B\u012D\u012F\u0131\u0133\u0135\u0137\u0138\u013A\u013C\u013E\u0140\u0142\u0144\u0146\u0148\u0149\u014B\u014D\u014F\u0151\u0153\u0155\u0157\u0159\u015B\u015D\u015F\u0161\u0163\u0165\u0167\u0169\u016B\u016D\u016F\u0171\u0173\u0175\u0177\u017A\u017C\u017E-\u0180\u0183\u0185\u0188\u018C\u018D\u0192\u0195\u0199-\u019B\u019E\u01A1\u01A3\u01A5\u01A8\u01AA\u01AB\u01AD\u01B0\u01B4\u01B6\u01B9\u01BA\u01BD-\u01BF\u01C6\u01C9\u01CC\u01CE\u01D0\u01D2\u01D4\u01D6\u01D8\u01DA\u01DC\u01DD\u01DF\u01E1\u01E3\u01E5\u01E7\u01E9\u01EB\u01ED\u01EF\u01F0\u01F3\u01F5\u01F9\u01FB\u01FD\u01FF\u0201\u0203\u0205\u0207\u0209\u020B\u020D\u020F\u0211\u0213\u0215\u0217\u0219\u021B\u021D\u021F\u0221\u0223\u0225\u0227\u0229\u022B\u022D\u022F\u0231\u0233-\u0239\u023C\u023F\u0240\u0242\u0247\u0249\u024B\u024D\u024F-\u0293\u0295-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E01\u1E03\u1E05\u1E07\u1E09\u1E0B\u1E0D\u1E0F\u1E11\u1E13\u1E15\u1E17\u1E19\u1E1B\u1E1D\u1E1F\u1E21\u1E23\u1E25\u1E27\u1E29\u1E2B\u1E2D\u1E2F\u1E31\u1E33\u1E35\u1E37\u1E39\u1E3B\u1E3D\u1E3F\u1E41\u1E43\u1E45\u1E47\u1E49\u1E4B\u1E4D\u1E4F\u1E51\u1E53\u1E55\u1E57\u1E59\u1E5B\u1E5D\u1E5F\u1E61\u1E63\u1E65\u1E67\u1E69\u1E6B\u1E6D\u1E6F\u1E71\u1E73\u1E75\u1E77\u1E79\u1E7B\u1E7D\u1E7F\u1E81\u1E83\u1E85\u1E87\u1E89\u1E8B\u1E8D\u1E8F\u1E91\u1E93\u1E95-\u1E9D\u1E9F\u1EA1\u1EA3\u1EA5\u1EA7\u1EA9\u1EAB\u1EAD\u1EAF\u1EB1\u1EB3\u1EB5\u1EB7\u1EB9\u1EBB\u1EBD\u1EBF\u1EC1\u1EC3\u1EC5\u1EC7\u1EC9\u1ECB\u1ECD\u1ECF\u1ED1\u1ED3\u1ED5\u1ED7\u1ED9\u1EDB\u1EDD\u1EDF\u1EE1\u1EE3\u1EE5\u1EE7\u1EE9\u1EEB\u1EED\u1EEF\u1EF1\u1EF3\u1EF5\u1EF7\u1EF9\u1EFB\u1EFD\u1EFF\u214E\u2184\u2C61\u2C65\u2C66\u2C68\u2C6A\u2C6C\u2C71\u2C73\u2C74\u2C76-\u2C7B\uA723\uA725\uA727\uA729\uA72B\uA72D\uA72F-\uA731\uA733\uA735\uA737\uA739\uA73B\uA73D\uA73F\uA741\uA743\uA745\uA747\uA749\uA74B\uA74D\uA74F\uA751\uA753\uA755\uA757\uA759\uA75B\uA75D\uA75F\uA761\uA763\uA765\uA767\uA769\uA76B\uA76D\uA76F\uA771-\uA778\uA77A\uA77C\uA77F\uA781\uA783\uA785\uA787\uA78C\uA78E\uA791\uA793-\uA795\uA797\uA799\uA79B\uA79D\uA79F\uA7A1\uA7A3\uA7A5\uA7A7\uA7A9\uA7AF\uA7B5\uA7B7\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\uFB00-\uFB06\uFF41-\uFF5A"
|
||||
# \p{Lu}&&\p{Latin}
|
||||
_latin_upper = r"A-Z\u00C0-\u00D6\u00D8-\u00DE\u0100\u0102\u0104\u0106\u0108\u010A\u010C\u010E\u0110\u0112\u0114\u0116\u0118\u011A\u011C\u011E\u0120\u0122\u0124\u0126\u0128\u012A\u012C\u012E\u0130\u0132\u0134\u0136\u0139\u013B\u013D\u013F\u0141\u0143\u0145\u0147\u014A\u014C\u014E\u0150\u0152\u0154\u0156\u0158\u015A\u015C\u015E\u0160\u0162\u0164\u0166\u0168\u016A\u016C\u016E\u0170\u0172\u0174\u0176\u0178\u0179\u017B\u017D\u0181\u0182\u0184\u0186\u0187\u0189-\u018B\u018E-\u0191\u0193\u0194\u0196-\u0198\u019C\u019D\u019F\u01A0\u01A2\u01A4\u01A6\u01A7\u01A9\u01AC\u01AE\u01AF\u01B1-\u01B3\u01B5\u01B7\u01B8\u01BC\u01C4\u01C7\u01CA\u01CD\u01CF\u01D1\u01D3\u01D5\u01D7\u01D9\u01DB\u01DE\u01E0\u01E2\u01E4\u01E6\u01E8\u01EA\u01EC\u01EE\u01F1\u01F4\u01F6-\u01F8\u01FA\u01FC\u01FE\u0200\u0202\u0204\u0206\u0208\u020A\u020C\u020E\u0210\u0212\u0214\u0216\u0218\u021A\u021C\u021E\u0220\u0222\u0224\u0226\u0228\u022A\u022C\u022E\u0230\u0232\u023A\u023B\u023D\u023E\u0241\u0243-\u0246\u0248\u024A\u024C\u024E\u1E00\u1E02\u1E04\u1E06\u1E08\u1E0A\u1E0C\u1E0E\u1E10\u1E12\u1E14\u1E16\u1E18\u1E1A\u1E1C\u1E1E\u1E20\u1E22\u1E24\u1E26\u1E28\u1E2A\u1E2C\u1E2E\u1E30\u1E32\u1E34\u1E36\u1E38\u1E3A\u1E3C\u1E3E\u1E40\u1E42\u1E44\u1E46\u1E48\u1E4A\u1E4C\u1E4E\u1E50\u1E52\u1E54\u1E56\u1E58\u1E5A\u1E5C\u1E5E\u1E60\u1E62\u1E64\u1E66\u1E68\u1E6A\u1E6C\u1E6E\u1E70\u1E72\u1E74\u1E76\u1E78\u1E7A\u1E7C\u1E7E\u1E80\u1E82\u1E84\u1E86\u1E88\u1E8A\u1E8C\u1E8E\u1E90\u1E92\u1E94\u1E9E\u1EA0\u1EA2\u1EA4\u1EA6\u1EA8\u1EAA\u1EAC\u1EAE\u1EB0\u1EB2\u1EB4\u1EB6\u1EB8\u1EBA\u1EBC\u1EBE\u1EC0\u1EC2\u1EC4\u1EC6\u1EC8\u1ECA\u1ECC\u1ECE\u1ED0\u1ED2\u1ED4\u1ED6\u1ED8\u1EDA\u1EDC\u1EDE\u1EE0\u1EE2\u1EE4\u1EE6\u1EE8\u1EEA\u1EEC\u1EEE\u1EF0\u1EF2\u1EF4\u1EF6\u1EF8\u1EFA\u1EFC\u1EFE\u212A\u212B\u2132\u2183\u2C60\u2C62-\u2C64\u2C67\u2C69\u2C6B\u2C6D-\u2C70\u2C72\u2C75\u2C7E\u2C7F\uA722\uA724\uA726\uA728\uA72A\uA72C\uA72E\uA732\uA734\uA736\uA738\uA73A\uA73C\uA73E\uA740\uA742\uA744\uA746\uA748\uA74A\uA74C\uA74E\uA750\uA752\uA754\uA756\uA758\uA75A\uA75C\uA75E\uA760\uA762\uA764\uA766\uA768\uA76A\uA76C\uA76E\uA779\uA77B\uA77D\uA77E\uA780\uA782\uA784\uA786\uA78B\uA78D\uA790\uA792\uA796\uA798\uA79A\uA79C\uA79E\uA7A0\uA7A2\uA7A4\uA7A6\uA7A8\uA7AA-\uA7AE\uA7B0-\uA7B4\uA7B6\uA7B8\uFF21-\uFF3A"
|
||||
# [\p{Ll}||\p{Lu}]&&\p{Latin}
|
||||
latin = r"A-Za-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u01BA\u01BC-\u01BF\u01C4\u01C6\u01C7\u01C9\u01CA\u01CC-\u01F1\u01F3-\u0293\u0295-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A\u1E00-\u1EFF\u212A\u212B\u2132\u214E\u2183\u2184\u2C60-\u2C7B\u2C7E\u2C7F\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA\uAB30-\uAB5A\uAB60-\uAB64\uFB00-\uFB06\uFF21-\uFF3A\uFF41-\uFF5A"
|
||||
|
||||
# \p{L}&&\p{Arabic}
|
||||
_persian = r"\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB"
|
||||
|
||||
_russian_lower = r"ёа-я"
|
||||
_russian_upper = r"ЁА-Я"
|
||||
# \p{L}&&\p{Sinhala}
|
||||
_sinhala = r"\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6"
|
||||
_tatar_lower = r"әөүҗңһ"
|
||||
_tatar_upper = r"ӘӨҮҖҢҺ"
|
||||
_greek_lower = r"α-ωάέίόώήύ"
|
||||
_greek_upper = r"Α-ΩΆΈΊΌΏΉΎ"
|
||||
|
||||
_upper = _latin_upper + _russian_upper + _tatar_upper + _greek_upper
|
||||
_lower = _latin_lower + _russian_lower + _tatar_lower + _greek_lower
|
||||
_uncased = _bengali + _hebrew + _persian + _sinhala
|
||||
|
||||
ALPHA = group_chars(_upper + _lower + _uncased)
|
||||
ALPHA_LOWER = group_chars(_lower + _uncased)
|
||||
ALPHA_UPPER = group_chars(_upper + _uncased)
|
||||
|
||||
_units = (
|
||||
"km km² km³ m m² m³ dm dm² dm³ cm cm² cm³ mm mm² mm³ ha µm nm yd in ft "
|
||||
|
|
@ -45,16 +56,15 @@ _currency = r"\$ £ € ¥ ฿ US\$ C\$ A\$ ₽ ﷼"
|
|||
_punct = (
|
||||
r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ؛ ٪"
|
||||
)
|
||||
_quotes = r'\' \'\' " ” “ `` ` ‘ ´ ‘‘ ’’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉'
|
||||
_quotes = r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉'
|
||||
_hyphens = "- – — -- --- —— ~"
|
||||
|
||||
# Various symbols like dingbats, but also emoji
|
||||
# Details: https://www.compart.com/en/unicode/category/So
|
||||
_other_symbols = r"[\p{So}]"
|
||||
_other_symbols = r"\u00A6\u00A9\u00AE\u00B0\u0482\u058D\u058E\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09FA\u0B70\u0BF3-\u0BF8\u0BFA\u0C7F\u0D4F\u0D79\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116\u2117\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u214A\u214C\u214D\u214F\u218A\u218B\u2195-\u2199\u219C-\u219F\u21A1\u21A2\u21A4\u21A5\u21A7-\u21AD\u21AF-\u21CD\u21D0\u21D1\u21D3\u21D5-\u21F3\u2300-\u2307\u230C-\u231F\u2322-\u2328\u232B-\u237B\u237D-\u239A\u23B4-\u23DB\u23E2-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u25B6\u25B8-\u25C0\u25C2-\u25F7\u2600-\u266E\u2670-\u2767\u2794-\u27BF\u2800-\u28FF\u2B00-\u2B2F\u2B45\u2B46\u2B4D-\u2B73\u2B76-\u2B95\u2B98-\u2BC8\u2BCA-\u2BFE\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA828-\uA82B\uA836\uA837\uA839\uAA77-\uAA79\uFDFD\uFFE4\uFFE8\uFFED\uFFEE\uFFFC\uFFFD\U00010137-\U0001013F\U00010179-\U00010189\U0001018C-\U0001018E\U00010190-\U0001019B\U000101A0\U000101D0-\U000101FC\U00010877\U00010878\U00010AC8\U0001173F\U00016B3C-\U00016B3F\U00016B45\U0001BC9C\U0001D000-\U0001D0F5\U0001D100-\U0001D126\U0001D129-\U0001D164\U0001D16A-\U0001D16C\U0001D183\U0001D184\U0001D18C-\U0001D1A9\U0001D1AE-\U0001D1E8\U0001D200-\U0001D241\U0001D245\U0001D300-\U0001D356\U0001D800-\U0001D9FF\U0001DA37-\U0001DA3A\U0001DA6D-\U0001DA74\U0001DA76-\U0001DA83\U0001DA85\U0001DA86\U0001ECAC\U0001F000-\U0001F02B\U0001F030-\U0001F093\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF\U0001F0D1-\U0001F0F5\U0001F110-\U0001F16B\U0001F170-\U0001F1AC\U0001F1E6-\U0001F202\U0001F210-\U0001F23B\U0001F240-\U0001F248\U0001F250\U0001F251\U0001F260-\U0001F265\U0001F300-\U0001F3FA\U0001F400-\U0001F6D4\U0001F6E0-\U0001F6EC\U0001F6F0-\U0001F6F9\U0001F700-\U0001F773\U0001F780-\U0001F7D8\U0001F800-\U0001F80B\U0001F810-\U0001F847\U0001F850-\U0001F859\U0001F860-\U0001F887\U0001F890-\U0001F8AD\U0001F900-\U0001F90B\U0001F910-\U0001F93E\U0001F940-\U0001F970\U0001F973-\U0001F976\U0001F97A\U0001F97C-\U0001F9A2\U0001F9B0-\U0001F9B9\U0001F9C0-\U0001F9C2\U0001F9D0-\U0001F9FF\U0001FA60-\U0001FA6D"
|
||||
|
||||
UNITS = merge_chars(_units)
|
||||
CURRENCY = merge_chars(_currency)
|
||||
QUOTES = merge_chars(_quotes)
|
||||
PUNCT = merge_chars(_punct)
|
||||
HYPHENS = merge_chars(_hyphens)
|
||||
ICONS = _other_symbols
|
||||
|
|
@ -65,4 +75,7 @@ LIST_QUOTES = split_chars(_quotes)
|
|||
LIST_PUNCT = split_chars(_punct)
|
||||
LIST_HYPHENS = split_chars(_hyphens)
|
||||
LIST_ELLIPSES = [r"\.\.+", "…"]
|
||||
LIST_ICONS = [_other_symbols]
|
||||
LIST_ICONS = [r"[{i}]".format(i=_other_symbols)]
|
||||
|
||||
CONCAT_QUOTES = group_chars(_quotes)
|
||||
CONCAT_ICONS = _other_symbols
|
||||
|
|
|
|||
|
|
@ -2,21 +2,21 @@
|
|||
from __future__ import unicode_literals
|
||||
|
||||
from ..char_classes import LIST_ELLIPSES, LIST_ICONS
|
||||
from ..char_classes import QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||||
from ..char_classes import CONCAT_QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||||
from ..punctuation import TOKENIZER_SUFFIXES
|
||||
|
||||
|
||||
_quotes = QUOTES.replace("'", "")
|
||||
_quotes = CONCAT_QUOTES.replace("'", "")
|
||||
|
||||
_infixes = (
|
||||
LIST_ELLIPSES
|
||||
+ LIST_ICONS
|
||||
+ [
|
||||
r"(?<=[{}])\.(?=[{}])".format(ALPHA_LOWER, ALPHA_UPPER),
|
||||
r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER),
|
||||
r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA),
|
||||
r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
|
||||
r'(?<=[{a}])[:<>=](?=[{a}])'.format(a=ALPHA),
|
||||
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}])([{q}\)\]\(\[])(?=[\{a}])".format(a=ALPHA, q=_quotes),
|
||||
r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes),
|
||||
r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA),
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,20 +2,20 @@
|
|||
from __future__ import unicode_literals
|
||||
|
||||
from ..char_classes import LIST_ELLIPSES, LIST_ICONS
|
||||
from ..char_classes import QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||||
from ..char_classes import CONCAT_QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||||
|
||||
|
||||
_quotes = QUOTES.replace("'", "")
|
||||
_quotes = CONCAT_QUOTES.replace("'", "")
|
||||
|
||||
_infixes = (
|
||||
LIST_ELLIPSES
|
||||
+ LIST_ICONS
|
||||
+ [
|
||||
r"(?<=[{}])\.(?=[{}])".format(ALPHA_LOWER, ALPHA_UPPER),
|
||||
r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER),
|
||||
r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA),
|
||||
r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
|
||||
r'(?<=[{a}])[:<>=](?=[{a}])'.format(a=ALPHA),
|
||||
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}])([{q}\)\]\(\[])(?=[\{a}])".format(a=ALPHA, q=_quotes),
|
||||
r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes),
|
||||
r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[0-9])-(?=[0-9])",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from __future__ import unicode_literals
|
|||
|
||||
from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY
|
||||
from ..char_classes import LIST_ICONS, ALPHA_LOWER, ALPHA_UPPER, ALPHA, HYPHENS
|
||||
from ..char_classes import QUOTES, CURRENCY
|
||||
from ..char_classes import CONCAT_QUOTES, CURRENCY
|
||||
|
||||
_units = (
|
||||
"km km² km³ m m² m³ dm dm² dm³ cm cm² cm³ mm mm² mm³ ha µm nm yd in ft "
|
||||
|
|
@ -57,10 +57,10 @@ _suffixes = (
|
|||
r"^([0-9]){1}\)$", # 12)
|
||||
r"(?<=°[FfCcKk])\.",
|
||||
r"([0-9])+\&", # 12&
|
||||
r"(?<=[0-9])(?:{})".format(CURRENCY),
|
||||
r"(?<=[0-9])(?:{})".format(UNITS),
|
||||
r"(?<=[0-9{}{}(?:{})])\.".format(ALPHA_LOWER, r"²\-\)\]\+", QUOTES),
|
||||
r"(?<=[{a}][{a}])\.".format(a=ALPHA_UPPER),
|
||||
r"(?<=[0-9])(?:{c})".format(c=CURRENCY),
|
||||
r"(?<=[0-9])(?:{u})".format(u=UNITS),
|
||||
r"(?<=[0-9{al}{e}(?:{q})])\.".format(al=ALPHA_LOWER, e=r"²\-\+", q=CONCAT_QUOTES),
|
||||
r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER),
|
||||
r"(?<=[Α-Ωα-ωίϊΐόάέύϋΰήώ])\-", # όνομα-
|
||||
r"(?<=[Α-Ωα-ωίϊΐόάέύϋΰήώ])\.",
|
||||
r"^[Α-Ω]{1}\.",
|
||||
|
|
@ -85,10 +85,10 @@ _infixes = (
|
|||
r"([0-9]){1,4}[\/]([0-9]){1,2}([\/]([0-9]){0,4}){0,1}",
|
||||
r"[A-Za-z]+\@[A-Za-z]+(\-[A-Za-z]+)*\.[A-Za-z]+", # abc@cde-fgh.a
|
||||
r"([a-zA-Z]+)(\-([a-zA-Z]+))+", # abc-abc
|
||||
r"(?<=[{}])\.(?=[{}])".format(ALPHA_LOWER, ALPHA_UPPER),
|
||||
r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER),
|
||||
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
|
||||
r'(?<=[{a}])[?";:=,.]*(?:{h})(?=[{a}])'.format(a=ALPHA, h=HYPHENS),
|
||||
r'(?<=[{a}"])[:<>=/](?=[{a}])'.format(a=ALPHA),
|
||||
r'(?<=[{a}])(?:{h})(?=[{a}])'.format(a=ALPHA, h=HYPHENS),
|
||||
r'(?<=[{a}])[:<>=/](?=[{a}])'.format(a=ALPHA),
|
||||
]
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ _suffixes = (
|
|||
r"(?<=[0-9])\+",
|
||||
r"(?<=[0-9])%", # 4% -> ["4", "%"]
|
||||
# Persian is written from Right-To-Left
|
||||
r"(?<=[0-9])(?:{})".format(CURRENCY),
|
||||
r"(?<=[0-9])(?:{})".format(UNITS),
|
||||
r"(?<=[0-9])(?:{c})".format(c=CURRENCY),
|
||||
r"(?<=[0-9])(?:{u})".format(u=UNITS),
|
||||
r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER),
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from __future__ import unicode_literals
|
|||
|
||||
from ..punctuation import TOKENIZER_INFIXES
|
||||
from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, CURRENCY
|
||||
from ..char_classes import QUOTES, UNITS, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||||
from ..char_classes import CONCAT_QUOTES, UNITS, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||||
|
||||
|
||||
ELISION = " ' ’ ".strip().replace(" ", "").replace("\n", "")
|
||||
|
|
@ -16,12 +16,12 @@ _suffixes = (
|
|||
+ LIST_QUOTES
|
||||
+ [
|
||||
r"(?<=[0-9])\+",
|
||||
r"(?<=°[FfCcKk])\.", # 4°C. -> ["4°C", "."]
|
||||
r"(?<=°[FfCcKk])\.", # °C. -> ["°C", "."]
|
||||
r"(?<=[0-9])°[FfCcKk]", # 4°C -> ["4", "°C"]
|
||||
r"(?<=[0-9])%", # 4% -> ["4", "%"]
|
||||
r"(?<=[0-9])(?:{})".format(CURRENCY),
|
||||
r"(?<=[0-9])(?:{})".format(UNITS),
|
||||
r"(?<=[0-9{}{}(?:{})])\.".format(ALPHA_LOWER, r"%²\-\)\]\+", QUOTES),
|
||||
r"(?<=[0-9])(?:{c})".format(c=CURRENCY),
|
||||
r"(?<=[0-9])(?:{u})".format(u=UNITS),
|
||||
r"(?<=[0-9{al}{e}(?:{q})])\.".format(al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES),
|
||||
r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER),
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import regex as re
|
||||
import re
|
||||
|
||||
from ._tokenizer_exceptions_list import FR_BASE_EXCEPTIONS
|
||||
from .punctuation import ELISION, HYPHENS
|
||||
from ..tokenizer_exceptions import URL_PATTERN
|
||||
from ..char_classes import ALPHA_LOWER
|
||||
from ..char_classes import ALPHA_LOWER, ALPHA
|
||||
from ...symbols import ORTH, LEMMA, TAG
|
||||
|
||||
|
||||
|
|
@ -320,67 +320,67 @@ _hyphen_prefix = [
|
|||
_other_hyphens = "".join([h for h in HYPHENS if h != "-"])
|
||||
|
||||
_regular_exp = [
|
||||
"^a[{hyphen}]sexualis[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^arginine[{hyphen}]méthyl[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^binge[{hyphen}]watch[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^black[{hyphen}]out[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^bouche[{hyphen}]por[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^burn[{hyphen}]out[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^by[{hyphen}]pass[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^ch[{elision}]tiis[{alpha}]+$".format(elision=ELISION, alpha=ALPHA_LOWER),
|
||||
"^chape[{hyphen}]chut[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^down[{hyphen}]load[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^[ée]tats[{hyphen}]uni[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^droits?[{hyphen}]de[{hyphen}]l'homm[{alpha}]+$".format(
|
||||
hyphen=HYPHENS, alpha=ALPHA_LOWER
|
||||
"^a[{hyphen}]sexualis[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^arginine[{hyphen}]méthyl[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^binge[{hyphen}]watch[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^black[{hyphen}]out[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^bouche[{hyphen}]por[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^burn[{hyphen}]out[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^by[{hyphen}]pass[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^ch[{elision}]tiis[{al}]+$".format(elision=ELISION, al=ALPHA_LOWER),
|
||||
"^chape[{hyphen}]chut[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^down[{hyphen}]load[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^[ée]tats[{hyphen}]uni[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^droits?[{hyphen}]de[{hyphen}]l'homm[{al}]+$".format(
|
||||
hyphen=HYPHENS, al=ALPHA_LOWER
|
||||
),
|
||||
"^fac[{hyphen}]simil[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^fleur[{hyphen}]bleuis[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^flic[{hyphen}]flaqu[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^fox[{hyphen}]trott[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^google[{hyphen}]is[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^hard[{hyphen}]discount[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^hip[{hyphen}]hop[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^jet[{hyphen}]set[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^knock[{hyphen}]out[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^lèche[{hyphen}]bott[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^litho[{hyphen}]typographi[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^lock[{hyphen}]out[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^lombri[{hyphen}]compost[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^mac[{hyphen}]adamis[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^marque[{hyphen}]pag[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^mouton[{hyphen}]noiris[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^new[{hyphen}]york[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^pair[{hyphen}]programm[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^people[{hyphen}]is[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^plan[{hyphen}]socialis[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^premier[{hyphen}]ministr[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^prud[{elision}]hom[{alpha}]+$".format(elision=ELISION, alpha=ALPHA_LOWER),
|
||||
"^réarc[{hyphen}]bout[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^refox[{hyphen}]trott[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^remicro[{hyphen}]ond[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^repique[{hyphen}]niqu[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^repetit[{hyphen}]déjeun[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^rick[{hyphen}]roll[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^rond[{hyphen}]ponn[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^shift[{hyphen}]cliqu[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^soudo[{hyphen}]bras[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^stabilo[{hyphen}]boss[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^strip[{hyphen}]teas[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^terra[{hyphen}]form[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^teuf[{hyphen}]teuf[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^yo[{hyphen}]yo[{alpha}]+$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^zig[{hyphen}]zag[{alpha}]*$".format(hyphen=HYPHENS, alpha=ALPHA_LOWER),
|
||||
"^z[{elision}]yeut[{alpha}]+$".format(elision=ELISION, alpha=ALPHA_LOWER),
|
||||
"^fac[{hyphen}]simil[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^fleur[{hyphen}]bleuis[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^flic[{hyphen}]flaqu[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^fox[{hyphen}]trott[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^google[{hyphen}]is[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^hard[{hyphen}]discount[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^hip[{hyphen}]hop[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^jet[{hyphen}]set[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^knock[{hyphen}]out[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^lèche[{hyphen}]bott[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^litho[{hyphen}]typographi[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^lock[{hyphen}]out[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^lombri[{hyphen}]compost[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^mac[{hyphen}]adamis[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^marque[{hyphen}]pag[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^mouton[{hyphen}]noiris[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^new[{hyphen}]york[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^pair[{hyphen}]programm[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^people[{hyphen}]is[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^plan[{hyphen}]socialis[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^premier[{hyphen}]ministr[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^prud[{elision}]hom[{al}]+$".format(elision=ELISION, al=ALPHA_LOWER),
|
||||
"^réarc[{hyphen}]bout[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^refox[{hyphen}]trott[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^remicro[{hyphen}]ond[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^repique[{hyphen}]niqu[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^repetit[{hyphen}]déjeun[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^rick[{hyphen}]roll[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^rond[{hyphen}]ponn[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^shift[{hyphen}]cliqu[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^soudo[{hyphen}]bras[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^stabilo[{hyphen}]boss[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^strip[{hyphen}]teas[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^terra[{hyphen}]form[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^teuf[{hyphen}]teuf[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^yo[{hyphen}]yo[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^zig[{hyphen}]zag[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER),
|
||||
"^z[{elision}]yeut[{al}]+$".format(elision=ELISION, al=ALPHA_LOWER),
|
||||
]
|
||||
# catching cases like faux-vampire
|
||||
_regular_exp += [
|
||||
"^{prefix}[{hyphen}][{alpha}][{alpha}{elision}{other_hyphen}\-]*$".format(
|
||||
"^{prefix}[{hyphen}][{al}][{al}{elision}{other_hyphen}\-]*$".format(
|
||||
prefix=p,
|
||||
hyphen=HYPHENS,
|
||||
other_hyphen=_other_hyphens,
|
||||
elision=ELISION,
|
||||
alpha=ALPHA_LOWER,
|
||||
al=ALPHA_LOWER,
|
||||
)
|
||||
for p in _hyphen_prefix
|
||||
]
|
||||
|
|
@ -388,8 +388,8 @@ _regular_exp += [
|
|||
# catching cases like entr'abat
|
||||
_elision_prefix = ["r?é?entr", "grande?s?", "r"]
|
||||
_regular_exp += [
|
||||
"^{prefix}[{elision}][{alpha}][{alpha}{elision}{hyphen}\-]*$".format(
|
||||
prefix=p, elision=ELISION, hyphen=_other_hyphens, alpha=ALPHA_LOWER
|
||||
"^{prefix}[{elision}][{al}][{al}{elision}{hyphen}\-]*$".format(
|
||||
prefix=p, elision=ELISION, hyphen=_other_hyphens, al=ALPHA_LOWER
|
||||
)
|
||||
for p in _elision_prefix
|
||||
]
|
||||
|
|
@ -410,8 +410,8 @@ _hyphen_combination = [
|
|||
"saint",
|
||||
]
|
||||
_regular_exp += [
|
||||
"^[{alpha}]+[{hyphen}]{hyphen_combo}[{hyphen}](?:l[{elision}])?[{alpha}]+$".format(
|
||||
hyphen_combo=hc, elision=ELISION, hyphen=HYPHENS, alpha=ALPHA_LOWER
|
||||
"^[{a}]+[{hyphen}]{hyphen_combo}[{hyphen}](?:l[{elision}])?[{a}]+$".format(
|
||||
hyphen_combo=hc, elision=ELISION, hyphen=HYPHENS, a=ALPHA
|
||||
)
|
||||
for hc in _hyphen_combination
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,20 +1,21 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES
|
||||
from ..char_classes import QUOTES, UNITS, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||||
from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_ICONS
|
||||
from ..char_classes import CONCAT_QUOTES, CONCAT_ICONS, UNITS, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||||
|
||||
LIST_ICONS = [r"[\p{So}--[°]]"]
|
||||
# removing ° from the special icons to keep e.g. 99° as one token
|
||||
_concat_icons = CONCAT_ICONS.replace("\u00B0", "")
|
||||
|
||||
_currency = r"\$|¢|£|€|¥|฿"
|
||||
_quotes = QUOTES.replace("'", "")
|
||||
_currency = r"\$¢£€¥฿"
|
||||
_quotes = CONCAT_QUOTES.replace("'", "")
|
||||
|
||||
_prefixes = (
|
||||
[r"\+"]
|
||||
+ LIST_PUNCT
|
||||
+ LIST_ELLIPSES
|
||||
+ LIST_QUOTES
|
||||
+ LIST_ICONS
|
||||
+ [_concat_icons]
|
||||
+ [r"[,.:](?=[{a}])".format(a=ALPHA)]
|
||||
)
|
||||
|
||||
|
|
@ -22,24 +23,24 @@ _suffixes = (
|
|||
LIST_PUNCT
|
||||
+ LIST_ELLIPSES
|
||||
+ LIST_QUOTES
|
||||
+ LIST_ICONS
|
||||
+ [_concat_icons]
|
||||
+ [
|
||||
r"(?<=[0-9])\+",
|
||||
r"(?<=°[FfCcKk])\.",
|
||||
r"(?<=[0-9])(?:{})".format(_currency),
|
||||
r"(?<=[0-9])(?:{})".format(UNITS),
|
||||
r"(?<=[{}{}{}(?:{})])\.".format(ALPHA_LOWER, r"%²\-\)\]\+", QUOTES, _currency),
|
||||
r"(?<=[{})])-e".format(ALPHA_LOWER),
|
||||
r"(?<=[0-9])(?:[{c}])".format(c=_currency),
|
||||
r"(?<=[0-9])(?:{u})".format(u=UNITS),
|
||||
r"(?<=[{al}{e}{q}(?:{c})])\.".format(al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, c=_currency),
|
||||
r"(?<=[{al})])-e".format(al=ALPHA_LOWER),
|
||||
]
|
||||
)
|
||||
|
||||
_infixes = (
|
||||
LIST_ELLIPSES
|
||||
+ LIST_ICONS
|
||||
+ [_concat_icons]
|
||||
+ [
|
||||
r"(?<=[{}])\.(?=[{}])".format(ALPHA_LOWER, ALPHA_UPPER),
|
||||
r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER),
|
||||
r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA),
|
||||
r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
|
||||
r'(?<=[{a}])[:<>=](?=[{a}])'.format(a=ALPHA),
|
||||
r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}])([{q}\)\]\(\[])(?=[\-{a}])".format(a=ALPHA, q=_quotes),
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import regex as re
|
||||
import re
|
||||
|
||||
from ..punctuation import ALPHA_LOWER, CURRENCY
|
||||
from ..tokenizer_exceptions import URL_PATTERN
|
||||
|
|
@ -639,7 +639,7 @@ for orth in [
|
|||
_ord_num_or_date = "([A-Z0-9]+[./-])*(\d+\.?)"
|
||||
_num = "[+\-]?\d+([,.]\d+)*"
|
||||
_ops = "[=<>+\-\*/^()÷%²]"
|
||||
_suffixes = "-[{a}]+".format(a=ALPHA_LOWER)
|
||||
_suffixes = "-[{al}]+".format(al=ALPHA_LOWER)
|
||||
_numeric_exp = "({n})(({o})({n}))*[%]?".format(n=_num, o=_ops)
|
||||
_time_exp = "\d+(:\d+)*(\.\d+)?"
|
||||
|
||||
|
|
|
|||
|
|
@ -35,10 +35,12 @@ _suffixes = (
|
|||
TOKENIZER_SUFFIXES
|
||||
+ [r"\-[Nn]ya", "-[KkMm]u", "[—-]"]
|
||||
+ [
|
||||
r"(?<={c})(?:[0-9]+)".format(c=CURRENCY),
|
||||
# disabled: variable width currency variable
|
||||
# r"(?<={c})(?:[0-9]+)".format(c=CURRENCY),
|
||||
r"(?<=[0-9])(?:{u})".format(u=UNITS),
|
||||
r"(?<=[0-9])%",
|
||||
r"(?<=[0-9{a}]{h})(?:[\.,:-])".format(a=ALPHA, h=HTML_SUFFIX),
|
||||
# disabled: variable width HTML_SUFFIX variable
|
||||
# r"(?<=[0-9{a}]{h})(?:[\.,:-])".format(a=ALPHA, h=HTML_SUFFIX),
|
||||
r"(?<=[0-9{a}])(?:{h})".format(a=ALPHA, h=HTML_SUFFIX),
|
||||
]
|
||||
)
|
||||
|
|
@ -46,13 +48,15 @@ _suffixes = (
|
|||
_infixes = TOKENIZER_INFIXES + [
|
||||
r"(?<=[0-9])[\\/](?=[0-9%-])",
|
||||
r"(?<=[0-9])%(?=[{a}0-9/])".format(a=ALPHA),
|
||||
r"(?<={u})[\/-](?=[0-9])".format(u=UNITS),
|
||||
r"(?<={m})[\/-](?=[0-9])".format(m=MONTHS),
|
||||
r'(?<=[0-9\)][\.,])"(?=[0-9])',
|
||||
r'(?<=[{a}\)][\.,\'])["—](?=[{a}])'.format(a=ALPHA),
|
||||
# disabled: variable width units variable
|
||||
# r"(?<={u})[\/-](?=[0-9])".format(u=UNITS),
|
||||
# disabled: variable width months variable
|
||||
# r"(?<={m})[\/-](?=[0-9])".format(m=MONTHS),
|
||||
r'(?<=[0-9)][.,])"(?=[0-9])',
|
||||
r'(?<=[{a})][.,\'])["—](?=[{a}])'.format(a=ALPHA),
|
||||
r"(?<=[{a}])-(?=[0-9])".format(a=ALPHA),
|
||||
r"(?<=[0-9])-(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}])[\/-](?={c}{a})".format(a=ALPHA, c=CURRENCY),
|
||||
r"(?<=[{a}])[\/-](?={c}|[{a}])".format(a=ALPHA, c=CURRENCY),
|
||||
]
|
||||
|
||||
TOKENIZER_PREFIXES = _prefixes
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
from __future__ import unicode_literals
|
||||
|
||||
import unicodedata
|
||||
import regex as re
|
||||
import re
|
||||
|
||||
from .. import attrs
|
||||
|
||||
|
|
|
|||
|
|
@ -2,21 +2,21 @@
|
|||
from __future__ import unicode_literals
|
||||
|
||||
from ..char_classes import LIST_ELLIPSES, LIST_ICONS
|
||||
from ..char_classes import QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||||
from ..char_classes import CONCAT_QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
|
||||
from ..punctuation import TOKENIZER_SUFFIXES
|
||||
|
||||
# Punctuation stolen from Danish
|
||||
_quotes = QUOTES.replace("'", "")
|
||||
_quotes = CONCAT_QUOTES.replace("'", "")
|
||||
|
||||
_infixes = (
|
||||
LIST_ELLIPSES
|
||||
+ LIST_ICONS
|
||||
+ [
|
||||
r"(?<=[{}])\.(?=[{}])".format(ALPHA_LOWER, ALPHA_UPPER),
|
||||
r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER),
|
||||
r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA),
|
||||
r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
|
||||
r'(?<=[{a}])[:<>=](?=[{a}])'.format(a=ALPHA),
|
||||
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}])([{q}\)\]\(\[])(?=[\{a}])".format(a=ALPHA, q=_quotes),
|
||||
r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes),
|
||||
r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA),
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY
|
||||
from .char_classes import LIST_ICONS, ALPHA_LOWER, ALPHA_UPPER, ALPHA, HYPHENS
|
||||
from .char_classes import QUOTES, CURRENCY, UNITS
|
||||
from .char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY, LIST_ICONS
|
||||
from .char_classes import HYPHENS
|
||||
from .char_classes import CURRENCY, UNITS
|
||||
from .char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA
|
||||
|
||||
|
||||
_prefixes = (
|
||||
|
|
@ -25,27 +26,25 @@ _suffixes = (
|
|||
+ [
|
||||
r"(?<=[0-9])\+",
|
||||
r"(?<=°[FfCcKk])\.",
|
||||
r"(?<=[0-9])(?:{})".format(CURRENCY),
|
||||
r"(?<=[0-9])(?:{})".format(UNITS),
|
||||
r"(?<=[0-9{}{}(?:{})])\.".format(ALPHA_LOWER, r"%²\-\)\]\+", QUOTES),
|
||||
r"(?<=[{a}][{a}])\.".format(a=ALPHA_UPPER),
|
||||
r"(?<=[0-9])(?:{c})".format(c=CURRENCY),
|
||||
r"(?<=[0-9])(?:{u})".format(u=UNITS),
|
||||
r"(?<=[0-9{al}{e}(?:{q})])\.".format(al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES),
|
||||
r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
_infixes = (
|
||||
LIST_ELLIPSES
|
||||
+ LIST_ICONS
|
||||
+ [
|
||||
r"(?<=[0-9])[+\-\*^](?=[0-9-])",
|
||||
r"(?<=[{}])\.(?=[{}])".format(ALPHA_LOWER, ALPHA_UPPER),
|
||||
r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER),
|
||||
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
|
||||
r'(?<=[{a}])[?";:=,.]*(?:{h})(?=[{a}])'.format(a=ALPHA, h=HYPHENS),
|
||||
r'(?<=[{a}"])[:<>=/](?=[{a}])'.format(a=ALPHA),
|
||||
r'(?<=[{a}])(?:{h})(?=[{a}])'.format(a=ALPHA, h=HYPHENS),
|
||||
r'(?<=[{a}])[:<>=/](?=[{a}])'.format(a=ALPHA),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
TOKENIZER_PREFIXES = _prefixes
|
||||
TOKENIZER_SUFFIXES = _suffixes
|
||||
TOKENIZER_INFIXES = _infixes
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
# The use of this module turns out to be important, to avoid pathological
|
||||
# back-tracking. See Issue #957
|
||||
import regex as re
|
||||
import re
|
||||
|
||||
from ..symbols import ORTH, POS, TAG, LEMMA, SPACE, PUNCT
|
||||
|
||||
|
|
@ -38,7 +36,7 @@ URL_PATTERN = (
|
|||
# host name
|
||||
r"(?:(?:[a-z0-9\-]*)?[a-z0-9]+)"
|
||||
# domain name
|
||||
r"(?:\.(?:[a-z0-9\-])*[a-z0-9]+)*"
|
||||
r"(?:\.(?:[a-z0-9])(?:[a-z0-9\-])*[a-z0-9])?"
|
||||
# TLD identifier
|
||||
r"(?:\.(?:[a-z]{2,}))"
|
||||
r")"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from ..char_classes import ALPHA, ALPHA_LOWER, ALPHA_UPPER, QUOTES, HYPHENS
|
||||
from ..char_classes import ALPHA, ALPHA_LOWER, ALPHA_UPPER, CONCAT_QUOTES, HYPHENS
|
||||
from ..char_classes import LIST_ELLIPSES, LIST_ICONS
|
||||
|
||||
_hyphens_no_dash = HYPHENS.replace("-", "").strip("|").replace("||", "")
|
||||
|
|
@ -9,13 +9,13 @@ _infixes = (
|
|||
LIST_ELLIPSES
|
||||
+ LIST_ICONS
|
||||
+ [
|
||||
r"(?<=[{}])\.(?=[{}])".format(ALPHA_LOWER, ALPHA_UPPER),
|
||||
r"(?<=[{a}])[,!?/\(\)]+(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}{q}])[:<>=](?=[{a}])".format(a=ALPHA, q=QUOTES),
|
||||
r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER),
|
||||
r"(?<=[{a}])[,!?/()]+(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}{q}])[:<>=](?=[{a}])".format(a=ALPHA, q=CONCAT_QUOTES),
|
||||
r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
|
||||
r"(?<=[{a}])([{q}\)\]\(\[])(?=[\-{a}])".format(a=ALPHA, q=QUOTES),
|
||||
r'(?<=[{a}])[?";:=,.]*(?:{h})(?=[{a}])'.format(a=ALPHA, h=_hyphens_no_dash),
|
||||
r"(?<=[{a}])([{q}\)\]\(\[])(?=[\-{a}])".format(a=ALPHA, q=CONCAT_QUOTES),
|
||||
r'(?<=[{a}])(?:{h})(?=[{a}])'.format(a=ALPHA, h=_hyphens_no_dash),
|
||||
r"(?<=[0-9])-(?=[0-9])",
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ def custom_en_tokenizer(en_vocab):
|
|||
custom_infixes = [
|
||||
"\.\.\.+",
|
||||
"(?<=[0-9])-(?=[0-9])",
|
||||
# '(?<=[0-9]+),(?=[0-9]+)',
|
||||
"[0-9]+(,[0-9]+)+",
|
||||
"[\[\]!&:,()\*—–\/-]",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ def test_en_tokenizer_excludes_ambiguous(en_tokenizer, exc):
|
|||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"wo_punct,w_punct", [("We've", "``We've"), ("couldn't", "couldn't)")]
|
||||
"wo_punct,w_punct", [("We've", "`We've"), ("couldn't", "couldn't)")]
|
||||
)
|
||||
def test_en_tokenizer_splits_defined_punct(en_tokenizer, wo_punct, w_punct):
|
||||
tokens = en_tokenizer(wo_punct)
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ def test_en_tokenizer_splits_open_appostrophe(en_tokenizer, text):
|
|||
assert tokens[0].text == "'"
|
||||
|
||||
|
||||
@pytest.mark.xfail
|
||||
@pytest.mark.parametrize("text", ["Hello''"])
|
||||
def test_en_tokenizer_splits_double_end_quote(en_tokenizer, text):
|
||||
tokens = en_tokenizer(text)
|
||||
|
|
|
|||
|
|
@ -7,33 +7,33 @@ import pytest
|
|||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"aujourd'hui",
|
||||
"Aujourd'hui",
|
||||
"prud'hommes",
|
||||
"prud’hommal",
|
||||
"audio-numérique",
|
||||
"Audio-numérique",
|
||||
"entr'amis",
|
||||
"entr'abat",
|
||||
"rentr'ouvertes",
|
||||
"grand'hamien",
|
||||
"Châteauneuf-la-Forêt",
|
||||
"Château-Guibert",
|
||||
"11-septembre",
|
||||
"11-Septembre",
|
||||
"refox-trottâmes",
|
||||
"K-POP",
|
||||
"K-Pop",
|
||||
"K-pop",
|
||||
"z'yeutes",
|
||||
"black-outeront",
|
||||
"états-unienne",
|
||||
"courtes-pattes",
|
||||
"court-pattes",
|
||||
"saut-de-ski",
|
||||
"Écourt-Saint-Quentin",
|
||||
"Bout-de-l'Îlien",
|
||||
"pet-en-l'air",
|
||||
u"aujourd'hui",
|
||||
u"Aujourd'hui",
|
||||
u"prud'hommes",
|
||||
u"prud’hommal",
|
||||
u"audio-numérique",
|
||||
u"Audio-numérique",
|
||||
u"entr'amis",
|
||||
u"entr'abat",
|
||||
u"rentr'ouvertes",
|
||||
u"grand'hamien",
|
||||
u"Châteauneuf-la-Forêt",
|
||||
u"Château-Guibert",
|
||||
u"11-septembre",
|
||||
u"11-Septembre",
|
||||
u"refox-trottâmes",
|
||||
u"K-POP",
|
||||
u"K-Pop",
|
||||
u"K-pop",
|
||||
u"z'yeutes",
|
||||
u"black-outeront",
|
||||
u"états-unienne",
|
||||
u"courtes-pattes",
|
||||
u"court-pattes",
|
||||
u"saut-de-ski",
|
||||
u"Écourt-Saint-Quentin",
|
||||
u"Bout-de-l'Îlien",
|
||||
u"pet-en-l'air",
|
||||
],
|
||||
)
|
||||
def test_fr_tokenizer_infix_exceptions(fr_tokenizer, text):
|
||||
|
|
|
|||
|
|
@ -80,6 +80,7 @@ def test_ru_tokenizer_splits_open_appostrophe(ru_tokenizer, text):
|
|||
assert tokens[0].text == "'"
|
||||
|
||||
|
||||
@pytest.mark.xfail
|
||||
@pytest.mark.parametrize("text", ["Тест''"])
|
||||
def test_ru_tokenizer_splits_double_end_quote(ru_tokenizer, text):
|
||||
tokens = ru_tokenizer(text)
|
||||
|
|
|
|||
|
|
@ -289,6 +289,7 @@ def test_control_issue792(en_tokenizer, text):
|
|||
assert "".join([token.text_with_ws for token in doc]) == text
|
||||
|
||||
|
||||
@pytest.mark.xfail
|
||||
@pytest.mark.parametrize(
|
||||
"text,tokens",
|
||||
[
|
||||
|
|
@ -402,15 +403,19 @@ def test_issue912(en_vocab, text, tag, lemma):
|
|||
assert doc[0].lemma_ == lemma
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_issue957(en_tokenizer):
|
||||
"""Test that spaCy doesn't hang on many periods."""
|
||||
"""Test that spaCy doesn't hang on many punctuation characters.
|
||||
If this test hangs, check (new) regular expressions for conflicting greedy operators
|
||||
"""
|
||||
# Skip test if pytest-timeout is not installed
|
||||
pytest.importorskip("pytest-timeout")
|
||||
string = "0"
|
||||
for i in range(1, 100):
|
||||
string += ".%d" % i
|
||||
doc = en_tokenizer(string)
|
||||
assert doc
|
||||
pytest.importorskip("pytest_timeout")
|
||||
for punct in ['.', ',', '\'', '\"', ':', '?', '!', ';', '-']:
|
||||
string = "0"
|
||||
for i in range(1, 100):
|
||||
string += punct + str(i)
|
||||
doc = en_tokenizer(string)
|
||||
assert doc
|
||||
|
||||
|
||||
@pytest.mark.xfail
|
||||
|
|
|
|||
12
spacy/tests/regression/test_issue2835.py
Normal file
12
spacy/tests/regression/test_issue2835.py
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
|
||||
def test_issue2835(en_tokenizer):
|
||||
"""Check that sentence doesn't cause an infinite loop in the tokenizer."""
|
||||
text = """
|
||||
oow.jspsearch.eventoracleopenworldsearch.technologyoraclesolarissearch.technologystoragesearch.technologylinuxsearch.technologyserverssearch.technologyvirtualizationsearch.technologyengineeredsystemspcodewwmkmppscem:
|
||||
"""
|
||||
doc = en_tokenizer(text)
|
||||
assert doc
|
||||
|
||||
|
|
@ -10,7 +10,7 @@ def test_issue2901():
|
|||
"""Test that `nlp` doesn't fail."""
|
||||
try:
|
||||
nlp = Japanese()
|
||||
except ImportError:
|
||||
except:
|
||||
pytest.skip()
|
||||
|
||||
doc = nlp("pythonが大好きです")
|
||||
|
|
|
|||
|
|
@ -28,6 +28,12 @@ def test_tokenizer_handles_punct(tokenizer):
|
|||
assert tokens[1].text != "Lorem"
|
||||
|
||||
|
||||
def test_tokenizer_handles_punct_braces(tokenizer):
|
||||
text = "Lorem, (ipsum)."
|
||||
tokens = tokenizer(text)
|
||||
assert len(tokens) == 6
|
||||
|
||||
|
||||
def test_tokenizer_handles_digits(tokenizer):
|
||||
exceptions = ["hu", "bn"]
|
||||
text = "Lorem ipsum: 1984."
|
||||
|
|
|
|||
|
|
@ -94,10 +94,10 @@ URLS_SHOULD_NOT_MATCH = [
|
|||
"http://.www.foo.bar./",
|
||||
"http://10.1.1.1",
|
||||
"NASDAQ:GOOG",
|
||||
"http://-a.b.co",
|
||||
pytest.param("foo.com", marks=pytest.mark.xfail()),
|
||||
pytest.param("http://1.1.1.1.1", marks=pytest.mark.xfail()),
|
||||
pytest.param("http://www.foo.bar./", marks=pytest.mark.xfail()),
|
||||
pytest.param("http://-a.b.co", marks=pytest.mark.xfail()),
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from cython.operator cimport dereference as deref
|
|||
from cython.operator cimport preincrement as preinc
|
||||
from cymem.cymem cimport Pool
|
||||
from preshed.maps cimport PreshMap
|
||||
import regex as re
|
||||
import re
|
||||
cimport cython
|
||||
|
||||
from .tokens.doc cimport Doc
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from __future__ import unicode_literals, print_function
|
|||
import os
|
||||
import pkg_resources
|
||||
import importlib
|
||||
import regex as re
|
||||
import re
|
||||
from pathlib import Path
|
||||
import random
|
||||
from collections import OrderedDict
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user