| Get a custom spaCy pipeline, tailor-made for your NLP problem by spaCy's core developers. Streamlined, production-ready, predictable and maintainable. Start by completing our 5-minute questionnaire to tell us what you need and we'll be in touch! **[Learn more →](https://explosion.ai/spacy-tailored-pipelines)** |
| 
| Bespoke advice for problem solving, strategy and analysis for applied NLP projects. Services include data strategy, code reviews, pipeline design and annotation coaching. Curious? Fill in our 5-minute questionnaire to tell us what you need and we'll be in touch! **[Learn more →](https://explosion.ai/spacy-tailored-analysis)** |
@@ -57,13 +58,13 @@ open-source software, released under the [MIT license](https://github.com/explos
[api reference]: https://spacy.io/api/
[models]: https://spacy.io/models
[universe]: https://spacy.io/universe
+[spaCy VS Code Extension]: https://github.com/explosion/spacy-vscode
[videos]: https://www.youtube.com/c/ExplosionAI
[online course]: https://course.spacy.io
[project templates]: https://github.com/explosion/projects
[changelog]: https://spacy.io/usage#changelog
[contribute]: https://github.com/explosion/spaCy/blob/master/CONTRIBUTING.md
-
## 💬 Where to ask questions
The spaCy project is maintained by the [spaCy team](https://explosion.ai/about).
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
deleted file mode 100644
index 83c57a164..000000000
--- a/azure-pipelines.yml
+++ /dev/null
@@ -1,120 +0,0 @@
-trigger:
- batch: true
- branches:
- include:
- - "*"
- exclude:
- - "spacy.io"
- - "nightly.spacy.io"
- - "v2.spacy.io"
- paths:
- exclude:
- - "website/*"
- - "*.md"
- - "*.mdx"
- - ".github/workflows/*"
-pr:
- paths:
- exclude:
- - "*.md"
- - "*.mdx"
- - "website/docs/*"
- - "website/src/*"
- - "website/meta/*.tsx"
- - "website/meta/*.mjs"
- - "website/meta/languages.json"
- - "website/meta/site.json"
- - "website/meta/sidebars.json"
- - "website/meta/type-annotations.json"
- - "website/pages/*"
- - ".github/workflows/*"
-
-jobs:
- # Check formatting and linting. Perform basic checks for most important errors
- # (syntax etc.) Uses the config defined in setup.cfg and overwrites the
- # selected codes.
- - job: "Validate"
- pool:
- vmImage: "ubuntu-latest"
- steps:
- - task: UsePythonVersion@0
- inputs:
- versionSpec: "3.7"
- - script: |
- pip install black -c requirements.txt
- python -m black spacy --check
- displayName: "black"
- - script: |
- pip install flake8==5.0.4
- python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics
- displayName: "flake8"
- - script: |
- python .github/validate_universe_json.py website/meta/universe.json
- displayName: 'Validate website/meta/universe.json'
-
- - job: "Test"
- dependsOn: "Validate"
- strategy:
- matrix:
- # We're only running one platform per Python version to speed up builds
- Python36Linux:
- imageName: "ubuntu-20.04"
- python.version: "3.6"
- # Python36Windows:
- # imageName: "windows-latest"
- # python.version: "3.6"
- # Python36Mac:
- # imageName: "macos-latest"
- # python.version: "3.6"
- # Python37Linux:
- # imageName: "ubuntu-20.04"
- # python.version: "3.7"
- Python37Windows:
- imageName: "windows-latest"
- python.version: "3.7"
- # Python37Mac:
- # imageName: "macos-latest"
- # python.version: "3.7"
- # Python38Linux:
- # imageName: "ubuntu-latest"
- # python.version: "3.8"
- # Python38Windows:
- # imageName: "windows-latest"
- # python.version: "3.8"
- Python38Mac:
- imageName: "macos-latest"
- python.version: "3.8"
- Python39Linux:
- imageName: "ubuntu-latest"
- python.version: "3.9"
- # Python39Windows:
- # imageName: "windows-latest"
- # python.version: "3.9"
- # Python39Mac:
- # imageName: "macos-latest"
- # python.version: "3.9"
- # Python310Linux:
- # imageName: "ubuntu-latest"
- # python.version: "3.10"
- Python310Windows:
- imageName: "windows-latest"
- python.version: "3.10"
- # Python310Mac:
- # imageName: "macos-latest"
- # python.version: "3.10"
- Python311Linux:
- imageName: 'ubuntu-latest'
- python.version: '3.11'
- Python311Windows:
- imageName: 'windows-latest'
- python.version: '3.11'
- Python311Mac:
- imageName: 'macos-latest'
- python.version: '3.11'
- maxParallel: 4
- pool:
- vmImage: $(imageName)
- steps:
- - template: .github/azure-steps.yml
- parameters:
- python_version: '$(python.version)'
diff --git a/pyproject.toml b/pyproject.toml
index 9cd96ac2d..dcb5cf10d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -9,3 +9,6 @@ requires = [
"numpy>=1.15.0",
]
build-backend = "setuptools.build_meta"
+
+[tool.isort]
+profile = "black"
diff --git a/requirements.txt b/requirements.txt
index 94d6f23f4..f5050fee2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,7 +9,7 @@ murmurhash>=0.28.0,<1.1.0
wasabi>=0.9.1,<1.2.0
srsly>=2.4.3,<3.0.0
catalogue>=2.0.6,<2.1.0
-typer>=0.3.0,<0.8.0
+typer>=0.3.0,<0.10.0
pathy>=0.10.0
smart-open>=5.2.1,<7.0.0
weasel>=0.1.0,<0.2.0
@@ -39,3 +39,4 @@ types-setuptools>=57.0.0
types-requests
types-setuptools>=57.0.0
black==22.3.0
+isort>=5.0,<6.0
diff --git a/setup.cfg b/setup.cfg
index 6d3c2f12c..048bb3719 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -53,7 +53,7 @@ install_requires =
catalogue>=2.0.6,<2.1.0
weasel>=0.1.0,<0.2.0
# Third-party dependencies
- typer>=0.3.0,<0.8.0
+ typer>=0.3.0,<0.10.0
pathy>=0.10.0
smart-open>=5.2.1,<7.0.0
tqdm>=4.38.0,<5.0.0
@@ -79,41 +79,41 @@ transformers =
ray =
spacy_ray>=0.1.0,<1.0.0
cuda =
- cupy>=5.0.0b4,<12.0.0
+ cupy>=5.0.0b4,<13.0.0
cuda80 =
- cupy-cuda80>=5.0.0b4,<12.0.0
+ cupy-cuda80>=5.0.0b4,<13.0.0
cuda90 =
- cupy-cuda90>=5.0.0b4,<12.0.0
+ cupy-cuda90>=5.0.0b4,<13.0.0
cuda91 =
- cupy-cuda91>=5.0.0b4,<12.0.0
+ cupy-cuda91>=5.0.0b4,<13.0.0
cuda92 =
- cupy-cuda92>=5.0.0b4,<12.0.0
+ cupy-cuda92>=5.0.0b4,<13.0.0
cuda100 =
- cupy-cuda100>=5.0.0b4,<12.0.0
+ cupy-cuda100>=5.0.0b4,<13.0.0
cuda101 =
- cupy-cuda101>=5.0.0b4,<12.0.0
+ cupy-cuda101>=5.0.0b4,<13.0.0
cuda102 =
- cupy-cuda102>=5.0.0b4,<12.0.0
+ cupy-cuda102>=5.0.0b4,<13.0.0
cuda110 =
- cupy-cuda110>=5.0.0b4,<12.0.0
+ cupy-cuda110>=5.0.0b4,<13.0.0
cuda111 =
- cupy-cuda111>=5.0.0b4,<12.0.0
+ cupy-cuda111>=5.0.0b4,<13.0.0
cuda112 =
- cupy-cuda112>=5.0.0b4,<12.0.0
+ cupy-cuda112>=5.0.0b4,<13.0.0
cuda113 =
- cupy-cuda113>=5.0.0b4,<12.0.0
+ cupy-cuda113>=5.0.0b4,<13.0.0
cuda114 =
- cupy-cuda114>=5.0.0b4,<12.0.0
+ cupy-cuda114>=5.0.0b4,<13.0.0
cuda115 =
- cupy-cuda115>=5.0.0b4,<12.0.0
+ cupy-cuda115>=5.0.0b4,<13.0.0
cuda116 =
- cupy-cuda116>=5.0.0b4,<12.0.0
+ cupy-cuda116>=5.0.0b4,<13.0.0
cuda117 =
- cupy-cuda117>=5.0.0b4,<12.0.0
+ cupy-cuda117>=5.0.0b4,<13.0.0
cuda11x =
- cupy-cuda11x>=11.0.0,<12.0.0
+ cupy-cuda11x>=11.0.0,<13.0.0
cuda-autodetect =
- cupy-wheel>=11.0.0,<12.0.0
+ cupy-wheel>=11.0.0,<13.0.0
apple =
thinc-apple-ops>=0.1.0.dev0,<1.0.0
# Language tokenizers with external dependencies
diff --git a/spacy/__init__.py b/spacy/__init__.py
index c3568bc5c..1a18ad0d5 100644
--- a/spacy/__init__.py
+++ b/spacy/__init__.py
@@ -1,6 +1,6 @@
-from typing import Union, Iterable, Dict, Any
-from pathlib import Path
import sys
+from pathlib import Path
+from typing import Any, Dict, Iterable, Union
# set library-specific custom warning handling before doing anything else
from .errors import setup_default_warnings
@@ -8,20 +8,17 @@ from .errors import setup_default_warnings
setup_default_warnings() # noqa: E402
# These are imported as part of the API
-from thinc.api import prefer_gpu, require_gpu, require_cpu # noqa: F401
-from thinc.api import Config
+from thinc.api import Config, prefer_gpu, require_cpu, require_gpu # noqa: F401
from . import pipeline # noqa: F401
-from .cli.info import info # noqa: F401
-from .glossary import explain # noqa: F401
-from .about import __version__ # noqa: F401
-from .util import registry, logger # noqa: F401
-
-from .errors import Errors
-from .language import Language
-from .vocab import Vocab
from . import util
-
+from .about import __version__ # noqa: F401
+from .cli.info import info # noqa: F401
+from .errors import Errors
+from .glossary import explain # noqa: F401
+from .language import Language
+from .util import logger, registry # noqa: F401
+from .vocab import Vocab
if sys.maxunicode == 65535:
raise SystemError(Errors.E130)
diff --git a/spacy/about.py b/spacy/about.py
index 4748d655c..3319860f1 100644
--- a/spacy/about.py
+++ b/spacy/about.py
@@ -1,5 +1,5 @@
# fmt: off
__title__ = "spacy"
-__version__ = "3.5.0"
+__version__ = "3.6.0"
__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
diff --git a/spacy/attrs.pxd b/spacy/attrs.pxd
index 33d5372de..6dc9ecaee 100644
--- a/spacy/attrs.pxd
+++ b/spacy/attrs.pxd
@@ -1,6 +1,7 @@
# Reserve 64 values for flag features
from . cimport symbols
+
cdef enum attr_id_t:
NULL_ATTR
IS_ALPHA
diff --git a/spacy/cli/__init__.py b/spacy/cli/__init__.py
index efabcb9cf..549a27616 100644
--- a/spacy/cli/__init__.py
+++ b/spacy/cli/__init__.py
@@ -1,28 +1,35 @@
from wasabi import msg
from ._util import app, setup_cli # noqa: F401
+from .apply import apply # noqa: F401
+from .assemble import assemble_cli # noqa: F401
# These are the actual functions, NOT the wrapped CLI commands. The CLI commands
# are registered automatically and won't have to be imported here.
from .benchmark_speed import benchmark_speed_cli # noqa: F401
-from .download import download # noqa: F401
-from .info import info # noqa: F401
-from .package import package # noqa: F401
-from .profile import profile # noqa: F401
-from .train import train_cli # noqa: F401
-from .assemble import assemble_cli # noqa: F401
-from .pretrain import pretrain # noqa: F401
-from .debug_data import debug_data # noqa: F401
-from .debug_config import debug_config # noqa: F401
-from .debug_model import debug_model # noqa: F401
-from .debug_diff import debug_diff # noqa: F401
-from .evaluate import evaluate # noqa: F401
-from .apply import apply # noqa: F401
from .convert import convert # noqa: F401
-from .init_pipeline import init_pipeline_cli # noqa: F401
-from .init_config import init_config, fill_config # noqa: F401
-from .validate import validate # noqa: F401
+from .debug_config import debug_config # noqa: F401
+from .debug_data import debug_data # noqa: F401
+from .debug_diff import debug_diff # noqa: F401
+from .debug_model import debug_model # noqa: F401
+from .download import download # noqa: F401
+from .evaluate import evaluate # noqa: F401
from .find_threshold import find_threshold # noqa: F401
+from .info import info # noqa: F401
+from .init_config import fill_config, init_config # noqa: F401
+from .init_pipeline import init_pipeline_cli # noqa: F401
+from .package import package # noqa: F401
+from .pretrain import pretrain # noqa: F401
+from .profile import profile # noqa: F401
+from .project.assets import project_assets # noqa: F401
+from .project.clone import project_clone # noqa: F401
+from .project.document import project_document # noqa: F401
+from .project.dvc import project_update_dvc # noqa: F401
+from .project.pull import project_pull # noqa: F401
+from .project.push import project_push # noqa: F401
+from .project.run import project_run # noqa: F401
+from .train import train_cli # noqa: F401
+from .validate import validate # noqa: F401
@app.command("link", no_args_is_help=True, deprecated=True, hidden=True)
diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py
index 4e8102e3d..b48e928f5 100644
--- a/spacy/cli/_util.py
+++ b/spacy/cli/_util.py
@@ -1,25 +1,45 @@
-from typing import Dict, Any, Union, List, Optional, Tuple, Iterable
-from typing import TYPE_CHECKING, overload
-import sys
-import shutil
-from pathlib import Path
-from wasabi import msg, Printer
-import srsly
import hashlib
+import os
+import shutil
+import sys
+from configparser import InterpolationError
+from contextlib import contextmanager
+from pathlib import Path
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Dict,
+ Iterable,
+ List,
+ Optional,
+ Tuple,
+ Union,
+ overload,
+)
+
+import srsly
import typer
from click import NoSuchOption
from click.parser import split_arg_string
-from typer.main import get_command
-from contextlib import contextmanager
-from thinc.api import ConfigValidationError, require_gpu
+from thinc.api import Config, ConfigValidationError, require_gpu
from thinc.util import gpu_is_available
-from configparser import InterpolationError
-import os
-
+from typer.main import get_command
+from wasabi import Printer, msg
from weasel import app as project_cli
+from .. import about
from ..compat import Literal
-from ..util import import_file, run_command, registry, logger, ENV_VARS
+from ..schemas import ProjectConfigSchema, validate
+from ..util import (
+ ENV_VARS,
+ SimpleFrozenDict,
+ import_file,
+ is_compatible_version,
+ logger,
+ make_tempdir,
+ registry,
+ run_command,
+)
if TYPE_CHECKING:
from pathy import FluidPath # noqa: F401
diff --git a/spacy/cli/apply.py b/spacy/cli/apply.py
index f0df4e757..8c4b4c8bf 100644
--- a/spacy/cli/apply.py
+++ b/spacy/cli/apply.py
@@ -1,18 +1,15 @@
-import tqdm
-import srsly
-
from itertools import chain
from pathlib import Path
-from typing import Optional, List, Iterable, cast, Union
+from typing import Iterable, List, Optional, Union, cast
+import srsly
+import tqdm
from wasabi import msg
-from ._util import app, Arg, Opt, setup_gpu, import_code, walk_directory
-
from ..tokens import Doc, DocBin
-from ..vocab import Vocab
from ..util import ensure_path, load_model
-
+from ..vocab import Vocab
+from ._util import Arg, Opt, app, import_code, setup_gpu, walk_directory
path_help = """Location of the documents to predict on.
Can be a single file in .spacy format or a .jsonl file.
diff --git a/spacy/cli/assemble.py b/spacy/cli/assemble.py
index 1cfa290a3..ee2500b27 100644
--- a/spacy/cli/assemble.py
+++ b/spacy/cli/assemble.py
@@ -1,13 +1,20 @@
-from typing import Optional
-from pathlib import Path
-from wasabi import msg
-import typer
import logging
+from pathlib import Path
+from typing import Optional
+
+import typer
+from wasabi import msg
-from ._util import app, Arg, Opt, parse_config_overrides, show_validation_error
-from ._util import import_code
from .. import util
from ..util import get_sourced_components, load_model_from_config
+from ._util import (
+ Arg,
+ Opt,
+ app,
+ import_code,
+ parse_config_overrides,
+ show_validation_error,
+)
@app.command(
diff --git a/spacy/cli/benchmark_speed.py b/spacy/cli/benchmark_speed.py
index 4eb20a5fa..a683d1591 100644
--- a/spacy/cli/benchmark_speed.py
+++ b/spacy/cli/benchmark_speed.py
@@ -1,11 +1,12 @@
-from typing import Iterable, List, Optional
import random
-from itertools import islice
-import numpy
-from pathlib import Path
import time
-from tqdm import tqdm
+from itertools import islice
+from pathlib import Path
+from typing import Iterable, List, Optional
+
+import numpy
import typer
+from tqdm import tqdm
from wasabi import msg
from .. import util
diff --git a/spacy/cli/convert.py b/spacy/cli/convert.py
index 68d454b3e..a66a68133 100644
--- a/spacy/cli/convert.py
+++ b/spacy/cli/convert.py
@@ -1,18 +1,22 @@
-from typing import Callable, Iterable, Mapping, Optional, Any, Union
-from enum import Enum
-from pathlib import Path
-from wasabi import Printer
-import srsly
+import itertools
import re
import sys
-import itertools
+from enum import Enum
+from pathlib import Path
+from typing import Any, Callable, Iterable, Mapping, Optional, Union
+
+import srsly
+from wasabi import Printer
-from ._util import app, Arg, Opt, walk_directory
-from ..training import docs_to_json
from ..tokens import Doc, DocBin
-from ..training.converters import iob_to_docs, conll_ner_to_docs, json_to_docs
-from ..training.converters import conllu_to_docs
-
+from ..training import docs_to_json
+from ..training.converters import (
+ conll_ner_to_docs,
+ conllu_to_docs,
+ iob_to_docs,
+ json_to_docs,
+)
+from ._util import Arg, Opt, app, walk_directory
# Converters are matched by file extension except for ner/iob, which are
# matched by file extension and content. To add a converter, add a new
diff --git a/spacy/cli/debug_config.py b/spacy/cli/debug_config.py
index 409fac4ed..0e5382cd9 100644
--- a/spacy/cli/debug_config.py
+++ b/spacy/cli/debug_config.py
@@ -1,15 +1,22 @@
-from typing import Optional, Dict, Any, Union, List
from pathlib import Path
-from wasabi import msg, table
+from typing import Any, Dict, List, Optional, Union
+
+import typer
from thinc.api import Config
from thinc.config import VARIABLE_RE
-import typer
+from wasabi import msg, table
-from ._util import Arg, Opt, show_validation_error, parse_config_overrides
-from ._util import import_code, debug_cli
+from .. import util
from ..schemas import ConfigSchemaInit, ConfigSchemaTraining
from ..util import registry
-from .. import util
+from ._util import (
+ Arg,
+ Opt,
+ debug_cli,
+ import_code,
+ parse_config_overrides,
+ show_validation_error,
+)
@debug_cli.command(
diff --git a/spacy/cli/debug_data.py b/spacy/cli/debug_data.py
index 97b4db285..af3c24f3b 100644
--- a/spacy/cli/debug_data.py
+++ b/spacy/cli/debug_data.py
@@ -1,31 +1,49 @@
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
-from typing import cast, overload
-from pathlib import Path
-from collections import Counter
-import sys
-import srsly
-from wasabi import Printer, MESSAGES, msg
-import typer
import math
-import numpy
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import (
+ Any,
+ Dict,
+ Iterable,
+ List,
+ Optional,
+ Sequence,
+ Set,
+ Tuple,
+ Union,
+ cast,
+ overload,
+)
-from ._util import app, Arg, Opt, show_validation_error, parse_config_overrides
-from ._util import import_code, debug_cli, _format_number
-from ..training import Example, remove_bilu_prefix
-from ..training.initialize import get_sourced_components
-from ..schemas import ConfigSchemaTraining
-from ..pipeline import TrainablePipe
+import numpy
+import srsly
+import typer
+from wasabi import MESSAGES, Printer, msg
+
+from .. import util
+from ..compat import Literal
+from ..language import Language
+from ..morphology import Morphology
+from ..pipeline import Morphologizer, SpanCategorizer, TrainablePipe
+from ..pipeline._edit_tree_internals.edit_trees import EditTrees
from ..pipeline._parser_internals import nonproj
from ..pipeline._parser_internals.nonproj import DELIMITER
-from ..pipeline import Morphologizer, SpanCategorizer
-from ..pipeline._edit_tree_internals.edit_trees import EditTrees
-from ..morphology import Morphology
-from ..language import Language
+from ..schemas import ConfigSchemaTraining
+from ..training import Example, remove_bilu_prefix
+from ..training.initialize import get_sourced_components
from ..util import registry, resolve_dot_names
-from ..compat import Literal
from ..vectors import Mode as VectorsMode
-from .. import util
-
+from ._util import (
+ Arg,
+ Opt,
+ _format_number,
+ app,
+ debug_cli,
+ import_code,
+ parse_config_overrides,
+ show_validation_error,
+)
# Minimum number of expected occurrences of NER label in data to train new label
NEW_LABEL_THRESHOLD = 50
@@ -212,7 +230,7 @@ def debug_data(
else:
msg.info("No word vectors present in the package")
- if "spancat" in factory_names:
+ if "spancat" in factory_names or "spancat_singlelabel" in factory_names:
model_labels_spancat = _get_labels_from_spancat(nlp)
has_low_data_warning = False
has_no_neg_warning = False
@@ -337,7 +355,7 @@ def debug_data(
show=verbose,
)
else:
- msg.good("Examples without ocurrences available for all labels")
+ msg.good("Examples without occurrences available for all labels")
if "ner" in factory_names:
# Get all unique NER labels present in the data
@@ -830,7 +848,7 @@ def _compile_gold(
data["boundary_cross_ents"] += 1
elif label == "-":
data["ner"]["-"] += 1
- if "spancat" in factory_names:
+ if "spancat" in factory_names or "spancat_singlelabel" in factory_names:
for spans_key in list(eg.reference.spans.keys()):
# Obtain the span frequency
if spans_key not in data["spancat"]:
@@ -1028,7 +1046,7 @@ def _get_labels_from_spancat(nlp: Language) -> Dict[str, Set[str]]:
pipe_names = [
pipe_name
for pipe_name in nlp.pipe_names
- if nlp.get_pipe_meta(pipe_name).factory == "spancat"
+ if nlp.get_pipe_meta(pipe_name).factory in ("spancat", "spancat_singlelabel")
]
labels: Dict[str, Set[str]] = {}
for pipe_name in pipe_names:
diff --git a/spacy/cli/debug_diff.py b/spacy/cli/debug_diff.py
index 6697c38ae..c53b0acab 100644
--- a/spacy/cli/debug_diff.py
+++ b/spacy/cli/debug_diff.py
@@ -1,13 +1,13 @@
+from pathlib import Path
from typing import Optional
import typer
-from wasabi import Printer, diff_strings, MarkdownRenderer
-from pathlib import Path
from thinc.api import Config
+from wasabi import MarkdownRenderer, Printer, diff_strings
-from ._util import debug_cli, Arg, Opt, show_validation_error, parse_config_overrides
from ..util import load_config
-from .init_config import init_config, Optimizations
+from ._util import Arg, Opt, debug_cli, parse_config_overrides, show_validation_error
+from .init_config import Optimizations, init_config
@debug_cli.command(
diff --git a/spacy/cli/debug_model.py b/spacy/cli/debug_model.py
index 190094d81..8a0fd4889 100644
--- a/spacy/cli/debug_model.py
+++ b/spacy/cli/debug_model.py
@@ -1,19 +1,32 @@
-from typing import Dict, Any, Optional
-from pathlib import Path
import itertools
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import typer
+from thinc.api import (
+ Model,
+ data_validation,
+ fix_random_seed,
+ set_dropout_rate,
+ set_gpu_allocator,
+)
+from wasabi import msg
from spacy.training import Example
from spacy.util import resolve_dot_names
-from wasabi import msg
-from thinc.api import fix_random_seed, set_dropout_rate
-from thinc.api import Model, data_validation, set_gpu_allocator
-import typer
-from ._util import Arg, Opt, debug_cli, show_validation_error
-from ._util import parse_config_overrides, string_to_list, setup_gpu
+from .. import util
from ..schemas import ConfigSchemaTraining
from ..util import registry
-from .. import util
+from ._util import (
+ Arg,
+ Opt,
+ debug_cli,
+ parse_config_overrides,
+ setup_gpu,
+ show_validation_error,
+ string_to_list,
+)
@debug_cli.command(
diff --git a/spacy/cli/download.py b/spacy/cli/download.py
index 0c9a32b93..de731b0fd 100644
--- a/spacy/cli/download.py
+++ b/spacy/cli/download.py
@@ -1,14 +1,14 @@
-from typing import Optional, Sequence
-import requests
import sys
-from wasabi import msg
-import typer
+from typing import Optional, Sequence
+
+import requests
+import typer
+from wasabi import msg
-from ._util import app, Arg, Opt, WHEEL_SUFFIX, SDIST_SUFFIX
from .. import about
-from ..util import is_package, get_minor_version, run_command
-from ..util import is_prerelease_version
from ..errors import OLD_MODEL_SHORTCUTS
+from ..util import get_minor_version, is_package, is_prerelease_version, run_command
+from ._util import SDIST_SUFFIX, WHEEL_SUFFIX, Arg, Opt, app
@app.command(
@@ -81,11 +81,8 @@ def download(
def get_model_filename(model_name: str, version: str, sdist: bool = False) -> str:
dl_tpl = "{m}-{v}/{m}-{v}{s}"
- egg_tpl = "#egg={m}=={v}"
suffix = SDIST_SUFFIX if sdist else WHEEL_SUFFIX
filename = dl_tpl.format(m=model_name, v=version, s=suffix)
- if sdist:
- filename += egg_tpl.format(m=model_name, v=version)
return filename
diff --git a/spacy/cli/evaluate.py b/spacy/cli/evaluate.py
index 8f3d6b859..6235b658d 100644
--- a/spacy/cli/evaluate.py
+++ b/spacy/cli/evaluate.py
@@ -1,16 +1,16 @@
-from typing import Optional, List, Dict, Any, Union
-from wasabi import Printer
-from pathlib import Path
import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
import srsly
from thinc.api import fix_random_seed
+from wasabi import Printer
-from ..training import Corpus
-from ..tokens import Doc
-from ._util import app, Arg, Opt, setup_gpu, import_code, benchmark_cli
+from .. import displacy, util
from ..scorer import Scorer
-from .. import util
-from .. import displacy
+from ..tokens import Doc
+from ..training import Corpus
+from ._util import Arg, Opt, app, benchmark_cli, import_code, setup_gpu
@benchmark_cli.command(
@@ -27,6 +27,7 @@ def evaluate_cli(
gold_preproc: bool = Opt(False, "--gold-preproc", "-G", help="Use gold preprocessing"),
displacy_path: Optional[Path] = Opt(None, "--displacy-path", "-dp", help="Directory to output rendered parses as HTML", exists=True, file_okay=False),
displacy_limit: int = Opt(25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"),
+ per_component: bool = Opt(False, "--per-component", "-P", help="Return scores per component, only applicable when an output JSON file is specified."),
# fmt: on
):
"""
@@ -50,6 +51,7 @@ def evaluate_cli(
gold_preproc=gold_preproc,
displacy_path=displacy_path,
displacy_limit=displacy_limit,
+ per_component=per_component,
silent=False,
)
@@ -64,6 +66,7 @@ def evaluate(
displacy_limit: int = 25,
silent: bool = True,
spans_key: str = "sc",
+ per_component: bool = False,
) -> Dict[str, Any]:
msg = Printer(no_print=silent, pretty=not silent)
fix_random_seed()
@@ -78,50 +81,61 @@ def evaluate(
corpus = Corpus(data_path, gold_preproc=gold_preproc)
nlp = util.load_model(model)
dev_dataset = list(corpus(nlp))
- scores = nlp.evaluate(dev_dataset)
- metrics = {
- "TOK": "token_acc",
- "TAG": "tag_acc",
- "POS": "pos_acc",
- "MORPH": "morph_acc",
- "LEMMA": "lemma_acc",
- "UAS": "dep_uas",
- "LAS": "dep_las",
- "NER P": "ents_p",
- "NER R": "ents_r",
- "NER F": "ents_f",
- "TEXTCAT": "cats_score",
- "SENT P": "sents_p",
- "SENT R": "sents_r",
- "SENT F": "sents_f",
- "SPAN P": f"spans_{spans_key}_p",
- "SPAN R": f"spans_{spans_key}_r",
- "SPAN F": f"spans_{spans_key}_f",
- "SPEED": "speed",
- }
- results = {}
- data = {}
- for metric, key in metrics.items():
- if key in scores:
- if key == "cats_score":
- metric = metric + " (" + scores.get("cats_score_desc", "unk") + ")"
- if isinstance(scores[key], (int, float)):
- if key == "speed":
- results[metric] = f"{scores[key]:.0f}"
+ scores = nlp.evaluate(dev_dataset, per_component=per_component)
+ if per_component:
+ data = scores
+ if output is None:
+ msg.warn(
+ "The per-component option is enabled but there is no output JSON file provided to save the scores to."
+ )
+ else:
+ msg.info("Per-component scores will be saved to output JSON file.")
+ else:
+ metrics = {
+ "TOK": "token_acc",
+ "TAG": "tag_acc",
+ "POS": "pos_acc",
+ "MORPH": "morph_acc",
+ "LEMMA": "lemma_acc",
+ "UAS": "dep_uas",
+ "LAS": "dep_las",
+ "NER P": "ents_p",
+ "NER R": "ents_r",
+ "NER F": "ents_f",
+ "TEXTCAT": "cats_score",
+ "SENT P": "sents_p",
+ "SENT R": "sents_r",
+ "SENT F": "sents_f",
+ "SPAN P": f"spans_{spans_key}_p",
+ "SPAN R": f"spans_{spans_key}_r",
+ "SPAN F": f"spans_{spans_key}_f",
+ "SPEED": "speed",
+ }
+ results = {}
+ data = {}
+ for metric, key in metrics.items():
+ if key in scores:
+ if key == "cats_score":
+ metric = metric + " (" + scores.get("cats_score_desc", "unk") + ")"
+ if isinstance(scores[key], (int, float)):
+ if key == "speed":
+ results[metric] = f"{scores[key]:.0f}"
+ else:
+ results[metric] = f"{scores[key]*100:.2f}"
else:
- results[metric] = f"{scores[key]*100:.2f}"
- else:
- results[metric] = "-"
- data[re.sub(r"[\s/]", "_", key.lower())] = scores[key]
+ results[metric] = "-"
+ data[re.sub(r"[\s/]", "_", key.lower())] = scores[key]
- msg.table(results, title="Results")
- data = handle_scores_per_type(scores, data, spans_key=spans_key, silent=silent)
+ msg.table(results, title="Results")
+ data = handle_scores_per_type(scores, data, spans_key=spans_key, silent=silent)
if displacy_path:
factory_names = [nlp.get_pipe_meta(pipe).factory for pipe in nlp.pipe_names]
docs = list(nlp.pipe(ex.reference.text for ex in dev_dataset[:displacy_limit]))
render_deps = "parser" in factory_names
render_ents = "ner" in factory_names
+ render_spans = "spancat" in factory_names
+
render_parses(
docs,
displacy_path,
@@ -129,6 +143,7 @@ def evaluate(
limit=displacy_limit,
deps=render_deps,
ents=render_ents,
+ spans=render_spans,
)
msg.good(f"Generated {displacy_limit} parses as HTML", displacy_path)
@@ -182,6 +197,7 @@ def render_parses(
limit: int = 250,
deps: bool = True,
ents: bool = True,
+ spans: bool = True,
):
docs[0].user_data["title"] = model_name
if ents:
@@ -195,6 +211,11 @@ def render_parses(
with (output_path / "parses.html").open("w", encoding="utf8") as file_:
file_.write(html)
+ if spans:
+ html = displacy.render(docs[:limit], style="span", page=True)
+ with (output_path / "spans.html").open("w", encoding="utf8") as file_:
+ file_.write(html)
+
def print_prf_per_type(
msg: Printer, scores: Dict[str, Dict[str, float]], name: str, type: str
diff --git a/spacy/cli/find_threshold.py b/spacy/cli/find_threshold.py
index 6d591053d..7aa32c0c6 100644
--- a/spacy/cli/find_threshold.py
+++ b/spacy/cli/find_threshold.py
@@ -1,17 +1,17 @@
import functools
+import logging
import operator
from pathlib import Path
-import logging
-from typing import Optional, Tuple, Any, Dict, List
+from typing import Any, Dict, List, Optional, Tuple
import numpy
import wasabi.tables
-from ..pipeline import TextCategorizer, MultiLabel_TextCategorizer
-from ..errors import Errors
-from ..training import Corpus
-from ._util import app, Arg, Opt, import_code, setup_gpu
from .. import util
+from ..errors import Errors
+from ..pipeline import MultiLabel_TextCategorizer, TextCategorizer
+from ..training import Corpus
+from ._util import Arg, Opt, app, import_code, setup_gpu
_DEFAULTS = {
"n_trials": 11,
diff --git a/spacy/cli/info.py b/spacy/cli/info.py
index d82bf3fbc..8bfc6b54f 100644
--- a/spacy/cli/info.py
+++ b/spacy/cli/info.py
@@ -1,15 +1,15 @@
-from typing import Optional, Dict, Any, Union, List
-import platform
import json
+import platform
from pathlib import Path
-from wasabi import Printer, MarkdownRenderer
-import srsly
+from typing import Any, Dict, List, Optional, Union
-from ._util import app, Arg, Opt, string_to_list
-from .download import get_model_filename, get_latest_version
-from .. import util
-from .. import about
+import srsly
+from wasabi import MarkdownRenderer, Printer
+
+from .. import about, util
from ..compat import importlib_metadata
+from ._util import Arg, Opt, app, string_to_list
+from .download import get_latest_version, get_model_filename
@app.command("info")
diff --git a/spacy/cli/init_config.py b/spacy/cli/init_config.py
index b634caa4c..a7c03d00f 100644
--- a/spacy/cli/init_config.py
+++ b/spacy/cli/init_config.py
@@ -1,19 +1,26 @@
-from typing import Optional, List, Tuple
+import re
from enum import Enum
from pathlib import Path
-from wasabi import Printer, diff_strings
-from thinc.api import Config
+from typing import List, Optional, Tuple
+
import srsly
-import re
from jinja2 import Template
+from thinc.api import Config
+from wasabi import Printer, diff_strings
from .. import util
from ..language import DEFAULT_CONFIG_PRETRAIN_PATH
from ..schemas import RecommendationSchema
from ..util import SimpleFrozenList
-from ._util import init_cli, Arg, Opt, show_validation_error, COMMAND
-from ._util import string_to_list, import_code
-
+from ._util import (
+ COMMAND,
+ Arg,
+ Opt,
+ import_code,
+ init_cli,
+ show_validation_error,
+ string_to_list,
+)
ROOT = Path(__file__).parent / "templates"
TEMPLATE_PATH = ROOT / "quickstart_training.jinja"
diff --git a/spacy/cli/init_pipeline.py b/spacy/cli/init_pipeline.py
index d53a61b8e..13202cb60 100644
--- a/spacy/cli/init_pipeline.py
+++ b/spacy/cli/init_pipeline.py
@@ -1,15 +1,23 @@
-from typing import Optional
import logging
from pathlib import Path
-from wasabi import msg
-import typer
+from typing import Optional
+
import srsly
+import typer
+from wasabi import msg
from .. import util
-from ..training.initialize import init_nlp, convert_vectors
from ..language import Language
-from ._util import init_cli, Arg, Opt, parse_config_overrides, show_validation_error
-from ._util import import_code, setup_gpu
+from ..training.initialize import convert_vectors, init_nlp
+from ._util import (
+ Arg,
+ Opt,
+ import_code,
+ init_cli,
+ parse_config_overrides,
+ setup_gpu,
+ show_validation_error,
+)
@init_cli.command("vectors")
@@ -24,6 +32,7 @@ def init_vectors_cli(
name: Optional[str] = Opt(None, "--name", "-n", help="Optional name for the word vectors, e.g. en_core_web_lg.vectors"),
verbose: bool = Opt(False, "--verbose", "-V", "-VV", help="Display more information for debugging purposes"),
jsonl_loc: Optional[Path] = Opt(None, "--lexemes-jsonl", "-j", help="Location of JSONL-formatted attributes file", hidden=True),
+ attr: str = Opt("ORTH", "--attr", "-a", help="Optional token attribute to use for vectors, e.g. LOWER or NORM"),
# fmt: on
):
"""Convert word vectors for use with spaCy. Will export an nlp object that
@@ -42,6 +51,7 @@ def init_vectors_cli(
prune=prune,
name=name,
mode=mode,
+ attr=attr,
)
msg.good(f"Successfully converted {len(nlp.vocab.vectors)} vectors")
nlp.to_disk(output_dir)
diff --git a/spacy/cli/package.py b/spacy/cli/package.py
index 6351f28eb..4545578e6 100644
--- a/spacy/cli/package.py
+++ b/spacy/cli/package.py
@@ -1,18 +1,18 @@
-from typing import Optional, Union, Any, Dict, List, Tuple, cast
-import shutil
-from pathlib import Path
-from wasabi import Printer, MarkdownRenderer, get_raw_input
-from thinc.api import Config
-from collections import defaultdict
-from catalogue import RegistryError
-import srsly
-import sys
import re
+import shutil
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
-from ._util import app, Arg, Opt, string_to_list, WHEEL_SUFFIX, SDIST_SUFFIX
-from ..schemas import validate, ModelMetaSchema
-from .. import util
-from .. import about
+import srsly
+from catalogue import RegistryError
+from thinc.api import Config
+from wasabi import MarkdownRenderer, Printer, get_raw_input
+
+from .. import about, util
+from ..schemas import ModelMetaSchema, validate
+from ._util import SDIST_SUFFIX, WHEEL_SUFFIX, Arg, Opt, app, string_to_list
@app.command("package")
diff --git a/spacy/cli/pretrain.py b/spacy/cli/pretrain.py
index 381d589cf..446c40510 100644
--- a/spacy/cli/pretrain.py
+++ b/spacy/cli/pretrain.py
@@ -1,13 +1,21 @@
-from typing import Optional
-from pathlib import Path
-from wasabi import msg
-import typer
import re
+from pathlib import Path
+from typing import Optional
+
+import typer
+from wasabi import msg
-from ._util import app, Arg, Opt, parse_config_overrides, show_validation_error
-from ._util import import_code, setup_gpu
from ..training.pretrain import pretrain
from ..util import load_config
+from ._util import (
+ Arg,
+ Opt,
+ app,
+ import_code,
+ parse_config_overrides,
+ setup_gpu,
+ show_validation_error,
+)
@app.command(
@@ -23,6 +31,7 @@ def pretrain_cli(
resume_path: Optional[Path] = Opt(None, "--resume-path", "-r", help="Path to pretrained weights from which to resume pretraining"),
epoch_resume: Optional[int] = Opt(None, "--epoch-resume", "-er", help="The epoch to resume counting from when using --resume-path. Prevents unintended overwriting of existing weight files."),
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
+ skip_last: bool = Opt(False, "--skip-last", "-L", help="Skip saving model-last.bin"),
# fmt: on
):
"""
@@ -74,6 +83,7 @@ def pretrain_cli(
epoch_resume=epoch_resume,
use_gpu=use_gpu,
silent=False,
+ skip_last=skip_last,
)
msg.good("Successfully finished pretrain")
diff --git a/spacy/cli/profile.py b/spacy/cli/profile.py
index 3c282c73d..e1f720327 100644
--- a/spacy/cli/profile.py
+++ b/spacy/cli/profile.py
@@ -1,17 +1,18 @@
-from typing import Optional, Sequence, Union, Iterator
-import tqdm
-from pathlib import Path
-import srsly
import cProfile
+import itertools
import pstats
import sys
-import itertools
-from wasabi import msg, Printer
-import typer
+from pathlib import Path
+from typing import Iterator, Optional, Sequence, Union
+
+import srsly
+import tqdm
+import typer
+from wasabi import Printer, msg
-from ._util import app, debug_cli, Arg, Opt, NAME
from ..language import Language
from ..util import load_model
+from ._util import NAME, Arg, Opt, app, debug_cli
@debug_cli.command("profile")
diff --git a/spacy/cli/templates/quickstart_training.jinja b/spacy/cli/templates/quickstart_training.jinja
index 9481e53be..e3ca73cfb 100644
--- a/spacy/cli/templates/quickstart_training.jinja
+++ b/spacy/cli/templates/quickstart_training.jinja
@@ -3,7 +3,7 @@ the docs and the init config command. It encodes various best practices and
can help generate the best possible configuration, given a user's requirements. #}
{%- set use_transformer = hardware != "cpu" and transformer_data -%}
{%- set transformer = transformer_data[optimize] if use_transformer else {} -%}
-{%- set listener_components = ["tagger", "morphologizer", "parser", "ner", "textcat", "textcat_multilabel", "entity_linker", "spancat", "spancat_singlelabel", "trainable_lemmatizer"] -%}
+{%- set listener_components = ["tagger", "morphologizer", "parser", "ner", "textcat", "textcat_multilabel", "entity_linker", "span_finder", "spancat", "spancat_singlelabel", "trainable_lemmatizer"] -%}
[paths]
train = null
dev = null
@@ -28,7 +28,7 @@ lang = "{{ lang }}"
tok2vec/transformer. #}
{%- set with_accuracy_or_transformer = (use_transformer or with_accuracy) -%}
{%- set textcat_needs_features = has_textcat and with_accuracy_or_transformer -%}
-{%- if ("tagger" in components or "morphologizer" in components or "parser" in components or "ner" in components or "spancat" in components or "spancat_singlelabel" in components or "trainable_lemmatizer" in components or "entity_linker" in components or textcat_needs_features) -%}
+{%- if ("tagger" in components or "morphologizer" in components or "parser" in components or "ner" in components or "span_finder" in components or "spancat" in components or "spancat_singlelabel" in components or "trainable_lemmatizer" in components or "entity_linker" in components or textcat_needs_features) -%}
{%- set full_pipeline = ["transformer" if use_transformer else "tok2vec"] + components -%}
{%- else -%}
{%- set full_pipeline = components -%}
@@ -127,6 +127,30 @@ grad_factor = 1.0
@layers = "reduce_mean.v1"
{% endif -%}
+{% if "span_finder" in components -%}
+[components.span_finder]
+factory = "span_finder"
+max_length = null
+min_length = null
+scorer = {"@scorers":"spacy.span_finder_scorer.v1"}
+spans_key = "sc"
+threshold = 0.5
+
+[components.span_finder.model]
+@architectures = "spacy.SpanFinder.v1"
+
+[components.span_finder.model.scorer]
+@layers = "spacy.LinearLogistic.v1"
+nO = 2
+
+[components.span_finder.model.tok2vec]
+@architectures = "spacy-transformers.TransformerListener.v1"
+grad_factor = 1.0
+
+[components.span_finder.model.tok2vec.pooling]
+@layers = "reduce_mean.v1"
+{% endif -%}
+
{% if "spancat" in components -%}
[components.spancat]
factory = "spancat"
@@ -392,6 +416,27 @@ nO = null
width = ${components.tok2vec.model.encode.width}
{% endif %}
+{% if "span_finder" in components %}
+[components.span_finder]
+factory = "span_finder"
+max_length = null
+min_length = null
+scorer = {"@scorers":"spacy.span_finder_scorer.v1"}
+spans_key = "sc"
+threshold = 0.5
+
+[components.span_finder.model]
+@architectures = "spacy.SpanFinder.v1"
+
+[components.span_finder.model.scorer]
+@layers = "spacy.LinearLogistic.v1"
+nO = 2
+
+[components.span_finder.model.tok2vec]
+@architectures = "spacy.Tok2VecListener.v1"
+width = ${components.tok2vec.model.encode.width}
+{% endif %}
+
{% if "spancat" in components %}
[components.spancat]
factory = "spancat"
diff --git a/spacy/cli/train.py b/spacy/cli/train.py
index cc22cbba6..8bdabd39c 100644
--- a/spacy/cli/train.py
+++ b/spacy/cli/train.py
@@ -1,15 +1,23 @@
-from typing import Optional, Dict, Any, Union
-from pathlib import Path
-from wasabi import msg
-import typer
import logging
import sys
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+import typer
+from wasabi import msg
-from ._util import app, Arg, Opt, parse_config_overrides, show_validation_error
-from ._util import import_code, setup_gpu
-from ..training.loop import train as train_nlp
-from ..training.initialize import init_nlp
from .. import util
+from ..training.initialize import init_nlp
+from ..training.loop import train as train_nlp
+from ._util import (
+ Arg,
+ Opt,
+ app,
+ import_code,
+ parse_config_overrides,
+ setup_gpu,
+ show_validation_error,
+)
@app.command(
diff --git a/spacy/cli/validate.py b/spacy/cli/validate.py
index a918e9a39..0426f05fd 100644
--- a/spacy/cli/validate.py
+++ b/spacy/cli/validate.py
@@ -1,14 +1,21 @@
-from typing import Tuple
-from pathlib import Path
import sys
-import requests
-from wasabi import msg, Printer
import warnings
+from pathlib import Path
+from typing import Tuple
+
+import requests
+from wasabi import Printer, msg
-from ._util import app
from .. import about
-from ..util import get_package_version, get_installed_models, get_minor_version
-from ..util import get_package_path, get_model_meta, is_compatible_version
+from ..util import (
+ get_installed_models,
+ get_minor_version,
+ get_model_meta,
+ get_package_path,
+ get_package_version,
+ is_compatible_version,
+)
+from ._util import app
@app.command("validate")
diff --git a/spacy/compat.py b/spacy/compat.py
index 89132735d..522fa30dd 100644
--- a/spacy/compat.py
+++ b/spacy/compat.py
@@ -1,5 +1,6 @@
"""Helpers for Python and platform compatibility."""
import sys
+
from thinc.util import copy_array
try:
diff --git a/spacy/displacy/__init__.py b/spacy/displacy/__init__.py
index ea6bba2c9..bde2d04fe 100644
--- a/spacy/displacy/__init__.py
+++ b/spacy/displacy/__init__.py
@@ -4,15 +4,13 @@ spaCy's built in visualization suite for dependencies and named entities.
DOCS: https://spacy.io/api/top-level#displacy
USAGE: https://spacy.io/usage/visualizers
"""
-from typing import Union, Iterable, Optional, Dict, Any, Callable
import warnings
+from typing import Any, Callable, Dict, Iterable, Optional, Union
-from .render import DependencyRenderer, EntityRenderer, SpanRenderer
-from ..tokens import Doc, Span
from ..errors import Errors, Warnings
-from ..util import is_in_jupyter
-from ..util import find_available_port
-
+from ..tokens import Doc, Span
+from ..util import find_available_port, is_in_jupyter
+from .render import DependencyRenderer, EntityRenderer, SpanRenderer
_html = {}
RENDER_WRAPPER = None
@@ -68,7 +66,7 @@ def render(
if jupyter or (jupyter is None and is_in_jupyter()):
# return HTML rendered by IPython display()
# See #4840 for details on span wrapper to disable mathjax
- from IPython.core.display import display, HTML
+ from IPython.core.display import HTML, display
return display(HTML('{}'.format(html)))
return html
@@ -125,13 +123,17 @@ def app(environ, start_response):
return [res]
-def parse_deps(orig_doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]:
+def parse_deps(
+ orig_doc: Union[Doc, Span], options: Dict[str, Any] = {}
+) -> Dict[str, Any]:
"""Generate dependency parse in {'words': [], 'arcs': []} format.
- orig_doc (Doc): Document to parse.
+ orig_doc (Union[Doc, Span]): Document to parse.
options (Dict[str, Any]): Dependency parse specific visualisation options.
RETURNS (dict): Generated dependency parse keyed by words and arcs.
"""
+ if isinstance(orig_doc, Span):
+ orig_doc = orig_doc.as_doc()
doc = Doc(orig_doc.vocab).from_bytes(
orig_doc.to_bytes(exclude=["user_data", "user_hooks"])
)
diff --git a/spacy/displacy/render.py b/spacy/displacy/render.py
index f74222dc2..86869e3b8 100644
--- a/spacy/displacy/render.py
+++ b/spacy/displacy/render.py
@@ -1,15 +1,29 @@
-from typing import Any, Dict, List, Optional, Tuple, Union
-import uuid
import itertools
+import uuid
+from typing import Any, Dict, List, Optional, Tuple, Union
from ..errors import Errors
from ..util import escape_html, minify_html, registry
-from .templates import TPL_DEP_ARCS, TPL_DEP_SVG, TPL_DEP_WORDS
-from .templates import TPL_DEP_WORDS_LEMMA, TPL_ENT, TPL_ENT_RTL, TPL_ENTS
-from .templates import TPL_FIGURE, TPL_KB_LINK, TPL_PAGE, TPL_SPAN
-from .templates import TPL_SPAN_RTL, TPL_SPAN_SLICE, TPL_SPAN_SLICE_RTL
-from .templates import TPL_SPAN_START, TPL_SPAN_START_RTL, TPL_SPANS
-from .templates import TPL_TITLE
+from .templates import (
+ TPL_DEP_ARCS,
+ TPL_DEP_SVG,
+ TPL_DEP_WORDS,
+ TPL_DEP_WORDS_LEMMA,
+ TPL_ENT,
+ TPL_ENT_RTL,
+ TPL_ENTS,
+ TPL_FIGURE,
+ TPL_KB_LINK,
+ TPL_PAGE,
+ TPL_SPAN,
+ TPL_SPAN_RTL,
+ TPL_SPAN_SLICE,
+ TPL_SPAN_SLICE_RTL,
+ TPL_SPAN_START,
+ TPL_SPAN_START_RTL,
+ TPL_SPANS,
+ TPL_TITLE,
+)
DEFAULT_LANG = "en"
DEFAULT_DIR = "ltr"
diff --git a/spacy/errors.py b/spacy/errors.py
index 526c4d0d6..24b60f8a3 100644
--- a/spacy/errors.py
+++ b/spacy/errors.py
@@ -1,4 +1,5 @@
import warnings
+
from .compat import Literal
@@ -215,6 +216,9 @@ class Warnings(metaclass=ErrorsWithCodes):
W123 = ("Argument `enable` with value {enable} does not contain all values specified in the config option "
"`enabled` ({enabled}). Be aware that this might affect other components in your pipeline.")
W124 = ("{host}:{port} is already in use, using the nearest available port {serve_port} as an alternative.")
+ W125 = ("The StaticVectors key_attr is no longer used. To set a custom "
+ "key attribute for vectors, configure it through Vectors(attr=) or "
+ "'spacy init vectors --attr'")
class Errors(metaclass=ErrorsWithCodes):
@@ -549,8 +553,8 @@ class Errors(metaclass=ErrorsWithCodes):
"during training, make sure to include it in 'annotating components'")
# New errors added in v3.x
- E850 = ("The PretrainVectors objective currently only supports default "
- "vectors, not {mode} vectors.")
+ E850 = ("The PretrainVectors objective currently only supports default or "
+ "floret vectors, not {mode} vectors.")
E851 = ("The 'textcat' component labels should only have values of 0 or 1, "
"but found value of '{val}'.")
E853 = ("Unsupported component factory name '{name}'. The character '.' is "
@@ -736,8 +740,8 @@ class Errors(metaclass=ErrorsWithCodes):
"model from a shortcut, which is obsolete as of spaCy v3.0. To "
"load the model, use its full name instead:\n\n"
"nlp = spacy.load(\"{full}\")\n\nFor more details on the available "
- "models, see the models directory: https://spacy.io/models. If you "
- "want to create a blank model, use spacy.blank: "
+ "models, see the models directory: https://spacy.io/models and if "
+ "you want to create a blank model, use spacy.blank: "
"nlp = spacy.blank(\"{name}\")")
E942 = ("Executing `after_{name}` callback failed. Expected the function to "
"return an initialized nlp object but got: {value}. Maybe "
@@ -968,6 +972,13 @@ class Errors(metaclass=ErrorsWithCodes):
E1050 = ("Port {port} is already in use. Please specify an available port with `displacy.serve(doc, port=port)` "
"or use `auto_select_port=True` to pick an available port automatically.")
E1051 = ("'allow_overlap' can only be False when max_positive is 1, but found 'max_positive': {max_positive}.")
+ E1052 = ("Unable to copy spans: the character offsets for the span at "
+ "index {i} in the span group do not align with the tokenization "
+ "in the target doc.")
+ E1053 = ("Both 'min_length' and 'max_length' should be larger than 0, but found"
+ " 'min_length': {min_length}, 'max_length': {max_length}")
+ E1054 = ("The text, including whitespace, must match between reference and "
+ "predicted docs when training {component}.")
# Deprecated model shortcuts, only used in errors and warnings
diff --git a/spacy/glossary.py b/spacy/glossary.py
index d2240fbba..1f628698b 100644
--- a/spacy/glossary.py
+++ b/spacy/glossary.py
@@ -1,4 +1,5 @@
import warnings
+
from .errors import Warnings
diff --git a/spacy/kb/__init__.py b/spacy/kb/__init__.py
index 1d70a9b34..3ce3e4c33 100644
--- a/spacy/kb/__init__.py
+++ b/spacy/kb/__init__.py
@@ -1,3 +1,3 @@
+from .candidate import Candidate, get_candidates, get_candidates_batch
from .kb import KnowledgeBase
from .kb_in_memory import InMemoryLookupKB
-from .candidate import Candidate, get_candidates, get_candidates_batch
diff --git a/spacy/kb/candidate.pxd b/spacy/kb/candidate.pxd
index 942ce9dd0..9fc4c4e9d 100644
--- a/spacy/kb/candidate.pxd
+++ b/spacy/kb/candidate.pxd
@@ -1,6 +1,8 @@
-from .kb cimport KnowledgeBase
from libcpp.vector cimport vector
+
from ..typedefs cimport hash_t
+from .kb cimport KnowledgeBase
+
# Object used by the Entity Linker that summarizes one entity-alias candidate combination.
cdef class Candidate:
diff --git a/spacy/kb/candidate.pyx b/spacy/kb/candidate.pyx
index c89efeb03..4cd734f43 100644
--- a/spacy/kb/candidate.pyx
+++ b/spacy/kb/candidate.pyx
@@ -1,9 +1,12 @@
# cython: infer_types=True, profile=True
from typing import Iterable
+
from .kb cimport KnowledgeBase
+
from ..tokens import Span
+
cdef class Candidate:
"""A `Candidate` object refers to a textual mention (`alias`) that may or may not be resolved
to a specific `entity` from a Knowledge Base. This will be used as input for the entity linking
diff --git a/spacy/kb/kb.pxd b/spacy/kb/kb.pxd
index 1adeef8ae..263469546 100644
--- a/spacy/kb/kb.pxd
+++ b/spacy/kb/kb.pxd
@@ -2,8 +2,10 @@
from cymem.cymem cimport Pool
from libc.stdint cimport int64_t
+
from ..vocab cimport Vocab
+
cdef class KnowledgeBase:
cdef Pool mem
cdef readonly Vocab vocab
diff --git a/spacy/kb/kb.pyx b/spacy/kb/kb.pyx
index ce4bc0138..a88e18e1f 100644
--- a/spacy/kb/kb.pyx
+++ b/spacy/kb/kb.pyx
@@ -2,12 +2,13 @@
from pathlib import Path
from typing import Iterable, Tuple, Union
+
from cymem.cymem cimport Pool
-from .candidate import Candidate
+from ..errors import Errors
from ..tokens import Span
from ..util import SimpleFrozenList
-from ..errors import Errors
+from .candidate import Candidate
cdef class KnowledgeBase:
diff --git a/spacy/kb/kb_in_memory.pxd b/spacy/kb/kb_in_memory.pxd
index 825a6bde9..08ec6b2a3 100644
--- a/spacy/kb/kb_in_memory.pxd
+++ b/spacy/kb/kb_in_memory.pxd
@@ -1,11 +1,11 @@
"""Knowledge-base for entity or concept linking."""
-from preshed.maps cimport PreshMap
-from libcpp.vector cimport vector
from libc.stdint cimport int32_t, int64_t
from libc.stdio cimport FILE
+from libcpp.vector cimport vector
+from preshed.maps cimport PreshMap
+from ..structs cimport AliasC, KBEntryC
from ..typedefs cimport hash_t
-from ..structs cimport KBEntryC, AliasC
from .kb cimport KnowledgeBase
ctypedef vector[KBEntryC] entry_vec
diff --git a/spacy/kb/kb_in_memory.pyx b/spacy/kb/kb_in_memory.pyx
index 2a74d047b..e991f7720 100644
--- a/spacy/kb/kb_in_memory.pyx
+++ b/spacy/kb/kb_in_memory.pyx
@@ -1,23 +1,28 @@
# cython: infer_types=True, profile=True
-from typing import Iterable, Callable, Dict, Any, Union
+from typing import Any, Callable, Dict, Iterable, Union
import srsly
-from preshed.maps cimport PreshMap
-from cpython.exc cimport PyErr_SetFromErrno
-from libc.stdio cimport fopen, fclose, fread, fwrite, feof, fseek
-from libc.stdint cimport int32_t, int64_t
-from libcpp.vector cimport vector
-from pathlib import Path
+from cpython.exc cimport PyErr_SetFromErrno
+from libc.stdint cimport int32_t, int64_t
+from libc.stdio cimport fclose, feof, fopen, fread, fseek, fwrite
+from libcpp.vector cimport vector
+from preshed.maps cimport PreshMap
+
import warnings
+from pathlib import Path
from ..tokens import Span
+
from ..typedefs cimport hash_t
-from ..errors import Errors, Warnings
+
from .. import util
+from ..errors import Errors, Warnings
from ..util import SimpleFrozenList, ensure_path
+
from ..vocab cimport Vocab
from .kb cimport KnowledgeBase
+
from .candidate import Candidate as Candidate
diff --git a/spacy/lang/af/__init__.py b/spacy/lang/af/__init__.py
index 553fcbf4c..8bd73c7ad 100644
--- a/spacy/lang/af/__init__.py
+++ b/spacy/lang/af/__init__.py
@@ -1,5 +1,5 @@
+from ...language import BaseDefaults, Language
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
class AfrikaansDefaults(BaseDefaults):
diff --git a/spacy/lang/am/__init__.py b/spacy/lang/am/__init__.py
index ddae556d6..284823eaa 100644
--- a/spacy/lang/am/__init__.py
+++ b/spacy/lang/am/__init__.py
@@ -1,12 +1,11 @@
-from .stop_words import STOP_WORDS
+from ...attrs import LANG
+from ...language import BaseDefaults, Language
+from ...util import update_exc
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
from .lex_attrs import LEX_ATTRS
from .punctuation import TOKENIZER_SUFFIXES
-
+from .stop_words import STOP_WORDS
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...language import Language, BaseDefaults
-from ...attrs import LANG
-from ...util import update_exc
class AmharicDefaults(BaseDefaults):
diff --git a/spacy/lang/am/punctuation.py b/spacy/lang/am/punctuation.py
index 555a179fa..87447b054 100644
--- a/spacy/lang/am/punctuation.py
+++ b/spacy/lang/am/punctuation.py
@@ -1,5 +1,11 @@
-from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, CURRENCY
-from ..char_classes import UNITS, ALPHA_UPPER
+from ..char_classes import (
+ ALPHA_UPPER,
+ CURRENCY,
+ LIST_ELLIPSES,
+ LIST_PUNCT,
+ LIST_QUOTES,
+ UNITS,
+)
_list_punct = LIST_PUNCT + "፡ ። ፣ ፤ ፥ ፦ ፧ ፠ ፨".strip().split()
diff --git a/spacy/lang/am/tokenizer_exceptions.py b/spacy/lang/am/tokenizer_exceptions.py
index 9472fe918..1ccf996ca 100644
--- a/spacy/lang/am/tokenizer_exceptions.py
+++ b/spacy/lang/am/tokenizer_exceptions.py
@@ -1,5 +1,4 @@
-from ...symbols import ORTH, NORM
-
+from ...symbols import NORM, ORTH
_exc = {}
diff --git a/spacy/lang/ar/__init__.py b/spacy/lang/ar/__init__.py
index 18c1f90ed..d50b0722c 100644
--- a/spacy/lang/ar/__init__.py
+++ b/spacy/lang/ar/__init__.py
@@ -1,8 +1,8 @@
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
from .punctuation import TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from ...language import Language, BaseDefaults
class ArabicDefaults(BaseDefaults):
diff --git a/spacy/lang/ar/punctuation.py b/spacy/lang/ar/punctuation.py
index f30204c02..cf03fc68e 100644
--- a/spacy/lang/ar/punctuation.py
+++ b/spacy/lang/ar/punctuation.py
@@ -1,5 +1,11 @@
-from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, CURRENCY
-from ..char_classes import UNITS, ALPHA_UPPER
+from ..char_classes import (
+ ALPHA_UPPER,
+ CURRENCY,
+ LIST_ELLIPSES,
+ LIST_PUNCT,
+ LIST_QUOTES,
+ UNITS,
+)
_suffixes = (
LIST_PUNCT
diff --git a/spacy/lang/ar/tokenizer_exceptions.py b/spacy/lang/ar/tokenizer_exceptions.py
index 7c385bef8..eb16876f5 100644
--- a/spacy/lang/ar/tokenizer_exceptions.py
+++ b/spacy/lang/ar/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/az/__init__.py b/spacy/lang/az/__init__.py
index 476898364..32949aa3e 100644
--- a/spacy/lang/az/__init__.py
+++ b/spacy/lang/az/__init__.py
@@ -1,6 +1,6 @@
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
-from ...language import Language, BaseDefaults
+from .stop_words import STOP_WORDS
class AzerbaijaniDefaults(BaseDefaults):
diff --git a/spacy/lang/az/lex_attrs.py b/spacy/lang/az/lex_attrs.py
index 73a5e2762..96fb7f020 100644
--- a/spacy/lang/az/lex_attrs.py
+++ b/spacy/lang/az/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
# Eleven, twelve etc. are written separate: on bir, on iki
_num_words = [
diff --git a/spacy/lang/bg/__init__.py b/spacy/lang/bg/__init__.py
index c9176b946..acca63ba1 100644
--- a/spacy/lang/bg/__init__.py
+++ b/spacy/lang/bg/__init__.py
@@ -1,12 +1,14 @@
+from ...attrs import LANG
+from ...language import BaseDefaults, Language
+from ...util import update_exc
+from ..punctuation import (
+ COMBINING_DIACRITICS_TOKENIZER_INFIXES,
+ COMBINING_DIACRITICS_TOKENIZER_SUFFIXES,
+)
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
+from .lex_attrs import LEX_ATTRS
from .stop_words import STOP_WORDS
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .lex_attrs import LEX_ATTRS
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_INFIXES
-from ..punctuation import COMBINING_DIACRITICS_TOKENIZER_SUFFIXES
-from ...language import Language, BaseDefaults
-from ...attrs import LANG
-from ...util import update_exc
class BulgarianDefaults(BaseDefaults):
diff --git a/spacy/lang/bg/lex_attrs.py b/spacy/lang/bg/lex_attrs.py
index bba3c74cd..0b7942aec 100644
--- a/spacy/lang/bg/lex_attrs.py
+++ b/spacy/lang/bg/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
_num_words = [
"нула",
"едно",
diff --git a/spacy/lang/bg/tokenizer_exceptions.py b/spacy/lang/bg/tokenizer_exceptions.py
index 0f484b778..89d466daf 100644
--- a/spacy/lang/bg/tokenizer_exceptions.py
+++ b/spacy/lang/bg/tokenizer_exceptions.py
@@ -4,8 +4,7 @@ References:
(countries, occupations, fields of studies and more).
"""
-from ...symbols import ORTH, NORM
-
+from ...symbols import NORM, ORTH
_exc = {}
diff --git a/spacy/lang/bn/__init__.py b/spacy/lang/bn/__init__.py
index 6d0331e00..6a5d37f5b 100644
--- a/spacy/lang/bn/__init__.py
+++ b/spacy/lang/bn/__init__.py
@@ -1,10 +1,12 @@
-from typing import Optional, Callable
+from typing import Callable, Optional
+
from thinc.api import Model
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
-from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
+
+from ...language import BaseDefaults, Language
from ...pipeline import Lemmatizer
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class BengaliDefaults(BaseDefaults):
diff --git a/spacy/lang/bn/punctuation.py b/spacy/lang/bn/punctuation.py
index becfe8d2a..ddb91cef1 100644
--- a/spacy/lang/bn/punctuation.py
+++ b/spacy/lang/bn/punctuation.py
@@ -1,6 +1,14 @@
-from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_ICONS
-from ..char_classes import ALPHA_LOWER, ALPHA, HYPHENS, CONCAT_QUOTES, UNITS
-
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ CONCAT_QUOTES,
+ HYPHENS,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+ LIST_PUNCT,
+ LIST_QUOTES,
+ UNITS,
+)
_currency = r"\$¢£€¥฿৳"
_quotes = CONCAT_QUOTES.replace("'", "")
diff --git a/spacy/lang/bn/tokenizer_exceptions.py b/spacy/lang/bn/tokenizer_exceptions.py
index e666522b8..016bf0fc5 100644
--- a/spacy/lang/bn/tokenizer_exceptions.py
+++ b/spacy/lang/bn/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/ca/__init__.py b/spacy/lang/ca/__init__.py
index a3def660d..8b2f3e85a 100755
--- a/spacy/lang/ca/__init__.py
+++ b/spacy/lang/ca/__init__.py
@@ -1,14 +1,14 @@
-from typing import Optional, Callable
+from typing import Callable, Optional
from thinc.api import Model
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES, TOKENIZER_PREFIXES
-from .stop_words import STOP_WORDS
-from .lex_attrs import LEX_ATTRS
-from .syntax_iterators import SYNTAX_ITERATORS
-from ...language import Language, BaseDefaults
+from ...language import BaseDefaults, Language
from .lemmatizer import CatalanLemmatizer
+from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class CatalanDefaults(BaseDefaults):
diff --git a/spacy/lang/ca/lex_attrs.py b/spacy/lang/ca/lex_attrs.py
index be8b7a6ea..3e99da0e0 100644
--- a/spacy/lang/ca/lex_attrs.py
+++ b/spacy/lang/ca/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
_num_words = [
"zero",
"un",
diff --git a/spacy/lang/ca/punctuation.py b/spacy/lang/ca/punctuation.py
index 8e2f09828..6914f67a7 100755
--- a/spacy/lang/ca/punctuation.py
+++ b/spacy/lang/ca/punctuation.py
@@ -1,9 +1,18 @@
-from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_ICONS
-from ..char_classes import LIST_CURRENCY
-from ..char_classes import CURRENCY
-from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA, PUNCT
-from ..char_classes import merge_chars, _units
-
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ CURRENCY,
+ LIST_CURRENCY,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+ LIST_PUNCT,
+ LIST_QUOTES,
+ PUNCT,
+ _units,
+ merge_chars,
+)
ELISION = " ' ’ ".strip().replace(" ", "").replace("\n", "")
diff --git a/spacy/lang/ca/syntax_iterators.py b/spacy/lang/ca/syntax_iterators.py
index 917e07c93..16a4c6a81 100644
--- a/spacy/lang/ca/syntax_iterators.py
+++ b/spacy/lang/ca/syntax_iterators.py
@@ -1,7 +1,8 @@
-from typing import Union, Iterator, Tuple
-from ...tokens import Doc, Span
-from ...symbols import NOUN, PROPN
+from typing import Iterator, Tuple, Union
+
from ...errors import Errors
+from ...symbols import NOUN, PROPN
+from ...tokens import Doc, Span
def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
diff --git a/spacy/lang/ca/tokenizer_exceptions.py b/spacy/lang/ca/tokenizer_exceptions.py
index b261b3498..67165780e 100755
--- a/spacy/lang/ca/tokenizer_exceptions.py
+++ b/spacy/lang/ca/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/cs/__init__.py b/spacy/lang/cs/__init__.py
index 3e70e4078..9ea60afdf 100644
--- a/spacy/lang/cs/__init__.py
+++ b/spacy/lang/cs/__init__.py
@@ -1,6 +1,6 @@
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
-from ...language import Language, BaseDefaults
+from .stop_words import STOP_WORDS
class CzechDefaults(BaseDefaults):
diff --git a/spacy/lang/da/__init__.py b/spacy/lang/da/__init__.py
index e148a7b4f..372f372dd 100644
--- a/spacy/lang/da/__init__.py
+++ b/spacy/lang/da/__init__.py
@@ -1,9 +1,9 @@
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
+from ...language import BaseDefaults, Language
+from .lex_attrs import LEX_ATTRS
from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES
from .stop_words import STOP_WORDS
-from .lex_attrs import LEX_ATTRS
from .syntax_iterators import SYNTAX_ITERATORS
-from ...language import Language, BaseDefaults
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class DanishDefaults(BaseDefaults):
diff --git a/spacy/lang/da/lex_attrs.py b/spacy/lang/da/lex_attrs.py
index 403af686c..8e0420912 100644
--- a/spacy/lang/da/lex_attrs.py
+++ b/spacy/lang/da/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
# Source http://fjern-uv.dk/tal.php
_num_words = """nul
en et to tre fire fem seks syv otte ni ti
diff --git a/spacy/lang/da/punctuation.py b/spacy/lang/da/punctuation.py
index e050ab7aa..f70fe3d64 100644
--- a/spacy/lang/da/punctuation.py
+++ b/spacy/lang/da/punctuation.py
@@ -1,8 +1,13 @@
-from ..char_classes import LIST_ELLIPSES, LIST_ICONS
-from ..char_classes import CONCAT_QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+)
from ..punctuation import TOKENIZER_SUFFIXES
-
_quotes = CONCAT_QUOTES.replace("'", "")
_infixes = (
diff --git a/spacy/lang/da/syntax_iterators.py b/spacy/lang/da/syntax_iterators.py
index a0b70f004..60224f0b1 100644
--- a/spacy/lang/da/syntax_iterators.py
+++ b/spacy/lang/da/syntax_iterators.py
@@ -1,7 +1,8 @@
-from typing import Union, Iterator, Tuple
-from ...tokens import Doc, Span
-from ...symbols import NOUN, PROPN, PRON, VERB, AUX
+from typing import Iterator, Tuple, Union
+
from ...errors import Errors
+from ...symbols import AUX, NOUN, PRON, PROPN, VERB
+from ...tokens import Doc, Span
def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
diff --git a/spacy/lang/da/tokenizer_exceptions.py b/spacy/lang/da/tokenizer_exceptions.py
index ce25c546b..649d12022 100644
--- a/spacy/lang/da/tokenizer_exceptions.py
+++ b/spacy/lang/da/tokenizer_exceptions.py
@@ -2,10 +2,9 @@
Tokenizer Exceptions.
Source: https://forkortelse.dk/ and various others.
"""
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/de/__init__.py b/spacy/lang/de/__init__.py
index 65863c098..4f45b2357 100644
--- a/spacy/lang/de/__init__.py
+++ b/spacy/lang/de/__init__.py
@@ -1,8 +1,8 @@
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
+from ...language import BaseDefaults, Language
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
from .stop_words import STOP_WORDS
from .syntax_iterators import SYNTAX_ITERATORS
-from ...language import Language, BaseDefaults
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class GermanDefaults(BaseDefaults):
diff --git a/spacy/lang/de/punctuation.py b/spacy/lang/de/punctuation.py
index 69d402237..862207649 100644
--- a/spacy/lang/de/punctuation.py
+++ b/spacy/lang/de/punctuation.py
@@ -1,9 +1,18 @@
-from ..char_classes import LIST_ELLIPSES, LIST_ICONS, LIST_PUNCT, LIST_QUOTES
-from ..char_classes import CURRENCY, UNITS, PUNCT
-from ..char_classes import CONCAT_QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ CURRENCY,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+ LIST_PUNCT,
+ LIST_QUOTES,
+ PUNCT,
+ UNITS,
+)
from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES
-
_prefixes = ["``"] + BASE_TOKENIZER_PREFIXES
_suffixes = (
diff --git a/spacy/lang/de/syntax_iterators.py b/spacy/lang/de/syntax_iterators.py
index e80504998..544fe299c 100644
--- a/spacy/lang/de/syntax_iterators.py
+++ b/spacy/lang/de/syntax_iterators.py
@@ -1,7 +1,7 @@
-from typing import Union, Iterator, Tuple
+from typing import Iterator, Tuple, Union
-from ...symbols import NOUN, PROPN, PRON
from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
from ...tokens import Doc, Span
diff --git a/spacy/lang/de/tokenizer_exceptions.py b/spacy/lang/de/tokenizer_exceptions.py
index 21d99cffe..3f1aeeccd 100644
--- a/spacy/lang/de/tokenizer_exceptions.py
+++ b/spacy/lang/de/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {
"auf'm": [{ORTH: "auf"}, {ORTH: "'m", NORM: "dem"}],
diff --git a/spacy/lang/dsb/__init__.py b/spacy/lang/dsb/__init__.py
index c66092a0c..096eced19 100644
--- a/spacy/lang/dsb/__init__.py
+++ b/spacy/lang/dsb/__init__.py
@@ -1,6 +1,6 @@
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
class LowerSorbianDefaults(BaseDefaults):
diff --git a/spacy/lang/el/__init__.py b/spacy/lang/el/__init__.py
index 53dd9be8e..00e52bd97 100644
--- a/spacy/lang/el/__init__.py
+++ b/spacy/lang/el/__init__.py
@@ -1,13 +1,14 @@
-from typing import Optional, Callable
+from typing import Callable, Optional
+
from thinc.api import Model
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .stop_words import STOP_WORDS
-from .lex_attrs import LEX_ATTRS
-from .syntax_iterators import SYNTAX_ITERATORS
-from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
+from ...language import BaseDefaults, Language
from .lemmatizer import GreekLemmatizer
-from ...language import Language, BaseDefaults
+from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class GreekDefaults(BaseDefaults):
diff --git a/spacy/lang/el/get_pos_from_wiktionary.py b/spacy/lang/el/get_pos_from_wiktionary.py
index 369973cc0..10b54d112 100644
--- a/spacy/lang/el/get_pos_from_wiktionary.py
+++ b/spacy/lang/el/get_pos_from_wiktionary.py
@@ -1,5 +1,6 @@
def get_pos_from_wiktionary():
import re
+
from gensim.corpora.wikicorpus import extract_pages
regex = re.compile(r"==={{(\w+)\|el}}===")
diff --git a/spacy/lang/el/punctuation.py b/spacy/lang/el/punctuation.py
index 2d5690407..b8b717bac 100644
--- a/spacy/lang/el/punctuation.py
+++ b/spacy/lang/el/punctuation.py
@@ -1,6 +1,16 @@
-from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY
-from ..char_classes import LIST_ICONS, ALPHA_LOWER, ALPHA_UPPER, ALPHA, HYPHENS
-from ..char_classes import CONCAT_QUOTES, CURRENCY
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ CURRENCY,
+ HYPHENS,
+ LIST_CURRENCY,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+ LIST_PUNCT,
+ LIST_QUOTES,
+)
_units = (
"km km² km³ m m² m³ dm dm² dm³ cm cm² cm³ mm mm² mm³ ha µm nm yd in ft "
diff --git a/spacy/lang/el/syntax_iterators.py b/spacy/lang/el/syntax_iterators.py
index 18fa46695..31c7dccf7 100644
--- a/spacy/lang/el/syntax_iterators.py
+++ b/spacy/lang/el/syntax_iterators.py
@@ -1,7 +1,7 @@
-from typing import Union, Iterator, Tuple
+from typing import Iterator, Tuple, Union
-from ...symbols import NOUN, PROPN, PRON
from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
from ...tokens import Doc, Span
diff --git a/spacy/lang/el/tokenizer_exceptions.py b/spacy/lang/el/tokenizer_exceptions.py
index 0a36d5d2b..41317ba97 100644
--- a/spacy/lang/el/tokenizer_exceptions.py
+++ b/spacy/lang/el/tokenizer_exceptions.py
@@ -1,6 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/en/__init__.py b/spacy/lang/en/__init__.py
index 876186979..c4bcfb938 100644
--- a/spacy/lang/en/__init__.py
+++ b/spacy/lang/en/__init__.py
@@ -1,13 +1,14 @@
-from typing import Optional, Callable
+from typing import Callable, Optional
+
from thinc.api import Model
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .stop_words import STOP_WORDS
-from .lex_attrs import LEX_ATTRS
-from .syntax_iterators import SYNTAX_ITERATORS
-from .punctuation import TOKENIZER_INFIXES
+from ...language import BaseDefaults, Language
from .lemmatizer import EnglishLemmatizer
-from ...language import Language, BaseDefaults
+from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class EnglishDefaults(BaseDefaults):
diff --git a/spacy/lang/en/punctuation.py b/spacy/lang/en/punctuation.py
index 5d3eb792e..775c6b001 100644
--- a/spacy/lang/en/punctuation.py
+++ b/spacy/lang/en/punctuation.py
@@ -1,5 +1,12 @@
-from ..char_classes import LIST_ELLIPSES, LIST_ICONS, HYPHENS
-from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ HYPHENS,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+)
_infixes = (
LIST_ELLIPSES
diff --git a/spacy/lang/en/syntax_iterators.py b/spacy/lang/en/syntax_iterators.py
index 7904e5621..140ae0a5c 100644
--- a/spacy/lang/en/syntax_iterators.py
+++ b/spacy/lang/en/syntax_iterators.py
@@ -1,7 +1,7 @@
-from typing import Union, Iterator, Tuple
+from typing import Iterator, Tuple, Union
-from ...symbols import NOUN, PROPN, PRON
from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
from ...tokens import Doc, Span
diff --git a/spacy/lang/en/tokenizer_exceptions.py b/spacy/lang/en/tokenizer_exceptions.py
index 7886e28cb..dd3650c18 100644
--- a/spacy/lang/en/tokenizer_exceptions.py
+++ b/spacy/lang/en/tokenizer_exceptions.py
@@ -1,8 +1,8 @@
from typing import Dict, List
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
-from ...util import update_exc
+from ...symbols import NORM, ORTH
+from ...util import update_exc
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc: Dict[str, List[Dict]] = {}
_exclude = [
diff --git a/spacy/lang/es/__init__.py b/spacy/lang/es/__init__.py
index e75955202..bcaed8672 100644
--- a/spacy/lang/es/__init__.py
+++ b/spacy/lang/es/__init__.py
@@ -1,12 +1,14 @@
-from typing import Optional, Callable
+from typing import Callable, Optional
+
from thinc.api import Model
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .stop_words import STOP_WORDS
-from .lex_attrs import LEX_ATTRS
+
+from ...language import BaseDefaults, Language
from .lemmatizer import SpanishLemmatizer
-from .syntax_iterators import SYNTAX_ITERATORS
+from .lex_attrs import LEX_ATTRS
from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES
-from ...language import Language, BaseDefaults
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class SpanishDefaults(BaseDefaults):
diff --git a/spacy/lang/es/lemmatizer.py b/spacy/lang/es/lemmatizer.py
index ca5fc08c8..44f968347 100644
--- a/spacy/lang/es/lemmatizer.py
+++ b/spacy/lang/es/lemmatizer.py
@@ -1,5 +1,5 @@
-from typing import List, Optional, Tuple
import re
+from typing import List, Optional, Tuple
from ...pipeline import Lemmatizer
from ...tokens import Token
diff --git a/spacy/lang/es/lex_attrs.py b/spacy/lang/es/lex_attrs.py
index 9d1fa93b8..4c477eaee 100644
--- a/spacy/lang/es/lex_attrs.py
+++ b/spacy/lang/es/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
_num_words = [
"cero",
"uno",
diff --git a/spacy/lang/es/punctuation.py b/spacy/lang/es/punctuation.py
index e9552371e..3d20518cd 100644
--- a/spacy/lang/es/punctuation.py
+++ b/spacy/lang/es/punctuation.py
@@ -1,8 +1,17 @@
-from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES
-from ..char_classes import LIST_ICONS, CURRENCY, LIST_UNITS, PUNCT
-from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA
-from ..char_classes import merge_chars
-
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ CURRENCY,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+ LIST_PUNCT,
+ LIST_QUOTES,
+ LIST_UNITS,
+ PUNCT,
+ merge_chars,
+)
_list_units = [u for u in LIST_UNITS if u != "%"]
_units = merge_chars(" ".join(_list_units))
diff --git a/spacy/lang/es/syntax_iterators.py b/spacy/lang/es/syntax_iterators.py
index f2ca2a678..96df444a3 100644
--- a/spacy/lang/es/syntax_iterators.py
+++ b/spacy/lang/es/syntax_iterators.py
@@ -1,7 +1,7 @@
-from typing import Union, Iterator, Tuple
+from typing import Iterator, Tuple, Union
-from ...symbols import NOUN, PROPN, PRON
from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
from ...tokens import Doc, Span
diff --git a/spacy/lang/es/tokenizer_exceptions.py b/spacy/lang/es/tokenizer_exceptions.py
index 74cdc143d..2ea0ed8b7 100644
--- a/spacy/lang/es/tokenizer_exceptions.py
+++ b/spacy/lang/es/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {
"pal": [{ORTH: "pa"}, {ORTH: "l", NORM: "el"}],
diff --git a/spacy/lang/et/__init__.py b/spacy/lang/et/__init__.py
index 274bc1309..9ec7e6006 100644
--- a/spacy/lang/et/__init__.py
+++ b/spacy/lang/et/__init__.py
@@ -1,5 +1,5 @@
+from ...language import BaseDefaults, Language
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
class EstonianDefaults(BaseDefaults):
diff --git a/spacy/lang/eu/__init__.py b/spacy/lang/eu/__init__.py
index 3346468bd..81f9c4a18 100644
--- a/spacy/lang/eu/__init__.py
+++ b/spacy/lang/eu/__init__.py
@@ -1,7 +1,7 @@
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
from .punctuation import TOKENIZER_SUFFIXES
-from ...language import Language, BaseDefaults
+from .stop_words import STOP_WORDS
class BasqueDefaults(BaseDefaults):
diff --git a/spacy/lang/eu/punctuation.py b/spacy/lang/eu/punctuation.py
index 5d35d0a25..382bfc75c 100644
--- a/spacy/lang/eu/punctuation.py
+++ b/spacy/lang/eu/punctuation.py
@@ -1,4 +1,3 @@
from ..punctuation import TOKENIZER_SUFFIXES
-
_suffixes = TOKENIZER_SUFFIXES
diff --git a/spacy/lang/fa/__init__.py b/spacy/lang/fa/__init__.py
index 914e4c27d..e5baa8b4a 100644
--- a/spacy/lang/fa/__init__.py
+++ b/spacy/lang/fa/__init__.py
@@ -1,12 +1,14 @@
-from typing import Optional, Callable
+from typing import Callable, Optional
+
from thinc.api import Model
-from .stop_words import STOP_WORDS
-from .lex_attrs import LEX_ATTRS
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .punctuation import TOKENIZER_SUFFIXES
-from .syntax_iterators import SYNTAX_ITERATORS
-from ...language import Language, BaseDefaults
+
+from ...language import BaseDefaults, Language
from ...pipeline import Lemmatizer
+from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class PersianDefaults(BaseDefaults):
diff --git a/spacy/lang/fa/lex_attrs.py b/spacy/lang/fa/lex_attrs.py
index 99b8e2787..065e81bd6 100644
--- a/spacy/lang/fa/lex_attrs.py
+++ b/spacy/lang/fa/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
MIM = "م"
ZWNJ_O_MIM = "ام"
YE_NUN = "ین"
diff --git a/spacy/lang/fa/punctuation.py b/spacy/lang/fa/punctuation.py
index 4b258c13d..c1ee570ce 100644
--- a/spacy/lang/fa/punctuation.py
+++ b/spacy/lang/fa/punctuation.py
@@ -1,5 +1,11 @@
-from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, CURRENCY
-from ..char_classes import UNITS, ALPHA_UPPER
+from ..char_classes import (
+ ALPHA_UPPER,
+ CURRENCY,
+ LIST_ELLIPSES,
+ LIST_PUNCT,
+ LIST_QUOTES,
+ UNITS,
+)
_suffixes = (
LIST_PUNCT
diff --git a/spacy/lang/fa/syntax_iterators.py b/spacy/lang/fa/syntax_iterators.py
index 8207884b0..3052369a7 100644
--- a/spacy/lang/fa/syntax_iterators.py
+++ b/spacy/lang/fa/syntax_iterators.py
@@ -1,7 +1,8 @@
-from typing import Union, Iterator, Tuple
-from ...tokens import Doc, Span
-from ...symbols import NOUN, PROPN, PRON
+from typing import Iterator, Tuple, Union
+
from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
+from ...tokens import Doc, Span
def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
diff --git a/spacy/lang/fa/tokenizer_exceptions.py b/spacy/lang/fa/tokenizer_exceptions.py
index 30df798ab..3b31b7f67 100644
--- a/spacy/lang/fa/tokenizer_exceptions.py
+++ b/spacy/lang/fa/tokenizer_exceptions.py
@@ -1,5 +1,4 @@
-from ...symbols import ORTH, NORM
-
+from ...symbols import NORM, ORTH
TOKENIZER_EXCEPTIONS = {
".ق ": [{ORTH: ".ق "}],
diff --git a/spacy/lang/fi/__init__.py b/spacy/lang/fi/__init__.py
index c3a0cf451..3e371b9b5 100644
--- a/spacy/lang/fi/__init__.py
+++ b/spacy/lang/fi/__init__.py
@@ -1,9 +1,9 @@
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
from .syntax_iterators import SYNTAX_ITERATORS
-from ...language import Language, BaseDefaults
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class FinnishDefaults(BaseDefaults):
diff --git a/spacy/lang/fi/lex_attrs.py b/spacy/lang/fi/lex_attrs.py
index 4d500cead..9eec41b3d 100644
--- a/spacy/lang/fi/lex_attrs.py
+++ b/spacy/lang/fi/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
_num_words = [
"nolla",
"yksi",
diff --git a/spacy/lang/fi/punctuation.py b/spacy/lang/fi/punctuation.py
index 6e14dde38..29ddc3111 100644
--- a/spacy/lang/fi/punctuation.py
+++ b/spacy/lang/fi/punctuation.py
@@ -1,8 +1,14 @@
-from ..char_classes import LIST_ELLIPSES, LIST_ICONS, LIST_HYPHENS
-from ..char_classes import CONCAT_QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ LIST_ELLIPSES,
+ LIST_HYPHENS,
+ LIST_ICONS,
+)
from ..punctuation import TOKENIZER_SUFFIXES
-
_quotes = CONCAT_QUOTES.replace("'", "")
DASHES = "|".join(x for x in LIST_HYPHENS if x != "-")
diff --git a/spacy/lang/fi/syntax_iterators.py b/spacy/lang/fi/syntax_iterators.py
index 6b481e51f..6e2216713 100644
--- a/spacy/lang/fi/syntax_iterators.py
+++ b/spacy/lang/fi/syntax_iterators.py
@@ -1,7 +1,8 @@
from typing import Iterator, Tuple, Union
-from ...tokens import Doc, Span
-from ...symbols import NOUN, PROPN, PRON
+
from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
+from ...tokens import Doc, Span
def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
diff --git a/spacy/lang/fi/tokenizer_exceptions.py b/spacy/lang/fi/tokenizer_exceptions.py
index 465333b0a..881d5b91d 100644
--- a/spacy/lang/fi/tokenizer_exceptions.py
+++ b/spacy/lang/fi/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/fr/__init__.py b/spacy/lang/fr/__init__.py
index 27d2a915e..a8bc7f53e 100644
--- a/spacy/lang/fr/__init__.py
+++ b/spacy/lang/fr/__init__.py
@@ -1,15 +1,14 @@
-from typing import Optional, Callable
+from typing import Callable, Optional
from thinc.api import Model
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS, TOKEN_MATCH
-from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES
-from .punctuation import TOKENIZER_SUFFIXES
-from .stop_words import STOP_WORDS
-from .lex_attrs import LEX_ATTRS
-from .syntax_iterators import SYNTAX_ITERATORS
+from ...language import BaseDefaults, Language
from .lemmatizer import FrenchLemmatizer
-from ...language import Language, BaseDefaults
+from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tokenizer_exceptions import TOKEN_MATCH, TOKENIZER_EXCEPTIONS
class FrenchDefaults(BaseDefaults):
diff --git a/spacy/lang/fr/lex_attrs.py b/spacy/lang/fr/lex_attrs.py
index 811312ad7..9cf508a07 100644
--- a/spacy/lang/fr/lex_attrs.py
+++ b/spacy/lang/fr/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
_num_words = set(
"""
zero un une deux trois quatre cinq six sept huit neuf dix
diff --git a/spacy/lang/fr/punctuation.py b/spacy/lang/fr/punctuation.py
index 873d01d87..a3b178a2f 100644
--- a/spacy/lang/fr/punctuation.py
+++ b/spacy/lang/fr/punctuation.py
@@ -1,8 +1,16 @@
-from ..punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES
-from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, CURRENCY
-from ..char_classes import CONCAT_QUOTES, UNITS, ALPHA, ALPHA_LOWER, ALPHA_UPPER
-from ..char_classes import merge_chars
-
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ CURRENCY,
+ LIST_ELLIPSES,
+ LIST_PUNCT,
+ LIST_QUOTES,
+ UNITS,
+ merge_chars,
+)
+from ..punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES
ELISION = "' ’".replace(" ", "")
HYPHENS = r"- – — ‐ ‑".replace(" ", "")
diff --git a/spacy/lang/fr/syntax_iterators.py b/spacy/lang/fr/syntax_iterators.py
index 5849c40b3..a6bf3d3ca 100644
--- a/spacy/lang/fr/syntax_iterators.py
+++ b/spacy/lang/fr/syntax_iterators.py
@@ -1,7 +1,7 @@
-from typing import Union, Iterator, Tuple
+from typing import Iterator, Tuple, Union
-from ...symbols import NOUN, PROPN, PRON
from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
from ...tokens import Doc, Span
diff --git a/spacy/lang/fr/tokenizer_exceptions.py b/spacy/lang/fr/tokenizer_exceptions.py
index 2e88b58cf..fa2062ef9 100644
--- a/spacy/lang/fr/tokenizer_exceptions.py
+++ b/spacy/lang/fr/tokenizer_exceptions.py
@@ -1,11 +1,10 @@
import re
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from .punctuation import ELISION, HYPHENS
-from ..char_classes import ALPHA_LOWER, ALPHA
from ...symbols import ORTH
from ...util import update_exc
-
+from ..char_classes import ALPHA, ALPHA_LOWER
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
+from .punctuation import ELISION, HYPHENS
# not using the large _tokenizer_exceptions_list by default as it slows down the tokenizer
# from ._tokenizer_exceptions_list import FR_BASE_EXCEPTIONS
diff --git a/spacy/lang/ga/__init__.py b/spacy/lang/ga/__init__.py
index 3be53bc7a..6f9a27a14 100644
--- a/spacy/lang/ga/__init__.py
+++ b/spacy/lang/ga/__init__.py
@@ -2,10 +2,10 @@ from typing import Optional
from thinc.api import Model
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
+from ...language import BaseDefaults, Language
from .lemmatizer import IrishLemmatizer
+from .stop_words import STOP_WORDS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class IrishDefaults(BaseDefaults):
diff --git a/spacy/lang/ga/lemmatizer.py b/spacy/lang/ga/lemmatizer.py
index 47aec8fd4..c9fbfbc19 100644
--- a/spacy/lang/ga/lemmatizer.py
+++ b/spacy/lang/ga/lemmatizer.py
@@ -1,4 +1,4 @@
-from typing import List, Dict, Tuple
+from typing import Dict, List, Tuple
from ...pipeline import Lemmatizer
from ...tokens import Token
diff --git a/spacy/lang/ga/tokenizer_exceptions.py b/spacy/lang/ga/tokenizer_exceptions.py
index 63af65fe9..eb4b413fb 100644
--- a/spacy/lang/ga/tokenizer_exceptions.py
+++ b/spacy/lang/ga/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {
"'acha'n": [{ORTH: "'ach", NORM: "gach"}, {ORTH: "a'n", NORM: "aon"}],
diff --git a/spacy/lang/grc/__init__.py b/spacy/lang/grc/__init__.py
index 019b3802e..ed742f4c5 100644
--- a/spacy/lang/grc/__init__.py
+++ b/spacy/lang/grc/__init__.py
@@ -1,8 +1,8 @@
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
-from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
-from ...language import Language, BaseDefaults
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class AncientGreekDefaults(BaseDefaults):
diff --git a/spacy/lang/grc/lex_attrs.py b/spacy/lang/grc/lex_attrs.py
index 0ab15e6fd..33cfca05b 100644
--- a/spacy/lang/grc/lex_attrs.py
+++ b/spacy/lang/grc/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
_num_words = [
# CARDINALS
"εἷς",
diff --git a/spacy/lang/grc/punctuation.py b/spacy/lang/grc/punctuation.py
index 8f3589e9a..8e9fc8bf2 100644
--- a/spacy/lang/grc/punctuation.py
+++ b/spacy/lang/grc/punctuation.py
@@ -1,6 +1,15 @@
-from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY
-from ..char_classes import LIST_ICONS, ALPHA_LOWER, ALPHA_UPPER, ALPHA, HYPHENS
-from ..char_classes import CONCAT_QUOTES
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ HYPHENS,
+ LIST_CURRENCY,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+ LIST_PUNCT,
+ LIST_QUOTES,
+)
_prefixes = (
[
diff --git a/spacy/lang/grc/tokenizer_exceptions.py b/spacy/lang/grc/tokenizer_exceptions.py
index bcee70f32..86527ff61 100644
--- a/spacy/lang/grc/tokenizer_exceptions.py
+++ b/spacy/lang/grc/tokenizer_exceptions.py
@@ -1,6 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/gu/__init__.py b/spacy/lang/gu/__init__.py
index e6fbc9d18..2f22034c1 100644
--- a/spacy/lang/gu/__init__.py
+++ b/spacy/lang/gu/__init__.py
@@ -1,5 +1,5 @@
+from ...language import BaseDefaults, Language
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
class GujaratiDefaults(BaseDefaults):
diff --git a/spacy/lang/he/__init__.py b/spacy/lang/he/__init__.py
index dd2ee478d..07084acf1 100644
--- a/spacy/lang/he/__init__.py
+++ b/spacy/lang/he/__init__.py
@@ -1,6 +1,6 @@
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
-from ...language import Language, BaseDefaults
+from .stop_words import STOP_WORDS
class HebrewDefaults(BaseDefaults):
diff --git a/spacy/lang/hi/__init__.py b/spacy/lang/hi/__init__.py
index 4c8ae446d..980dc31c1 100644
--- a/spacy/lang/hi/__init__.py
+++ b/spacy/lang/hi/__init__.py
@@ -1,6 +1,6 @@
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
-from ...language import Language, BaseDefaults
+from .stop_words import STOP_WORDS
class HindiDefaults(BaseDefaults):
diff --git a/spacy/lang/hi/lex_attrs.py b/spacy/lang/hi/lex_attrs.py
index ee845e8b1..4ecd1db66 100644
--- a/spacy/lang/hi/lex_attrs.py
+++ b/spacy/lang/hi/lex_attrs.py
@@ -1,6 +1,5 @@
+from ...attrs import LIKE_NUM, NORM
from ..norm_exceptions import BASE_NORMS
-from ...attrs import NORM, LIKE_NUM
-
# fmt: off
_stem_suffixes = [
diff --git a/spacy/lang/hr/__init__.py b/spacy/lang/hr/__init__.py
index 30870b522..fd7622a3d 100644
--- a/spacy/lang/hr/__init__.py
+++ b/spacy/lang/hr/__init__.py
@@ -1,5 +1,5 @@
+from ...language import BaseDefaults, Language
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
class CroatianDefaults(BaseDefaults):
diff --git a/spacy/lang/hsb/__init__.py b/spacy/lang/hsb/__init__.py
index 034d82319..e8b2ffc9f 100644
--- a/spacy/lang/hsb/__init__.py
+++ b/spacy/lang/hsb/__init__.py
@@ -1,7 +1,7 @@
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
from .stop_words import STOP_WORDS
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from ...language import Language, BaseDefaults
class UpperSorbianDefaults(BaseDefaults):
diff --git a/spacy/lang/hsb/tokenizer_exceptions.py b/spacy/lang/hsb/tokenizer_exceptions.py
index 4b9a4f98a..cd3bac913 100644
--- a/spacy/lang/hsb/tokenizer_exceptions.py
+++ b/spacy/lang/hsb/tokenizer_exceptions.py
@@ -1,6 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = dict()
for exc_data in [
diff --git a/spacy/lang/hu/__init__.py b/spacy/lang/hu/__init__.py
index 9426bacea..799e6d230 100644
--- a/spacy/lang/hu/__init__.py
+++ b/spacy/lang/hu/__init__.py
@@ -1,7 +1,7 @@
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS, TOKEN_MATCH
-from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
+from ...language import BaseDefaults, Language
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
+from .tokenizer_exceptions import TOKEN_MATCH, TOKENIZER_EXCEPTIONS
class HungarianDefaults(BaseDefaults):
diff --git a/spacy/lang/hu/punctuation.py b/spacy/lang/hu/punctuation.py
index f827cd677..dbf93c622 100644
--- a/spacy/lang/hu/punctuation.py
+++ b/spacy/lang/hu/punctuation.py
@@ -1,6 +1,14 @@
-from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, CONCAT_QUOTES
-from ..char_classes import CONCAT_ICONS, UNITS, ALPHA, ALPHA_LOWER, ALPHA_UPPER
-
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_ICONS,
+ CONCAT_QUOTES,
+ LIST_ELLIPSES,
+ LIST_PUNCT,
+ LIST_QUOTES,
+ UNITS,
+)
# removing ° from the special icons to keep e.g. 99° as one token
_concat_icons = CONCAT_ICONS.replace("\u00B0", "")
diff --git a/spacy/lang/hu/tokenizer_exceptions.py b/spacy/lang/hu/tokenizer_exceptions.py
index ffaa74f50..3f79b02d2 100644
--- a/spacy/lang/hu/tokenizer_exceptions.py
+++ b/spacy/lang/hu/tokenizer_exceptions.py
@@ -1,10 +1,9 @@
import re
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ..punctuation import ALPHA_LOWER, CURRENCY
from ...symbols import ORTH
from ...util import update_exc
-
+from ..punctuation import ALPHA_LOWER, CURRENCY
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/hy/__init__.py b/spacy/lang/hy/__init__.py
index 481eaae0a..e00d4fd11 100644
--- a/spacy/lang/hy/__init__.py
+++ b/spacy/lang/hy/__init__.py
@@ -1,6 +1,6 @@
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
-from ...language import Language, BaseDefaults
+from .stop_words import STOP_WORDS
class ArmenianDefaults(BaseDefaults):
diff --git a/spacy/lang/hy/lex_attrs.py b/spacy/lang/hy/lex_attrs.py
index 9c9c0380c..4c96b8ab5 100644
--- a/spacy/lang/hy/lex_attrs.py
+++ b/spacy/lang/hy/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
_num_words = [
"զրո",
"մեկ",
diff --git a/spacy/lang/id/__init__.py b/spacy/lang/id/__init__.py
index 0d72cfa9d..93eb3214a 100644
--- a/spacy/lang/id/__init__.py
+++ b/spacy/lang/id/__init__.py
@@ -1,9 +1,9 @@
-from .stop_words import STOP_WORDS
-from .punctuation import TOKENIZER_SUFFIXES, TOKENIZER_PREFIXES, TOKENIZER_INFIXES
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
from .syntax_iterators import SYNTAX_ITERATORS
-from ...language import Language, BaseDefaults
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class IndonesianDefaults(BaseDefaults):
diff --git a/spacy/lang/id/lex_attrs.py b/spacy/lang/id/lex_attrs.py
index 3167f4659..5952c4d06 100644
--- a/spacy/lang/id/lex_attrs.py
+++ b/spacy/lang/id/lex_attrs.py
@@ -1,8 +1,7 @@
import unicodedata
-from .punctuation import LIST_CURRENCY
from ...attrs import IS_CURRENCY, LIKE_NUM
-
+from .punctuation import LIST_CURRENCY
_num_words = [
"nol",
diff --git a/spacy/lang/id/punctuation.py b/spacy/lang/id/punctuation.py
index f6c2387d8..8303b8eaa 100644
--- a/spacy/lang/id/punctuation.py
+++ b/spacy/lang/id/punctuation.py
@@ -1,6 +1,5 @@
-from ..punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
-from ..char_classes import ALPHA, merge_chars, split_chars, _currency, _units
-
+from ..char_classes import ALPHA, _currency, _units, merge_chars, split_chars
+from ..punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
_units = (
_units + "s bit Gbps Mbps mbps Kbps kbps ƒ ppi px "
diff --git a/spacy/lang/id/syntax_iterators.py b/spacy/lang/id/syntax_iterators.py
index fa984d411..027798687 100644
--- a/spacy/lang/id/syntax_iterators.py
+++ b/spacy/lang/id/syntax_iterators.py
@@ -1,7 +1,7 @@
-from typing import Union, Iterator, Tuple
+from typing import Iterator, Tuple, Union
-from ...symbols import NOUN, PROPN, PRON
from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
from ...tokens import Doc, Span
diff --git a/spacy/lang/id/tokenizer_exceptions.py b/spacy/lang/id/tokenizer_exceptions.py
index ff77ede9f..8dea4e97f 100644
--- a/spacy/lang/id/tokenizer_exceptions.py
+++ b/spacy/lang/id/tokenizer_exceptions.py
@@ -1,8 +1,7 @@
+from ...symbols import NORM, ORTH
+from ...util import update_exc
from ..tokenizer_exceptions import BASE_EXCEPTIONS
from ._tokenizer_exceptions_list import ID_BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
-from ...util import update_exc
-
# Daftar singkatan dan Akronim dari:
# https://id.wiktionary.org/wiki/Wiktionary:Daftar_singkatan_dan_akronim_bahasa_Indonesia#A
diff --git a/spacy/lang/is/__init__.py b/spacy/lang/is/__init__.py
index 318363beb..af1260045 100644
--- a/spacy/lang/is/__init__.py
+++ b/spacy/lang/is/__init__.py
@@ -1,5 +1,5 @@
+from ...language import BaseDefaults, Language
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
class IcelandicDefaults(BaseDefaults):
diff --git a/spacy/lang/it/__init__.py b/spacy/lang/it/__init__.py
index ecf322bd7..14458d811 100644
--- a/spacy/lang/it/__init__.py
+++ b/spacy/lang/it/__init__.py
@@ -1,12 +1,13 @@
-from typing import Optional, Callable
+from typing import Callable, Optional
+
from thinc.api import Model
-from .stop_words import STOP_WORDS
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_INFIXES
-from ...language import Language, BaseDefaults
+from ...language import BaseDefaults, Language
from .lemmatizer import ItalianLemmatizer
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES
+from .stop_words import STOP_WORDS
from .syntax_iterators import SYNTAX_ITERATORS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class ItalianDefaults(BaseDefaults):
diff --git a/spacy/lang/it/lemmatizer.py b/spacy/lang/it/lemmatizer.py
index e44e64e3a..bf869166d 100644
--- a/spacy/lang/it/lemmatizer.py
+++ b/spacy/lang/it/lemmatizer.py
@@ -1,4 +1,4 @@
-from typing import List, Dict, Tuple
+from typing import Dict, List, Tuple
from ...pipeline import Lemmatizer
from ...tokens import Token
diff --git a/spacy/lang/it/punctuation.py b/spacy/lang/it/punctuation.py
index f01ab4f0d..51318b22d 100644
--- a/spacy/lang/it/punctuation.py
+++ b/spacy/lang/it/punctuation.py
@@ -1,8 +1,13 @@
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ HYPHENS,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+)
from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES
-from ..char_classes import LIST_ELLIPSES, LIST_ICONS
-from ..char_classes import ALPHA, HYPHENS, CONCAT_QUOTES
-from ..char_classes import ALPHA_LOWER, ALPHA_UPPER
-
ELISION = "'’"
diff --git a/spacy/lang/it/syntax_iterators.py b/spacy/lang/it/syntax_iterators.py
index f63df3fad..924627648 100644
--- a/spacy/lang/it/syntax_iterators.py
+++ b/spacy/lang/it/syntax_iterators.py
@@ -1,7 +1,7 @@
-from typing import Union, Iterator, Tuple
+from typing import Iterator, Tuple, Union
-from ...symbols import NOUN, PROPN, PRON
from ...errors import Errors
+from ...symbols import NOUN, PRON, PROPN
from ...tokens import Doc, Span
diff --git a/spacy/lang/it/tokenizer_exceptions.py b/spacy/lang/it/tokenizer_exceptions.py
index 42883863b..2e7a5a1a3 100644
--- a/spacy/lang/it/tokenizer_exceptions.py
+++ b/spacy/lang/it/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
from ...symbols import ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {
"all'art.": [{ORTH: "all'"}, {ORTH: "art."}],
diff --git a/spacy/lang/ja/__init__.py b/spacy/lang/ja/__init__.py
index bf86305fb..0d5f97ac8 100644
--- a/spacy/lang/ja/__init__.py
+++ b/spacy/lang/ja/__init__.py
@@ -1,27 +1,27 @@
-from typing import Optional, Union, Dict, Any, Callable
-from pathlib import Path
-import srsly
-from collections import namedtuple
-from thinc.api import Model
import re
+from collections import namedtuple
+from pathlib import Path
+from typing import Any, Callable, Dict, Optional, Union
-from .stop_words import STOP_WORDS
-from .syntax_iterators import SYNTAX_ITERATORS
-from .tag_map import TAG_MAP
-from .tag_orth_map import TAG_ORTH_MAP
-from .tag_bigram_map import TAG_BIGRAM_MAP
+import srsly
+from thinc.api import Model
+
+from ... import util
from ...errors import Errors
-from ...language import Language, BaseDefaults
+from ...language import BaseDefaults, Language
from ...pipeline import Morphologizer
from ...pipeline.morphologizer import DEFAULT_MORPH_MODEL
from ...scorer import Scorer
from ...symbols import POS
from ...tokens import Doc, MorphAnalysis
from ...training import validate_examples
-from ...util import DummyTokenizer, registry, load_config_from_str
+from ...util import DummyTokenizer, load_config_from_str, registry
from ...vocab import Vocab
-from ... import util
-
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tag_bigram_map import TAG_BIGRAM_MAP
+from .tag_map import TAG_MAP
+from .tag_orth_map import TAG_ORTH_MAP
DEFAULT_CONFIG = """
[nlp]
diff --git a/spacy/lang/ja/syntax_iterators.py b/spacy/lang/ja/syntax_iterators.py
index 588a9ba03..34670083e 100644
--- a/spacy/lang/ja/syntax_iterators.py
+++ b/spacy/lang/ja/syntax_iterators.py
@@ -1,9 +1,8 @@
-from typing import Union, Iterator, Tuple, Set
+from typing import Iterator, Set, Tuple, Union
-from ...symbols import NOUN, PROPN, PRON, VERB
+from ...symbols import NOUN, PRON, PROPN, VERB
from ...tokens import Doc, Span
-
# TODO: this can probably be pruned a bit
# fmt: off
labels = ["nsubj", "nmod", "ddoclike", "nsubjpass", "pcomp", "pdoclike", "doclike", "obl", "dative", "appos", "attr", "ROOT"]
diff --git a/spacy/lang/ja/tag_map.py b/spacy/lang/ja/tag_map.py
index c6de3831a..5c14f41bf 100644
--- a/spacy/lang/ja/tag_map.py
+++ b/spacy/lang/ja/tag_map.py
@@ -1,6 +1,23 @@
-from ...symbols import POS, PUNCT, INTJ, ADJ, AUX, ADP, PART, SCONJ, NOUN
-from ...symbols import SYM, PRON, VERB, ADV, PROPN, NUM, DET, SPACE, CCONJ
-
+from ...symbols import (
+ ADJ,
+ ADP,
+ ADV,
+ AUX,
+ CCONJ,
+ DET,
+ INTJ,
+ NOUN,
+ NUM,
+ PART,
+ POS,
+ PRON,
+ PROPN,
+ PUNCT,
+ SCONJ,
+ SPACE,
+ SYM,
+ VERB,
+)
TAG_MAP = {
# Explanation of Unidic tags:
diff --git a/spacy/lang/kn/__init__.py b/spacy/lang/kn/__init__.py
index ccd46a394..44d53f6b7 100644
--- a/spacy/lang/kn/__init__.py
+++ b/spacy/lang/kn/__init__.py
@@ -1,5 +1,5 @@
+from ...language import BaseDefaults, Language
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
class KannadaDefaults(BaseDefaults):
diff --git a/spacy/lang/ko/__init__.py b/spacy/lang/ko/__init__.py
index 0e02e4a2d..e2c860f7d 100644
--- a/spacy/lang/ko/__init__.py
+++ b/spacy/lang/ko/__init__.py
@@ -1,17 +1,16 @@
-from typing import Iterator, Any, Dict
+from typing import Any, Dict, Iterator
+from ...language import BaseDefaults, Language
+from ...scorer import Scorer
+from ...symbols import POS, X
+from ...tokens import Doc
+from ...training import validate_examples
+from ...util import DummyTokenizer, load_config_from_str, registry
+from ...vocab import Vocab
+from .lex_attrs import LEX_ATTRS
from .punctuation import TOKENIZER_INFIXES
from .stop_words import STOP_WORDS
from .tag_map import TAG_MAP
-from .lex_attrs import LEX_ATTRS
-from ...language import Language, BaseDefaults
-from ...tokens import Doc
-from ...scorer import Scorer
-from ...symbols import POS, X
-from ...training import validate_examples
-from ...util import DummyTokenizer, registry, load_config_from_str
-from ...vocab import Vocab
-
DEFAULT_CONFIG = """
[nlp]
diff --git a/spacy/lang/ko/lex_attrs.py b/spacy/lang/ko/lex_attrs.py
index ac5bc7e48..2c49aa389 100644
--- a/spacy/lang/ko/lex_attrs.py
+++ b/spacy/lang/ko/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
_num_words = [
"영",
"공",
diff --git a/spacy/lang/ko/punctuation.py b/spacy/lang/ko/punctuation.py
index f5f1c51da..c3c32ea1f 100644
--- a/spacy/lang/ko/punctuation.py
+++ b/spacy/lang/ko/punctuation.py
@@ -1,7 +1,6 @@
from ..char_classes import LIST_QUOTES
from ..punctuation import TOKENIZER_INFIXES as BASE_TOKENIZER_INFIXES
-
_infixes = (
["·", "ㆍ", r"\(", r"\)"]
+ [r"(?<=[0-9])~(?=[0-9-])"]
diff --git a/spacy/lang/ko/tag_map.py b/spacy/lang/ko/tag_map.py
index 26a8c56b9..85598c3ef 100644
--- a/spacy/lang/ko/tag_map.py
+++ b/spacy/lang/ko/tag_map.py
@@ -1,5 +1,21 @@
-from ...symbols import POS, PUNCT, INTJ, X, SYM, ADJ, AUX, ADP, CONJ, NOUN, PRON
-from ...symbols import VERB, ADV, PROPN, NUM, DET
+from ...symbols import (
+ ADJ,
+ ADP,
+ ADV,
+ AUX,
+ CONJ,
+ DET,
+ INTJ,
+ NOUN,
+ NUM,
+ POS,
+ PRON,
+ PROPN,
+ PUNCT,
+ SYM,
+ VERB,
+ X,
+)
# 은전한닢(mecab-ko-dic)의 품사 태그를 universal pos tag로 대응시킴
# https://docs.google.com/spreadsheets/d/1-9blXKjtjeKZqsf4NzHeYJCrr49-nXeRF6D80udfcwY/edit#gid=589544265
diff --git a/spacy/lang/ky/__init__.py b/spacy/lang/ky/__init__.py
index ccca384bd..fafc0f020 100644
--- a/spacy/lang/ky/__init__.py
+++ b/spacy/lang/ky/__init__.py
@@ -1,8 +1,8 @@
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
from .punctuation import TOKENIZER_INFIXES
from .stop_words import STOP_WORDS
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from ...language import Language, BaseDefaults
class KyrgyzDefaults(BaseDefaults):
diff --git a/spacy/lang/ky/punctuation.py b/spacy/lang/ky/punctuation.py
index fa9819f80..6d89da2f7 100644
--- a/spacy/lang/ky/punctuation.py
+++ b/spacy/lang/ky/punctuation.py
@@ -1,5 +1,12 @@
-from ..char_classes import ALPHA, ALPHA_LOWER, ALPHA_UPPER, CONCAT_QUOTES, HYPHENS
-from ..char_classes import LIST_ELLIPSES, LIST_ICONS
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ HYPHENS,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+)
_hyphens_no_dash = HYPHENS.replace("-", "").strip("|").replace("||", "")
_infixes = (
diff --git a/spacy/lang/ky/tokenizer_exceptions.py b/spacy/lang/ky/tokenizer_exceptions.py
index 8ec727ac1..c93e3dac3 100644
--- a/spacy/lang/ky/tokenizer_exceptions.py
+++ b/spacy/lang/ky/tokenizer_exceptions.py
@@ -1,6 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/la/__init__.py b/spacy/lang/la/__init__.py
index 15b87c5b9..d77ae267e 100644
--- a/spacy/lang/la/__init__.py
+++ b/spacy/lang/la/__init__.py
@@ -1,13 +1,15 @@
-from ...language import Language, BaseDefaults
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class LatinDefaults(BaseDefaults):
tokenizer_exceptions = TOKENIZER_EXCEPTIONS
stop_words = STOP_WORDS
lex_attr_getters = LEX_ATTRS
+ syntax_iterators = SYNTAX_ITERATORS
class Latin(Language):
diff --git a/spacy/lang/la/examples.py b/spacy/lang/la/examples.py
new file mode 100644
index 000000000..db8550070
--- /dev/null
+++ b/spacy/lang/la/examples.py
@@ -0,0 +1,22 @@
+"""
+Example sentences to test spaCy and its language models.
+
+>>> from spacy.lang.la.examples import sentences
+>>> docs = nlp.pipe(sentences)
+"""
+
+# > Caes. BG 1.1
+# > Cic. De Amic. 1
+# > V. Georg. 1.1-5
+# > Gen. 1:1
+# > Galileo, Sid. Nunc.
+# > van Schurman, Opusc. arg. 1
+
+sentences = [
+ "Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.",
+ "Q. Mucius augur multa narrare de C. Laelio socero suo memoriter et iucunde solebat nec dubitare illum in omni sermone appellare sapientem.",
+ "Quid faciat laetas segetes, quo sidere terram uertere, Maecenas, ulmisque adiungere uitis conueniat, quae cura boum, qui cultus habendo sit pecori, apibus quanta experientia parcis, hinc canere incipiam",
+ "In principio creavit Deus caelum et terram.",
+ "Quo sumpto, intelligatur lunaris globus, cuius maximus circulus CAF, centrum vero E, dimetiens CF, qui ad Terre diametrum est ut duo ad septem.",
+ "Cuicunque natura indita sunt principia, seu potentiae principiorum omnium artium, ac scientiarum, ei conveniunt omnes artes ac scientiae.",
+]
diff --git a/spacy/lang/la/lex_attrs.py b/spacy/lang/la/lex_attrs.py
index 9efb4dd3c..fcb35defc 100644
--- a/spacy/lang/la/lex_attrs.py
+++ b/spacy/lang/la/lex_attrs.py
@@ -1,22 +1,22 @@
-from ...attrs import LIKE_NUM
import re
+from ...attrs import LIKE_NUM
+
# cf. Goyvaerts/Levithan 2009; case-insensitive, allow 4
roman_numerals_compile = re.compile(
r"(?i)^(?=[MDCLXVI])M*(C[MD]|D?C{0,4})(X[CL]|L?X{0,4})(I[XV]|V?I{0,4})$"
)
-_num_words = set(
- """
-unus una unum duo duae tres tria quattuor quinque sex septem octo novem decem
+_num_words = """unus una unum duo duae tres tria quattuor quinque sex septem octo novem decem undecim duodecim tredecim quattuordecim quindecim sedecim septendecim duodeviginti undeviginti viginti triginta quadraginta quinquaginta sexaginta septuaginta octoginta nonaginta centum ducenti ducentae ducenta trecenti trecentae trecenta quadringenti quadringentae quadringenta quingenti quingentae quingenta sescenti sescentae sescenta septingenti septingentae septingenta octingenti octingentae octingenta nongenti nongentae nongenta mille
""".split()
-)
-_ordinal_words = set(
- """
-primus prima primum secundus secunda secundum tertius tertia tertium
-""".split()
-)
+_num_words += [item.replace("v", "u") for item in _num_words]
+_num_words = set(_num_words)
+
+_ordinal_words = """primus prima primum secundus secunda secundum tertius tertia tertium quartus quarta quartum quintus quinta quintum sextus sexta sextum septimus septima septimum octavus octava octavum nonus nona nonum decimus decima decimum undecimus undecima undecimum duodecimus duodecima duodecimum duodevicesimus duodevicesima duodevicesimum undevicesimus undevicesima undevicesimum vicesimus vicesima vicesimum tricesimus tricesima tricesimum quadragesimus quadragesima quadragesimum quinquagesimus quinquagesima quinquagesimum sexagesimus sexagesima sexagesimum septuagesimus septuagesima septuagesimum octogesimus octogesima octogesimum nonagesimus nonagesima nonagesimum centesimus centesima centesimum ducentesimus ducentesima ducentesimum trecentesimus trecentesima trecentesimum quadringentesimus quadringentesima quadringentesimum quingentesimus quingentesima quingentesimum sescentesimus sescentesima sescentesimum septingentesimus septingentesima septingentesimum octingentesimus octingentesima octingentesimum nongentesimus nongentesima nongentesimum millesimus millesima millesimum""".split()
+
+_ordinal_words += [item.replace("v", "u") for item in _ordinal_words]
+_ordinal_words = set(_ordinal_words)
def like_num(text):
diff --git a/spacy/lang/la/syntax_iterators.py b/spacy/lang/la/syntax_iterators.py
new file mode 100644
index 000000000..39b4fb39d
--- /dev/null
+++ b/spacy/lang/la/syntax_iterators.py
@@ -0,0 +1,86 @@
+from typing import Iterator, Tuple, Union
+
+from ...errors import Errors
+from ...symbols import AUX, NOUN, PRON, PROPN, VERB
+from ...tokens import Doc, Span
+
+# NB: Modified from da on suggestion from https://github.com/explosion/spaCy/issues/7457#issuecomment-800349751 [PJB]
+
+
+def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]:
+ def is_verb_token(tok):
+ return tok.pos in [VERB, AUX]
+
+ def get_left_bound(root):
+ left_bound = root
+ for tok in reversed(list(root.lefts)):
+ if tok.dep in np_left_deps:
+ left_bound = tok
+ return left_bound
+
+ def get_right_bound(doc, root):
+ right_bound = root
+ for tok in root.rights:
+ if tok.dep in np_right_deps:
+ right = get_right_bound(doc, tok)
+ if list(
+ filter(
+ lambda t: is_verb_token(t) or t.dep in stop_deps,
+ doc[root.i : right.i],
+ )
+ ):
+ break
+ else:
+ right_bound = right
+ return right_bound
+
+ def get_bounds(doc, root):
+ return get_left_bound(root), get_right_bound(doc, root)
+
+ doc = doclike.doc # Ensure works on both Doc and Span.
+
+ if not doc.has_annotation("DEP"):
+ raise ValueError(Errors.E029)
+
+ if not len(doc):
+ return
+
+ left_labels = [
+ "det",
+ "fixed",
+ "nmod:poss",
+ "amod",
+ "flat",
+ "goeswith",
+ "nummod",
+ "appos",
+ ]
+ right_labels = [
+ "fixed",
+ "nmod:poss",
+ "amod",
+ "flat",
+ "goeswith",
+ "nummod",
+ "appos",
+ "nmod",
+ "det",
+ ]
+ stop_labels = ["punct"]
+
+ np_label = doc.vocab.strings.add("NP")
+ np_left_deps = [doc.vocab.strings.add(label) for label in left_labels]
+ np_right_deps = [doc.vocab.strings.add(label) for label in right_labels]
+ stop_deps = [doc.vocab.strings.add(label) for label in stop_labels]
+
+ prev_right = -1
+ for token in doclike:
+ if token.pos in [PROPN, NOUN, PRON]:
+ left, right = get_bounds(doc, token)
+ if left.i <= prev_right:
+ continue
+ yield left.i, right.i + 1, np_label
+ prev_right = right.i
+
+
+SYNTAX_ITERATORS = {"noun_chunks": noun_chunks}
diff --git a/spacy/lang/la/tokenizer_exceptions.py b/spacy/lang/la/tokenizer_exceptions.py
index 060f6e085..c0b98116f 100644
--- a/spacy/lang/la/tokenizer_exceptions.py
+++ b/spacy/lang/la/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
from ...symbols import ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
## TODO: Look into systematically handling u/v
_exc = {
@@ -12,65 +11,15 @@ _exc = {
"uobiscum": [{ORTH: "uobis"}, {ORTH: "cum"}],
}
-for orth in [
- "A.",
- "Agr.",
- "Ap.",
- "C.",
- "Cn.",
- "D.",
- "F.",
- "K.",
- "L.",
- "M'.",
- "M.",
- "Mam.",
- "N.",
- "Oct.",
- "Opet.",
- "P.",
- "Paul.",
- "Post.",
- "Pro.",
- "Q.",
- "S.",
- "Ser.",
- "Sert.",
- "Sex.",
- "St.",
- "Sta.",
- "T.",
- "Ti.",
- "V.",
- "Vol.",
- "Vop.",
- "U.",
- "Uol.",
- "Uop.",
- "Ian.",
- "Febr.",
- "Mart.",
- "Apr.",
- "Mai.",
- "Iun.",
- "Iul.",
- "Aug.",
- "Sept.",
- "Oct.",
- "Nov.",
- "Nou.",
- "Dec.",
- "Non.",
- "Id.",
- "A.D.",
- "Coll.",
- "Cos.",
- "Ord.",
- "Pl.",
- "S.C.",
- "Suff.",
- "Trib.",
-]:
+_abbrev_exc = """A. A.D. Aa. Aaa. Acc. Agr. Ap. Apr. April. A.U.C. Aug. C. Caes. Caess. Cc. Cn. Coll. Cons. Conss. Cos. Coss. D. D.N. Dat. Dd. Dec. Decemb. Decembr. F. Feb. Febr. Februar. Ian. Id. Imp. Impp. Imppp. Iul. Iun. K. Kal. L. M'. M. Mai. Mam. Mar. Mart. Med. N. Nn. Nob. Non. Nov. Novemb. Oct. Octob. Opet. Ord. P. Paul. Pf. Pl. Plur. Post. Pp. Prid. Pro. Procos. Q. Quint. S. S.C. Scr. Sept. Septemb. Ser. Sert. Sex. Sext. St. Sta. Suff. T. Ti. Trib. V. Vol. Vop. Vv.""".split()
+
+_abbrev_exc += [item.lower() for item in _abbrev_exc]
+_abbrev_exc += [item.upper() for item in _abbrev_exc]
+_abbrev_exc += [item.replace("v", "u").replace("V", "U") for item in _abbrev_exc]
+
+_abbrev_exc += ["d.N."]
+
+for orth in set(_abbrev_exc):
_exc[orth] = [{ORTH: orth}]
TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc)
diff --git a/spacy/lang/lb/__init__.py b/spacy/lang/lb/__init__.py
index 7827e7762..2386b4356 100644
--- a/spacy/lang/lb/__init__.py
+++ b/spacy/lang/lb/__init__.py
@@ -1,8 +1,8 @@
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .punctuation import TOKENIZER_INFIXES
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class LuxembourgishDefaults(BaseDefaults):
diff --git a/spacy/lang/lb/lex_attrs.py b/spacy/lang/lb/lex_attrs.py
index d2d50d9dc..119231374 100644
--- a/spacy/lang/lb/lex_attrs.py
+++ b/spacy/lang/lb/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
_num_words = set(
"""
null eent zwee dräi véier fënnef sechs ziwen aacht néng zéng eelef zwielef dräizéng
diff --git a/spacy/lang/lb/punctuation.py b/spacy/lang/lb/punctuation.py
index e382c56c5..8bdbf9713 100644
--- a/spacy/lang/lb/punctuation.py
+++ b/spacy/lang/lb/punctuation.py
@@ -1,4 +1,4 @@
-from ..char_classes import LIST_ELLIPSES, LIST_ICONS, ALPHA, ALPHA_LOWER, ALPHA_UPPER
+from ..char_classes import ALPHA, ALPHA_LOWER, ALPHA_UPPER, LIST_ELLIPSES, LIST_ICONS
ELISION = " ' ’ ".strip().replace(" ", "")
diff --git a/spacy/lang/lb/tokenizer_exceptions.py b/spacy/lang/lb/tokenizer_exceptions.py
index d00dc9610..844826e27 100644
--- a/spacy/lang/lb/tokenizer_exceptions.py
+++ b/spacy/lang/lb/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-from ...symbols import ORTH, NORM
+from ...symbols import NORM, ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
# TODO
# treat other apostrophes within words as part of the word: [op d'mannst], [fir d'éischt] (= exceptions)
diff --git a/spacy/lang/lex_attrs.py b/spacy/lang/lex_attrs.py
index 6ed981a06..3ac20420d 100644
--- a/spacy/lang/lex_attrs.py
+++ b/spacy/lang/lex_attrs.py
@@ -1,11 +1,10 @@
-from typing import Set
-import unicodedata
import re
+import unicodedata
+from typing import Set
from .. import attrs
from .tokenizer_exceptions import URL_MATCH
-
_like_email = re.compile(r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)").match
_tlds = set(
"com|org|edu|gov|net|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|"
diff --git a/spacy/lang/lg/__init__.py b/spacy/lang/lg/__init__.py
index 6f7153fce..a87685375 100644
--- a/spacy/lang/lg/__init__.py
+++ b/spacy/lang/lg/__init__.py
@@ -1,7 +1,7 @@
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
from .punctuation import TOKENIZER_INFIXES
-from ...language import Language, BaseDefaults
+from .stop_words import STOP_WORDS
class LugandaDefaults(BaseDefaults):
diff --git a/spacy/lang/lg/punctuation.py b/spacy/lang/lg/punctuation.py
index 5d3eb792e..775c6b001 100644
--- a/spacy/lang/lg/punctuation.py
+++ b/spacy/lang/lg/punctuation.py
@@ -1,5 +1,12 @@
-from ..char_classes import LIST_ELLIPSES, LIST_ICONS, HYPHENS
-from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ HYPHENS,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+)
_infixes = (
LIST_ELLIPSES
diff --git a/spacy/lang/lij/__init__.py b/spacy/lang/lij/__init__.py
index b7e11f77e..3b8e972c6 100644
--- a/spacy/lang/lij/__init__.py
+++ b/spacy/lang/lij/__init__.py
@@ -1,7 +1,7 @@
+from ...language import BaseDefaults, Language
+from .punctuation import TOKENIZER_INFIXES
from .stop_words import STOP_WORDS
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .punctuation import TOKENIZER_INFIXES
-from ...language import Language, BaseDefaults
class LigurianDefaults(BaseDefaults):
diff --git a/spacy/lang/lij/punctuation.py b/spacy/lang/lij/punctuation.py
index d50b75589..c5c150d0a 100644
--- a/spacy/lang/lij/punctuation.py
+++ b/spacy/lang/lij/punctuation.py
@@ -1,6 +1,5 @@
-from ..punctuation import TOKENIZER_INFIXES
from ..char_classes import ALPHA
-
+from ..punctuation import TOKENIZER_INFIXES
ELISION = " ' ’ ".strip().replace(" ", "").replace("\n", "")
diff --git a/spacy/lang/lij/tokenizer_exceptions.py b/spacy/lang/lij/tokenizer_exceptions.py
index 52eae2c89..cf5a1af66 100644
--- a/spacy/lang/lij/tokenizer_exceptions.py
+++ b/spacy/lang/lij/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
from ...symbols import ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/lt/__init__.py b/spacy/lang/lt/__init__.py
index 3ae000e5f..f3ea257b1 100644
--- a/spacy/lang/lt/__init__.py
+++ b/spacy/lang/lt/__init__.py
@@ -1,8 +1,8 @@
-from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES
-from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
-from ...language import Language, BaseDefaults
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
class LithuanianDefaults(BaseDefaults):
diff --git a/spacy/lang/lt/punctuation.py b/spacy/lang/lt/punctuation.py
index 22aee0941..deef24854 100644
--- a/spacy/lang/lt/punctuation.py
+++ b/spacy/lang/lt/punctuation.py
@@ -1,9 +1,14 @@
-from ..char_classes import LIST_ICONS, LIST_ELLIPSES
-from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA
-from ..char_classes import HYPHENS
+from ..char_classes import (
+ ALPHA,
+ ALPHA_LOWER,
+ ALPHA_UPPER,
+ CONCAT_QUOTES,
+ HYPHENS,
+ LIST_ELLIPSES,
+ LIST_ICONS,
+)
from ..punctuation import TOKENIZER_SUFFIXES
-
_infixes = (
LIST_ELLIPSES
+ LIST_ICONS
diff --git a/spacy/lang/lt/tokenizer_exceptions.py b/spacy/lang/lt/tokenizer_exceptions.py
index 118fb2190..d39b86dfc 100644
--- a/spacy/lang/lt/tokenizer_exceptions.py
+++ b/spacy/lang/lt/tokenizer_exceptions.py
@@ -1,7 +1,6 @@
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
from ...symbols import ORTH
from ...util import update_exc
-
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
_exc = {}
diff --git a/spacy/lang/lv/__init__.py b/spacy/lang/lv/__init__.py
index a05e5b939..fdfca5e97 100644
--- a/spacy/lang/lv/__init__.py
+++ b/spacy/lang/lv/__init__.py
@@ -1,5 +1,5 @@
+from ...language import BaseDefaults, Language
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
class LatvianDefaults(BaseDefaults):
diff --git a/spacy/lang/mk/__init__.py b/spacy/lang/mk/__init__.py
index fa07cfef9..413f0038d 100644
--- a/spacy/lang/mk/__init__.py
+++ b/spacy/lang/mk/__init__.py
@@ -1,15 +1,16 @@
-from typing import Optional, Callable
+from typing import Callable, Optional
+
from thinc.api import Model
+
+from ...attrs import LANG
+from ...language import BaseDefaults, Language
+from ...lookups import Lookups
+from ...util import update_exc
+from ..tokenizer_exceptions import BASE_EXCEPTIONS
from .lemmatizer import MacedonianLemmatizer
+from .lex_attrs import LEX_ATTRS
from .stop_words import STOP_WORDS
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
-from .lex_attrs import LEX_ATTRS
-from ..tokenizer_exceptions import BASE_EXCEPTIONS
-
-from ...language import Language, BaseDefaults
-from ...attrs import LANG
-from ...util import update_exc
-from ...lookups import Lookups
class MacedonianDefaults(BaseDefaults):
diff --git a/spacy/lang/mk/lemmatizer.py b/spacy/lang/mk/lemmatizer.py
index a792095e7..f5a5eca85 100644
--- a/spacy/lang/mk/lemmatizer.py
+++ b/spacy/lang/mk/lemmatizer.py
@@ -1,5 +1,5 @@
-from typing import List
from collections import OrderedDict
+from typing import List
from ...pipeline import Lemmatizer
from ...tokens import Token
diff --git a/spacy/lang/mk/tokenizer_exceptions.py b/spacy/lang/mk/tokenizer_exceptions.py
index 3b589b2a9..40f2c1d80 100644
--- a/spacy/lang/mk/tokenizer_exceptions.py
+++ b/spacy/lang/mk/tokenizer_exceptions.py
@@ -1,5 +1,4 @@
-from ...symbols import ORTH, NORM
-
+from ...symbols import NORM, ORTH
_exc = {}
diff --git a/spacy/lang/ml/__init__.py b/spacy/lang/ml/__init__.py
index 9f90605f0..0b17b8a7a 100644
--- a/spacy/lang/ml/__init__.py
+++ b/spacy/lang/ml/__init__.py
@@ -1,6 +1,6 @@
-from .stop_words import STOP_WORDS
+from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
-from ...language import Language, BaseDefaults
+from .stop_words import STOP_WORDS
class MalayalamDefaults(BaseDefaults):
diff --git a/spacy/lang/ml/lex_attrs.py b/spacy/lang/ml/lex_attrs.py
index 9ac19b6a7..33a144f6b 100644
--- a/spacy/lang/ml/lex_attrs.py
+++ b/spacy/lang/ml/lex_attrs.py
@@ -1,6 +1,5 @@
from ...attrs import LIKE_NUM
-
# reference 2: https://www.omniglot.com/language/numbers/malayalam.htm
_num_words = [
diff --git a/spacy/lang/mr/__init__.py b/spacy/lang/mr/__init__.py
index 3e172fa60..f980efbd0 100644
--- a/spacy/lang/mr/__init__.py
+++ b/spacy/lang/mr/__init__.py
@@ -1,5 +1,5 @@
+from ...language import BaseDefaults, Language
from .stop_words import STOP_WORDS
-from ...language import Language, BaseDefaults
class MarathiDefaults(BaseDefaults):
diff --git a/spacy/lang/ms/__init__.py b/spacy/lang/ms/__init__.py
new file mode 100644
index 000000000..f53ebfcf2
--- /dev/null
+++ b/spacy/lang/ms/__init__.py
@@ -0,0 +1,24 @@
+from ...language import BaseDefaults, Language
+from .lex_attrs import LEX_ATTRS
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
+from .stop_words import STOP_WORDS
+from .syntax_iterators import SYNTAX_ITERATORS
+from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
+
+
+class MalayDefaults(BaseDefaults):
+ tokenizer_exceptions = TOKENIZER_EXCEPTIONS
+ prefixes = TOKENIZER_PREFIXES
+ suffixes = TOKENIZER_SUFFIXES
+ infixes = TOKENIZER_INFIXES
+ syntax_iterators = SYNTAX_ITERATORS
+ lex_attr_getters = LEX_ATTRS
+ stop_words = STOP_WORDS
+
+
+class Malay(Language):
+ lang = "ms"
+ Defaults = MalayDefaults
+
+
+__all__ = ["Malay"]
diff --git a/spacy/lang/ms/_tokenizer_exceptions_list.py b/spacy/lang/ms/_tokenizer_exceptions_list.py
new file mode 100644
index 000000000..fba1dd70f
--- /dev/null
+++ b/spacy/lang/ms/_tokenizer_exceptions_list.py
@@ -0,0 +1,1943 @@
+# from https://prpm.dbp.gov.my/cari1?keyword=
+# dbp https://en.wikipedia.org/wiki/Dewan_Bahasa_dan_Pustaka
+MS_BASE_EXCEPTIONS = set(
+ """
+aba-aba
+abah-abah
+abar-abar
+abrit-abritan
+abu-abu
+abuk-abuk
+abun-abun
+acak-acak
+acak-acakan
+acang-acang
+aci-aci
+aci-acian
+aci-acinya
+adang-adang
+adap-adapan
+adik-beradik
+aduk-adukan
+agak-agak
+agar-agar
+agut-agut
+air-cooled
+ajar-ajar
+aji-aji
+akal-akal
+akhir-akhir
+aki-aki
+alah-mengalahi
+alan-alan
+alang-alang
+alang-alangan
+alap-alap
+ali-ali
+alih-alih
+aling-aling
+aling-alingan
+alip-alipan
+alon-alon
+alu-alu
+alu-aluan
+alun-alun
+alur-alur
+ambah-ambah
+ambai-ambai
+ambil-mengambil
+ambring-ambringan
+ambu-ambu
+ambung-ambung
+amin-amin
+ampai-ampai
+amung-amung
+anai-anai
+anak-anak
+anak-anakan
+anak-beranak
+ancak-ancak
+ancang-ancang
+andang-andang
+angan-angan
+anggar-anggar
+angin-angin
+angin-anginan
+angkul-angkul
+angkup-angkup
+angkut-angkut
+ani-ani
+aning-aning
+anjang-anjang
+anjing-anjing
+anjung-anjung
+anjung-anjungan
+antar-antar
+ante-mortem
+anting-anting
+antung-antung
+anyam-menganyam
+apa-apa
+api-api
+apit-apit
+aprit-apritan
+arah-arah
+arak-arakan
+aram-aram
+ari-ari
+aru-aru
+asa-asaan
+asam-asaman
+asuh-asuh
+atas-mengatasi
+ati-ati
+audio-visual
+avant-garde
+awang-awang
+awang-gemawang
+ayak-ayak
+ayam-ayam
+ayam-ayaman
+ayang-ayang
+ayeng-ayengan
+ayun-temayun
+back-up
+bahu-membahu
+baik-baik
+bajang-bajang
+baji-baji
+balai-balai
+balam-balam
+balas-membalas
+baling-baling
+balut-balut
+bangun-bangun
+bantal-bantal
+barat-barat
+barau-barau
+bari-bari
+barung-barung
+basa-basi
+bata-bata
+batir-batir
+bau-bauan
+bayang-bayang
+bedil-bedal
+begana-begini
+bekal-bekalan
+belat-belit
+belu-belai
+benggal-benggil
+bengkal-bengkil
+bengkang-bengkok
+bengkang-bengkong
+berabad-abad
+berabun-rabun
+berada-ada
+beragah-agah
+beragak-agak
+beragam-ragam
+beraja-raja
+berakit-rakit
+beraku-akuan
+beralun-alun
+beramah-ramahan
+beramah-tamah
+beramai-ramai
+berambai-ambai
+berambal-ambalan
+beramuk-amukan
+berandai-andai
+berandai-randai
+berang-berang
+berangan-angan
+beranggap-anggapan
+berangguk-angguk
+berangin-angin
+berangka-angka
+berangka-angkaan
+berangkai-rangkai
+beranja-anja
+berantai-rantai
+berapi-api
+berapung-apung
+berarak-arakan
+beras-beras
+berasing-asingan
+beratus-ratus
+berawas-awas
+berayal-ayalan
+berayun-ayun
+berbagai-bagai
+berbahas-bahasan
+berbalas-balasan
+berbalik-balik
+berbanjar-banjar
+berbantah-bantah
+berbanyak-banyak
+berbarik-barik
+berbasah-basah
+berbatu-batu
+berbayang-bayang
+berbecak-becak
+berbedil-bedilan
+berbeka-beka
+berbelakang-belakangan
+berbelang-belang
+berbeli-belian
+berbelit-belit
+berbelok-belok
+berbenar-benar
+berbencah-bencah
+berbesar-besar
+berbidai-bidai
+berbiku-biku
+berbilik-bilik
+berbinar-binar
+berbincang-bincang
+berbingkah-bingkah
+berbintang-bintang
+berbintik-bintik
+berbintil-bintil
+berbisik-bisik
+berbolak-balik
+berbolong-bolong
+berbondong-bondong
+berbongkah-bongkah
+berbuai-buai
+berbual-bual
+berbukit-bukit
+berbulan-bulan
+berbunga-bunga
+berbuntut-buntut
+berbunuh-bunuhan
+berburu-buru
+berburuk-buruk
+berbutir-butir
+bercabang-cabang
+bercaci-cacian
+bercakap-cakap
+bercakar-cakaran
+bercantik-cantik
+bercari-cari
+bercari-carian
+bercarik-carik
+bercepat-cepat
+bercerai-berai
+bercerai-cerai
+bercetai-cetai
+bercikun-cikun
+bercinta-cintaan
+bercita-cita
+berciut-ciut
+berconteng-conteng
+bercoreng-coreng
+bercoreng-moreng
+bercuit-cuit
+bercumbu-cumbu
+bercumbu-cumbuan
+bercura-bura
+bercura-cura
+berdada-dadaan
+berdahulu-dahuluan
+berdalam-dalam
+berdebar-debar
+berdecap-decap
+berdedai-dedai
+berdegap-degap
+berdegar-degar
+berdeham-deham
+berdekah-dekah
+berdekat-dekat
+berdelat-delat
+berdembun-dembun
+berdempang-dempang
+berdendam-dendaman
+berdengkang-dengkang
+berdentang-dentang
+berdentum-dentum
+berdentung-dentung
+berdepak-depak
+berdepan-depan
+berderai-derai
+berderak-derak
+berderau-derau
+berdering-dering
+berderung-derung
+berdesak-desakan
+berdesing-desing
+berdesus-desus
+berdikit-dikit
+berdingkit-dingkit
+berdua-dua
+berduri-duri
+berduru-duru
+berduyun-duyun
+berebut-rebut
+berebut-rebutan
+beregang-regang
+berek-berek
+berembut-rembut
+berempat-empat
+berenak-enak
+berenteng-renteng
+beresah-resah
+berfoya-foya
+bergagah-gagahan
+bergagap-gagap
+bergalur-galur
+berganda-ganda
+berganti-ganti
+bergarah-garah
+bergaruk-garuk
+bergegas-gegas
+bergelang-gelang
+bergelap-gelap
+bergelas-gelasan
+bergeleng-geleng
+bergemal-gemal
+bergembut-gembut
+bergerek-gerek
+bergesa-gesa
+bergilir-gilir
+bergolek-golek
+bergores-gores
+bergotong-royong
+bergugus-gugus
+bergulung-gulung
+bergulut-gulut
+bergumpal-gumpal
+bergunung-gunung
+berhadap-hadapan
+berhamun-hamun
+berhandai-handai
+berhanyut-hanyut
+berhari-hari
+berhati-hati
+berhilau-hilau
+berhujan-hujan
+beria-ia
+beria-ria
+beriak-riak
+beribu-ribu
+berigi-rigi
+bering-bering
+beringat-ingat
+beringgit-ringgit
+berintik-rintik
+beriring-iring
+beriring-iringan
+berjabir-jabir
+berjaga-jaga
+berjagung-jagung
+berjalan-jalan
+berjalar-jalar
+berjalin-jalin
+berjalur-jalur
+berjam-jam
+berjauh-jauhan
+berjejal-jejal
+berjela-jela
+berjenis-jenis
+berjenjang-jenjang
+berjilid-jilid
+berjinak-jinak
+berjingkat-jingkat
+berjingkrak-jingkrak
+berjongkok-jongkok
+berjubel-jubel
+berjujut-jujutan
+berjulai-julai
+berjumbai-jumbai
+berjurai-jurai
+berjurus-jurus
+berjuta-juta
+berkaca-kaca
+berkait-kaitan
+berkala-kala
+berkali-kali
+berkanjar-kanjar
+berkaok-kaok
+berkarung-karung
+berkasih-kasihan
+berkata-kata
+berkatak-katak
+berkecai-kecai
+berkecek-kecek
+berkecil-kecil
+berkecil-kecilan
+berkedip-kedip
+berkejang-kejang
+berkejap-kejap
+berkejar-kejaran
+berkelar-kelar
+berkelip-kelip
+berkelit-kelit
+berkelok-kelok
+berkelompok-kelompok
+berkelun-kelun
+berkembur-kembur
+berkempul-kempul
+berkena-kenaan
+berkenal-kenalan
+berkendur-kendur
+berkeok-keok
+berkepak-kepak
+berkepal-kepal
+berkeping-keping
+berkepul-kepul
+berkeras-kerasan
+berkeritik-keritik
+berkeruit-keruit
+berkerut-kerut
+berketak-ketak
+berketak-ketik
+berketi-keti
+berketil-ketil
+berketuk-ketak
+berketul-ketul
+berkial-kial
+berkian-kian
+berkias-kiasan
+berkibar-kibar
+berkilah-kilah
+berkilat-kilat
+berkilau-kilauan
+berkilo-kilo
+berkinja-kinja
+berkipas-kipas
+berkira-kira
+berkirim-kiriman
+berkobar-kobar
+berkobok-kobok
+berkocak-kocak
+berkodi-kodi
+berkolek-kolek
+berkopah-kopah
+berkotak-kotak
+berkuat-kuatan
+berkunang-kunang
+berkurun-kurun
+berkusau-kusau
+berkusu-kusu
+berkusut-kusut
+berkuting-kuting
+berkutu-kutuan
+berlabun-labun
+berlain-lainan
+berlalai-lalai
+berlama-lama
+berlambai-lambai
+berlambak-lambak
+berlampang-lampang
+berlapang-lapang
+berlapis-lapis
+berlapuk-lapuk
+berlarah-larah
+berlarat-larat
+berlari-larian
+berlarik-larik
+berlarut-larut
+berlawak-lawak
+berlayap-layapan
+berlebih-lebih
+berlebih-lebihan
+berlekas-lekas
+berlena-lena
+berlengah-lengah
+berlenggek-lenggek
+berlenggok-lenggok
+berleret-leret
+berliang-liuk
+berliku-liku
+berlimpah-limpah
+berlimpap-limpap
+berlimpit-limpit
+berlinang-linang
+berlindak-lindak
+berlipat-lipat
+berlompok-lompok
+berloncat-loncatan
+berlopak-lopak
+berlubang-lubang
+bermaaf-maafan
+bermacam-macam
+bermain-main
+bermalas-malas
+bermanik-manik
+bermanis-manis
+bermanja-manja
+bermasak-masak
+bermati-mati
+bermegah-megah
+bermemek-memek
+bermesra-mesraan
+bermewah-mewah
+berminggu-minggu
+berminta-minta
+bermuda-muda
+bermudah-mudah
+bermuka-muka
+bermula-mula
+bermulut-mulut
+bernafsi-nafsi
+bernaka-naka
+berniat-niat
+berogak-ogak
+beroleng-oleng
+berolok-olok
+beromong-omong
+beronggok-onggok
+berorang-orang
+beroyal-royal
+berpada-pada
+berpahit-pahit
+berpair-pair
+berpal-pal
+berpalu-palu
+berpalu-paluan
+berpalun-palun
+berpandai-pandai
+berpandang-pandangan
+berpangkat-pangkat
+berpanjang-panjang
+berpasang-pasang
+berpasang-pasangan
+berpayah-payah
+berpeluh-peluh
+berpeluk-pelukan
+berpenat-penat
+berpencar-pencar
+berpendar-pendar
+berpenggal-penggal
+berperai-perai
+berpesai-pesai
+berpesta-pesta
+berpesuk-pesuk
+berpetak-petak
+berpeti-peti
+berpihak-pihak
+berpijar-pijar
+berpikul-pikul
+berpilih-pilih
+berpilin-pilin
+berpindah-pindah
+berpintal-pintal
+berpirau-pirau
+berpisah-pisah
+berpolah-polah
+berpongah-pongah
+berpontang-panting
+berporah-porah
+berpotong-potong
+berpuak-puak
+berpual-pual
+berpugak-pugak
+berpuluh-puluh
+berpulun-pulun
+berpuntal-puntal
+berpura-pura
+berpusar-pusar
+berpusing-pusing
+berpusu-pusu
+berputar-putar
+bersaf-saf
+bersahut-sahutan
+bersakit-sakit
+bersalah-salahan
+bersalam-salaman
+bersalin-salin
+bersama-sama
+bersambut-sambutan
+bersampan-sampan
+bersantai-santai
+bersapa-sapaan
+bersarang-sarang
+bersedan-sedan
+bersedia-sedia
+bersedu-sedu
+bersekat-sekat
+berselang-selang
+berselang-seli
+bersembur-semburan
+bersempit-sempit
+bersenang-senang
+bersenang-senangkan
+bersenda-senda
+bersendi-sendi
+bersepah-sepah
+bersepi-sepi
+berserak-serak
+berseri-seri
+bersesak-sesak
+bersetai-setai
+bersia-sia
+bersiap-siap
+bersiar-siar
+bersilir-silir
+bersimbur-simburan
+bersinau-sinau
+bersorak-sorai
+bersuap-suapan
+bersudah-sudah
+bersuka-suka
+bersuka-sukaan
+bersuku-suku
+bersumpah-sumpahan
+bersungguh-sungguh
+bersungut-sungut
+bersunyi-sunyi
+bersusah-susah
+bersusuk-susuk
+bersusuk-susukan
+bersutan-sutan
+bertabur-tabur
+bertahu-tahu
+bertahun-tahun
+bertajuk-tajuk
+bertakik-takik
+bertala-tala
+bertali-tali
+bertalu-talu
+bertambah-tambah
+bertanda-tandaan
+bertangis-tangisan
+bertangkil-tangkil
+bertanya-tanya
+bertarik-tarikan
+bertatai-tatai
+bertatih-tatih
+bertawan-tawan
+bertawar-tawaran
+bertebu-tebu
+bertebu-tebukan
+berteguh-teguh
+berteguh-teguhan
+berteka-teki
+bertelau-telau
+bertele-tele
+bertempat-tempat
+bertempuh-tempuh
+bertenang-tenang
+bertenggang-tenggangan
+bertentu-tentu
+bertepek-tepek
+berterang-terang
+berterang-terangan
+bertikam-tikaman
+bertimbal-timbalan
+bertimbun-timbun
+bertimpa-timpa
+bertimpas-timpas
+bertingkah-tingkah
+bertingkat-tingkat
+bertinjau-tinjauan
+bertiras-tiras
+bertitar-titar
+bertoboh-toboh
+bertolak-tolak
+bertolak-tolakan
+bertolong-tolongan
+bertonjol-tonjol
+bertua-tua
+bertua-tuaan
+bertual-tual
+bertubi-tubi
+bertukar-tukar
+bertukar-tukaran
+bertukas-tukas
+bertumpak-tumpak
+bertunda-tunda
+bertunjuk-tunjukan
+bertura-tura
+berturut-turut
+bertutur-tutur
+beruas-ruas
+berubah-ubah
+berulang-alik
+berulang-ulang
+berumbai-rumbai
+berundung-undung
+berunggas-runggas
+berungkur-ungkuran
+beruntai-untai
+beruntun-runtun
+berunyai-unyai
+berupa-rupa
+berura-ura
+beruris-uris
+berurut-urutan
+berwarna-warna
+berwarna-warni
+berwindu-windu
+berwiru-wiru
+beryang-yang
+besar-besaran
+betak-betak
+beti-beti
+betul-betul
+biang-biang
+biar-biar
+biji-bijian
+bila-bila
+bilang-bilang
+bincang-bincut
+bini-binian
+biri-biri
+biru-biru
+bisik-bisik
+biti-biti
+bolak-balik
+bolang-baling
+bongkar-bangkir
+buah-buahan
+buat-buatan
+buaya-buaya
+bubun-bubun
+bugi-bugi
+built-in
+bukan-bukan
+bulan-bulan
+bulan-bulanan
+bulang-bulang
+bulat-bulat
+buli-buli
+bulu-bulu
+buluh-buluh
+bulus-bulus
+bunga-bungaan
+bunuh-membunuh
+bunyi-bunyian
+buru-buru
+burung-burungan
+bye-bye
+cabik-cabik
+caing-caing
+calar-balar
+cara-cara
+carut-marut
+cawi-cawi
+cebar-cebur
+celam-celum
+celangak-celinguk
+celas-celus
+celedang-celedok
+celengkak-celengkok
+cemas-cemas
+centang-perenang
+cepat-cepat
+cerai-berai
+ceruk-menceruk
+ceruk-meruk
+check-up
+chit-chat
+cirit-birit
+cita-cita
+close-up
+closed-circuit
+cobak-cabik
+cobar-cabir
+cola-cala
+compang-camping
+congak-cangit
+congkah-cangkih
+congkah-mangkih
+copak-capik
+corak-carik
+corat-coret
+coreng-moreng
+cuang-caing
+cubung-cubung
+culik-culik
+cuma-cuma
+cumi-cumi
+cungap-cangip
+cupu-cupu
+dahulu-mendahului
+dali-dali
+dapur-dapur
+dari-dari
+daru-daru
+datang-datang
+datang-mendatangi
+daun-daunan
+dawai-dawai
+dayang-dayang
+degap-degap
+dekak-dekak
+dekat-dekat
+dengar-dengaran
+desas-desus
+diam-diam
+do-it-yourself
+dokok-dokok
+dolak-dalik
+dorong-mendorong
+drive-in
+dua-dua
+dua-duanya
+duduk-duduk
+dulang-dulang
+ecek-ecek
+embuh-embuhan
+empek-empek
+empok-empok
+encal-encal
+endap-endap
+endut-endutan
+engah-engah
+enggan-enggan
+engkah-engkah
+entah-berentah
+erang-erot
+erong-erong
+fast-food
+fifty-fifty
+flip-flop
+follow-up
+foya-foya
+gaba-gaba
+gabai-gabai
+gada-gada
+gading-gading
+gado-gado
+gajah-gajahan
+gala-gala
+gali-galian
+galing-galing
+galu-galu
+gamit-gamitan
+gampang-gampangan
+ganal-ganal
+ganda-berganda
+gapah-gopoh
+gara-gara
+garah-garah
+gatal-gatal
+gawar-gawar
+gaya-gayanya
+gedebak-gedebuk
+gelang-gelang
+gelembung-gelembungan
+geli-geli
+geliang-geliut
+geliat-geliut
+gempul-gempul
+gendang-gendang
+genjang-genjot
+gerabak-gerubuk
+gerak-gerik
+gerbas-gerbus
+gerit-gerit
+geruh-gerah
+getak-getuk
+geti-geti
+gila-gila
+gila-gilaan
+gilang-gemilang
+gilap-gemilap
+gili-gili
+giling-giling
+ginang-ginang
+girik-girik
+giring-giring
+go-kart
+golak-galik
+gonta-ganti
+gotong-royong
+gual-gail
+gudu-gudu
+gula-gula
+gulang-gulang
+guna-guna
+guntang-guntang
+gunung-ganang
+gunung-gemunung
+gunung-gunungan
+habis-habis
+habis-habisan
+halai-balai
+half-time
+hampir-hampir
+harap-harapan
+harum-haruman
+hati-hati
+heavy-duty
+hebat-hebatan
+hidup-hidup
+hiru-biru
+hiruk-pikuk
+hubaya-hubaya
+hula-hula
+huru-hara
+ibar-ibar
+icak-icak
+igau-igauan
+ikut-ikut
+ikut-ikutan
+ilam-ilam
+imbang-imbangan
+inang-inang
+inca-binca
+incang-incut
+ingat-ingat
+ingat-ingatan
+ingau-ingauan
+inggang-inggung
+injak-injak
+iras-iras
+iring-iringan
+iseng-iseng
+jadi-jadian
+jala-jala
+jamah-jamahan
+jambu-jambu
+jangan-jangan
+jarang-jarang
+jari-jari
+jaring-jaring
+jarum-jarum
+jauh-jauh
+jawi-jawi
+jebat-jebatan
+jelur-jelir
+jendal-jendul
+jenggar-jenggur
+jentik-jentik
+jerah-jerih
+jolong-jolong
+jongkar-jangkir
+juak-juak
+juang-juang
+julung-julung
+jurai-jurai
+kabu-kabu
+kacang-kacang
+kacang-kacangan
+kacau-balau
+kadang-kadang
+kail-kail
+kait-kait
+kakek-kakek
+kalau-kalau
+kaleng-kalengan
+kalut-malut
+kambing-kambing
+kanak-kanak
+kapa-kapa
+kapan-kapan
+kapu-kapu
+karang-karangan
+karang-mengarang
+kareseh-peseh
+karut-marut
+katang-katang
+kawa-kawa
+kayu-kayuan
+keabu-abuan
+keasyik-asyikan
+kebarat-baratan
+kebasah-basahan
+kebat-kebit
+kebata-bataan
+kebelanda-belandaan
+kebiru-biruan
+kebudak-budakan
+kecil-kecilan
+kecil-mengecil
+kecuh-kecah
+kedek-kedek
+kegadis-gadisan
+kegelap-gelapan
+kegila-gilaan
+kegirang-girangan
+kehijau-hijauan
+kehitam-hitaman
+kejaga-jagaan
+kejingga-jinggaan
+kekabur-kaburan
+kekanak-kanakan
+kekoboi-koboian
+kekuning-kuningan
+kelak-kelik
+kelak-keluk
+kelaki-lakian
+kelang-kelok
+kelap-kelip
+kelek-kelek
+kelek-kelekan
+kelik-kelik
+kelip-kelip
+kelusuh-kelasah
+kelut-melut
+kemak-kemik
+kemalu-maluan
+kemanja-manjaan
+kemarah-marahan
+kemasam-masaman
+kemati-matian
+kemerah-merahan
+kempang-kempis
+kempas-kempis
+kemuda-mudaan
+kena-mengena
+kenal-mengenal
+kenang-kenangan
+kencang-kencung
+kendang-kendang
+kendang-kendangan
+kentung-kentung
+kenyat-kenyit
+kepandir-pandiran
+kepang-kepot
+keperak-perakan
+kepilu-piluan
+kepura-puraan
+keputih-putihan
+kerah-kerahan
+kerancak-rancakan
+kerang-kerangan
+kerang-keroh
+kerang-kerung
+kerap-kerap
+keras-mengerasi
+kercap-kercip
+kercap-kercup
+keriang-keriut
+kernyat-kernyut
+kerong-kerong
+keropas-kerapis
+kertak-kertuk
+keruntang-pungkang
+kesap-kesip
+kesenak-senakan
+kesewenang-wenangan
+kesia-siaan
+kesik-kesik
+kesipu-sipuan
+kesu-kesi
+kesuh-kesih
+kesuk-kesik
+ketergesa-gesaan
+keti-keti
+ketidur-tiduran
+ketiga-tiganya
+ketua-tuaan
+ketuan-tuanan
+keungu-unguan
+kia-kia
+kiak-kiak
+kial-kial
+kiang-kiut
+kibang-kibut
+kicang-kecoh
+kicang-kicu
+kida-kida
+kilau-mengilau
+kili-kili
+kira-kira
+kira-kiraan
+kisi-kisi
+kocah-kacih
+kodok-kodok
+kolang-kaling
+koleh-koleh
+kolong-kolong
+koma-koma
+komat-kamit
+kontal-kantil
+kontang-kanting
+kosak-kasik
+kotak-katik
+kotak-kotak
+kuat-kuat
+kucar-kacir
+kucing-kucing
+kucing-kucingan
+kuda-kuda
+kuda-kudaan
+kudap-kudap
+kulah-kulah
+kulak-kulak
+kulik-kulik
+kulum-kulum
+kumat-kamit
+kunang-kunang
+kupat-kapit
+kupu-kupu
+kura-kura
+kurang-kurang
+kusat-mesat
+kutat-kutet
+kuti-kuti
+labi-labi
+labu-labu
+lagi-lagi
+laguh-lagah
+laki-laki
+lalu-lalang
+lama-kelamaan
+lama-lama
+lamat-lamat
+lambat-lambat
+lancar-lancar
+langak-longok
+langit-langit
+lanja-lanjaan
+lapat-lapat
+large-scale
+lari-lari
+lauk-pauk
+lawah-lawah
+lawak-lawak
+lawi-lawi
+layang-layang
+layu-layuan
+lebih-lebih
+legak-legok
+lekak-lekuk
+lekap-lekup
+lekas-lekas
+lekuh-lekih
+lekup-lekap
+lenggak-lenggok
+lenggok-lenggok
+lengket-lengket
+lentam-lentum
+lentang-lentok
+lentang-lentung
+lepa-lepa
+lerang-lerang
+lereng-lereng
+letah-letai
+letup-letup
+liang-liuk
+lidah-lidah
+line-up
+liuk-liuk
+liung-liung
+lobi-lobi
+lock-up
+lopak-lapik
+lopak-lopak
+lumba-lumba
+lumi-lumi
+luntang-lantung
+lupa-lupa
+lupa-lupaan
+main-mainan
+makan-makanan
+make-up
+malai-malai
+malam-malam
+malar-malar
+mali-mali
+malu-malu
+mana-mana
+manik-manik
+manis-manisan
+mark-up
+masing-masing
+mata-mata
+mati-matian
+maya-maya
+megap-megap
+megrek-megrek
+melak-melak
+melambai-lambai
+melambai-lambaikan
+melambat-lambatkan
+melaun-laun
+melawak-lawak
+melayap-layap
+melayap-layapkan
+melebih-lebihi
+melebih-lebihkan
+melejang-lejangkan
+melengah-lengah
+melihat-lihat
+melimpah-limpah
+melincah-lincah
+meloncat-loncat
+melonco-lonco
+melonjak-lonjak
+memacak-macak
+memaki-maki
+memaksa-maksa
+memandai-mandai
+memanggil-manggil
+memanis-manis
+memanjut-manjut
+memasak-masak
+memata-matai
+mematah-matah
+mematut-matut
+memayah-mayahkan
+membagi-bagikan
+membalik-balik
+membangkit-bangkit
+membayang-bayangi
+membayang-bayangkan
+membelai-belai
+membenar-benar
+membenar-benari
+memberai-beraikan
+membesar-besarkan
+membolak-balikkan
+membuang-buang
+membuat-buat
+membunga-bungai
+memburu-buru
+memburu-burukan
+memburuk-burukkan
+memencak-mencak
+memencar-mencar
+memetak-metak
+memetang-metangkan
+memetir-metir
+memikir-mikirkan
+memilih-milih
+meminang-minang
+meminta-minta
+memisah-misahkan
+memontang-mantingkan
+memperamat-amat
+memperamat-amatkan
+memperbagai-bagaikan
+memperganda-gandakan
+memperganduh-ganduhkan
+mempermacam-macamkan
+memperolok-olokkan
+mempersama-samakan
+mempertubi-tubi
+mempertubi-tubikan
+memperturut-turutkan
+memuja-muja
+memukang-mukang
+memulun-mulun
+memundi-mundi
+memundi-mundikan
+memuyu-muyu
+menagak-nagak
+menakut-nakuti
+menanjur-nanjur
+menanti-nanti
+menari-nari
+mencabik-cabik
+mencabik-cabikkan
+mencaing-caing
+mencak-mencak
+mencakup-cakup
+mencapak-capak
+mencari-cari
+mencarik-carik
+mencarut-carut
+mencengis-cengis
+mencepak-cepak
+mencepuk-cepuk
+mencerai-beraikan
+mencetai-cetai
+menciap-ciap
+menciar-ciar
+mencita-citakan
+menciut-ciut
+mencoang-coang
+mencubit-cubit
+mencuri-curi
+mendecap-decap
+mendengking-dengking
+menderak-derakkan
+menderau-derau
+menderu-deru
+mendesas-desuskan
+mendesus-desus
+mendewa-dewakan
+mendudu-dudu
+menebu-nebu
+menegur-neguri
+mengabung-ngabung
+mengaci-acikan
+mengada-ada
+mengaduk-aduk
+mengagak-agak
+mengagak-agihkan
+mengagut-agut
+mengais-ngais
+mengali-ali
+mengalur-alur
+mengamang-amang
+mengamat-amati
+mengambai-ambaikan
+mengambang-ambang
+mengancak-ancak
+mengangan-angankan
+mengangguk-angguk
+mengangin-anginkan
+mengangkat-angkat
+mengap-mengap
+mengapa-apai
+mengapi-apikan
+mengarah-arahi
+mengata-ngatai
+mengaum-aumkan
+mengejan-ejan
+mengelai-ngelai
+mengelepik-ngelepik
+mengelus-elus
+mengembut-embut
+mengenap-enapkan
+mengenjak-enjak
+mengepak-ngepak
+mengepak-ngepakkan
+menggaba-gabai
+menggalur-galur
+menggamak-gamak
+menggapai-gapai
+menggapai-gapaikan
+menggelepar-gelepar
+menggelepar-geleparkan
+menggemak-gemak
+menggerecak-gerecak
+menggesa-gesakan
+menggili-gili
+menggorek-gorek
+menggosok-gosok
+mengguit-guit
+menghalai-balaikan
+menghinap-hinap
+mengiang-ngiang
+mengibas-ngibas
+mengidam-idamkan
+mengilah-ngilahkan
+mengilai-ilai
+mengilat-ngilatkan
+mengilik-ngilik
+mengimak-imak
+mengiming-iming
+menginjak-injak
+mengipas-ngipas
+mengira-ngira
+mengira-ngirakan
+mengiras-iras
+mengiras-irasi
+mengitar-ngitar
+mengitik-ngitik
+mengogok-ogok
+mengolak-alikkan
+mengoleng-oleng
+mengongkang-ongkang
+mengongkok-ongkok
+mengonyah-anyih
+mengotak-ngatikkan
+mengoyak-ngoyakkan
+mengoyak-oyak
+menguar-nguarkan
+menguar-uarkan
+menguber-uber
+mengubit-ubit
+mengubrak-abrik
+mengucar-ngacirkan
+mengucek-ngucek
+menguik-uik
+menguis-uis
+mengulit-ulit
+menguman-uman
+mengumbang-ambingkan
+mengumpak-umpak
+mengungkat-ungkat
+mengungkit-ungkit
+mengurik-urik
+mengutak-ngatikkan
+mengutik-ngutik
+menimang-nimang
+meningkat-ningkat
+meniru-niru
+meniup-niup
+menjadi-jadi
+menjengek-jengek
+menjengit-jengit
+menjilat-jilat
+mentah-mentah
+mentang-mentang
+menunda-nunda
+menusuk-nusuk
+menyama-nyama
+menyambar-nyambar
+menyanjung-nyanjung
+menyapu-nyapu
+menyarat-nyarat
+menyendi-nyendi
+menyeret-nyeret
+menyeru-nyerukan
+menyia-nyiakan
+menyungguh-nyungguhi
+meraba-raba
+merangkak-rangkak
+merasa-rasai
+meraung-raung
+meraung-raungkan
+merayau-rayau
+merayu-rayu
+mereka-reka
+merelap-relap
+meremah-remah
+meremeh-temehkan
+merempah-rempahi
+merengek-rengek
+merenik-renik
+merenta-renta
+merenyai-renyai
+merintang-rintang
+merintik-rintik
+merobek-robek
+meronta-ronta
+merungus-rungus
+merungut-rungut
+mewarna-warnikan
+meyakin-yakini
+miju-miju
+minta-minta
+moga-moga
+morat-marit
+muda-mudi
+mudah-mudahan
+muka-muka
+mula-mula
+muluk-muluk
+naga-naga
+nanti-nantian
+nasi-nasi
+nasib-nasiban
+nenek-nenek
+nyolong-nyolong
+ogah-ogahan
+ogak-ogak
+olak-alik
+olak-olak
+olang-aling
+olang-alingan
+oleh-oleh
+olok-olok
+olok-olokan
+olong-olong
+on-screen
+onde-onde
+one-to-one
+oneng-oneng
+ongkang-ongkang
+ongol-ongol
+onyah-anyih
+orak-arik
+orang-aring
+orang-orangan
+orok-orok
+orong-orong
+otak-otak
+otak-otakan
+padi-padian
+pagi-pagi
+palas-palas
+paling-paling
+palu-memalu
+panas-panas
+pandang-memandang
+panji-panji
+para-para
+paru-paru
+pasang-memasang
+pasu-pasu
+paya-paya
+pecah-pecah
+pelan-pelan
+pengundang-undang
+perang-perangan
+perintang-rintang
+perlahan-lahan
+perlip-perlipan
+pertama-tama
+perundang-undangan
+pesan-pesan
+piat-piut
+pick-up
+pijak-pijak
+pijar-pijar
+pijat-pijat
+pina-pina
+pisang-pisang
+play-off
+pohon-pohonan
+pokrol-pokrolan
+polang-paling
+poma-poma
+pontang-panting
+porak-parik
+porak-peranda
+potong-memotong
+puji-pujian
+pukang-pukang
+pukul-memukul
+pulang-pergi
+pulut-pulut
+pundi-pundi
+punggung-memunggung
+pura-pura
+pusar-pusar
+push-up
+pusing-pusing
+putus-putus
+rada-rada
+radio-frequency
+ragu-ragu
+rama-rama
+rambu-rambu
+rango-rango
+rasa-rasanya
+rata-rata
+real-time
+rebah-rebah
+rebah-rebahan
+redam-redam
+reka-reka
+reka-rekaan
+remah-remah
+remang-remang
+rembah-rembih
+remeh-temeh
+rempah-rempah
+repuh-repuh
+riang-riang
+ribu-ribu
+rigi-rigi
+robak-rabik
+robat-rabit
+role-play
+roll-on
+rombang-rambing
+ruak-ruak
+ruku-ruku
+rumah-rumah
+rumah-rumahan
+rumput-rumputan
+runding-merunding
+runggu-rangga
+runner-up
+rupa-rupa
+rupa-rupanya
+saban-saban
+sabung-menyabung
+saing-menyaing
+salah-salah
+sama-sama
+samar-samar
+sambar-menyambar
+sambung-bersambung
+sambung-menyambung
+sambut-menyambut
+sampai-sampai
+sandar-menyandar
+sangat-sangat
+sangkut-menyangkut
+sapa-menyapa
+sapu-sapu
+sarit-sarit
+satu-satu
+satu-satunya
+sayup-menyayup
+sayup-sayup
+sayur-mayur
+sayur-sayuran
+sci-fi
+seakal-akal
+seakan-akan
+sealak-alak
+sebaik-baiknya
+sebelah-menyebelah
+sebentar-sebentar
+seberang-menyeberang
+seboleh-bolehnya
+sedalam-dalamnya
+sedang-menyedang
+sedap-sedapan
+sedapat-dapatnya
+sedikit-dikitnya
+sedikit-sedikit
+sedikit-sedikitnya
+seelok-eloknya
+segala-galanya
+segan-menyegan
+segan-menyegani
+segan-segan
+sehari-hari
+sehari-harian
+sejadi-jadinya
+sekali-kali
+sekali-sekali
+sekira-kira
+sekonyong-konyong
+sekuasa-kuasanya
+sekurang-kurangnya
+sela-menyela
+sela-sela
+selama-lamanya
+selambat-lambatnya
+selang-seli
+selang-seling
+selar-belar
+selat-latnya
+selekas-lekasnya
+selepas-lepas
+self-esteem
+self-help
+sema-sema
+semah-semah
+semak-semak
+semalam-malaman
+semasa-masa
+semata-mata
+sembunyi-sembunyi
+sembunyi-sembunyian
+semena-mena
+semenda-menyemenda
+semengga-mengga
+sementang-mentang
+semu-semu
+semut-semutan
+sengal-sengal
+sengau-sengauan
+seolah-olah
+sepala-pala
+sepandai-pandai
+sepetang-petangan
+sepoi-sepoi
+sepuas-puasnya
+serang-menyerang
+seraya-menyeraya
+serba-serbi
+serbah-serbih
+serembah-serembih
+sering-sering
+serta-menyertai
+serta-serta
+sesal-menyesali
+sesudah-sudah
+sesudah-sudahnya
+sesuka-suka
+setempat-setempat
+setengah-setengah
+setidak-tidaknya
+seupaya-upaya
+seupaya-upayanya
+sewaktu-waktu
+sewenang-wenang
+short-term
+sia-sia
+siang-siang
+siapa-siapa
+sibar-sibar
+sibur-sibur
+sida-sida
+siku-siku
+silah-silah
+silang-menyilang
+silir-semilir
+sinar-seminar
+sindir-menyindir
+singgah-menyinggah
+sorak-sorai
+stand-by
+stand-up
+sudu-sudu
+sudung-sudung
+suka-suka
+sulang-menyulang
+sulur-suluran
+sumpah-sumpah
+sumpit-sumpit
+sungguh-sungguh
+sungut-sungut
+suram-suram
+surat-menyurat
+suruh-suruhan
+tabar-tabar
+tabir-mabir
+tabrak-tubruk
+tabuh-tabuhan
+tahu-menahu
+tahu-tahu
+takang-takik
+take-off
+takut-takut
+takut-takutan
+tali-bertali
+tali-tali
+tampak-tampak
+tanam-menanam
+tanam-tanaman
+tanda-tanda
+tangan-menangan
+tangan-tangan
+tanggung-menanggung
+tapa-tapa
+tapak-tapak
+tari-menari
+tari-tarian
+tarik-menarik
+tatah-tatah
+tawak-tawak
+tawang-tawang
+tawar-menawar
+tawar-tawar
+tayum-temayum
+tebu-tebu
+tegak-tegak
+teka-teki
+temas-temas
+tembak-menembak
+temut-temut
+tenggang-menenggang
+teraba-raba
+terambang-ambang
+terang-terang
+terang-terangan
+teranggar-anggar
+terangguk-angguk
+teranggul-anggul
+terangin-angin
+terangkup-angkup
+teranja-anja
+terapung-apung
+terayan-rayan
+terayap-rayap
+terbada-bada
+terbahak-bahak
+terbata-bata
+terbatuk-batuk
+terbayang-bayang
+terbengkil-bengkil
+terbirit-birit
+terbuai-buai
+terbuang-buang
+terburu-buru
+tercangak-cangak
+tercengang-cengang
+tercilap-cilap
+tercongget-congget
+tercungap-cungap
+terdangka-dangka
+terdengih-dengih
+terekeh-ekeh
+terembut-embut
+terembut-rembut
+terengah-engah
+teresak-esak
+tergagap-gagap
+tergagau-gagau
+tergaguk-gaguk
+tergapai-gapai
+tergegap-gegap
+tergegas-gegas
+tergelung-gelung
+tergerenyeng-gerenyeng
+tergesa-gesa
+tergila-gila
+tergontai-gontai
+tergudik-gudik
+terguling-guling
+tergulut-gulut
+terharak-harak
+terharap-harap
+terhengit-hengit
+terhinggut-hinggut
+terigau-igau
+terincut-incut
+teringa-inga
+teringat-ingat
+terinjak-injak
+terjembak-jembak
+terjerit-jerit
+terkadang-kadang
+terkakah-kakah
+terkakak-kakak
+terkanjar-kanjar
+terkapah-kapah
+terkapai-kapai
+terkapung-kapung
+terkatah-katah
+terkatung-katung
+terkecap-kecap
+terkedek-kedek
+terkedip-kedip
+terkejar-kejar
+terkekau-kekau
+terkekeh-kekeh
+terkekek-kekek
+terkelinjat-kelinjat
+terkelip-kelip
+terkempul-kempul
+terkemut-kemut
+terkencar-kencar
+terkepak-kepak
+terkesot-kesot
+terkesut-kesut
+terkial-kial
+terkincak-kincak
+terkindap-kindap
+terkinja-kinja
+terkirai-kirai
+terkitar-kitar
+terkocoh-kocoh
+terkokol-kokol
+terkosel-kosel
+terkoteng-koteng
+terkumpal-kumpal
+terlara-lara
+terlayang-layang
+terlebih-lebih
+terlincah-lincah
+terliuk-liuk
+terlolong-lolong
+terlongong-longong
+termangu-mangu
+termanja-manja
+termata-mata
+termengah-mengah
+termimpi-mimpi
+ternanti-nanti
+terngiang-ngiang
+teroleng-oleng
+terpandang-pandang
+terpecah-pecah
+terpekik-pekik
+terpereh-pereh
+terpikau-pikau
+terpinga-pinga
+terpingkal-pingkal
+terpontang-panting
+terpusing-pusing
+terputus-putus
+tersanga-sanga
+tersaruk-saruk
+tersedan-sedan
+tersedih-sedih
+tersedu-sedu
+tersendat-sendat
+tersendeng-sendeng
+tersengal-sengal
+tersengguk-sengguk
+tersengut-sengut
+tersera-sera
+terserak-serak
+tersetai-setai
+tersia-sia
+tersipu-sipu
+tersoja-soja
+tersungkuk-sungkuk
+tertagak-tagak
+tertahan-tahan
+tertatih-tatih
+tertegun-tegun
+tertekan-tekan
+terteleng-teleng
+terumbang-ambing
+terumbang-umbang
+terungkap-ungkap
+terus-menerus
+terus-terusan
+think-tank
+tiap-tiap
+tiba-tiba
+tidak-tidak
+tidur-tidur
+tie-dye
+tiga-tiganya
+tikam-menikam
+tilik-menilik
+timah-timah
+timang-timangan
+timbang-menimbang
+timu-timu
+tindih-bertindih
+tinjau-meninjau
+tip-off
+tiru-tiruan
+tiup-tiup
+tokak-takik
+tokok-menokok
+tolak-menolak
+tolong-menolong
+top-level
+trade-in
+tua-tua
+tuan-tuan
+tuang-tuang
+tuban-tuban
+tukang-menukang
+tukar-menukar
+tulang-tulangan
+tuli-tuli
+tulis-menulis
+tumbuh-tumbuhan
+tune-up
+tunggang-tunggit
+tupai-tupai
+turun-temurun
+turut-menurut
+turut-turutan
+two-tone
+uar-uar
+ubel-ubel
+ubun-ubun
+ubur-ubur
+uci-uci
+udap-udapan
+ugal-ugalan
+uir-uir
+ujar-ujar
+ukir-mengukir
+ula-ula
+ulak-ulak
+ulang-alik
+ulang-aling
+ulang-ulang
+ulap-ulap
+ular-ular
+ular-ularan
+ulung-ulung
+umang-umang
+umbang-ambing
+umbi-umbian
+umbul-umbul
+umbut-umbut
+uncang-uncit
+undak-undakan
+undang-undang
+unduk-unduk
+undung-undung
+undur-undur
+unggat-unggit
+ungkit-ungkit
+unting-unting
+untung-untung
+untung-untungan
+upside-down
+ura-ura
+uran-uran
+urat-urat
+uring-uringan
+urup-urup
+urup-urupan
+urus-urus
+user-user
+user-useran
+utar-utar
+voice-over
+walk-out
+wangi-wangian
+wanti-wanti
+wara-wara
+warna-warni
+water-cooled
+world-class
+yang-yang
+""".split()
+)
diff --git a/spacy/lang/ms/examples.py b/spacy/lang/ms/examples.py
new file mode 100644
index 000000000..97ab19b6e
--- /dev/null
+++ b/spacy/lang/ms/examples.py
@@ -0,0 +1,17 @@
+"""
+Example sentences to test spaCy and its language models.
+
+>>> from spacy.lang.ms.examples import sentences
+>>> docs = nlp.pipe(sentences)
+"""
+
+
+sentences = [
+ "Malaysia ialah sebuah negara yang terletak di Asia Tenggara.",
+ "Berapa banyak pelajar yang akan menghadiri majlis perpisahan sekolah?",
+ "Pengeluaran makanan berasal dari beberapa lokasi termasuk Cameron Highlands, Johor Bahru, dan Kuching.",
+ "Syarikat XYZ telah menghasilkan 20,000 unit produk baharu dalam setahun terakhir",
+ "Kuala Lumpur merupakan ibu negara Malaysia." "Kau berada di mana semalam?",
+ "Siapa yang akan memimpin projek itu?",
+ "Siapa perdana menteri Malaysia sekarang?",
+]
diff --git a/spacy/lang/ms/lex_attrs.py b/spacy/lang/ms/lex_attrs.py
new file mode 100644
index 000000000..2088c9955
--- /dev/null
+++ b/spacy/lang/ms/lex_attrs.py
@@ -0,0 +1,65 @@
+import unicodedata
+
+from ...attrs import IS_CURRENCY, LIKE_NUM
+from .punctuation import LIST_CURRENCY
+
+_num_words = [
+ "kosong",
+ "satu",
+ "dua",
+ "tiga",
+ "empat",
+ "lima",
+ "enam",
+ "tujuh",
+ "lapan",
+ "sembilan",
+ "sepuluh",
+ "sebelas",
+ "belas",
+ "puluh",
+ "ratus",
+ "ribu",
+ "juta",
+ "billion",
+ "trillion",
+ "kuadrilion",
+ "kuintilion",
+ "sekstilion",
+ "septilion",
+ "oktilion",
+ "nonilion",
+ "desilion",
+]
+
+
+def like_num(text):
+ if text.startswith(("+", "-", "±", "~")):
+ text = text[1:]
+ text = text.replace(",", "").replace(".", "")
+ if text.isdigit():
+ return True
+ if text.count("/") == 1:
+ num, denom = text.split("/")
+ if num.isdigit() and denom.isdigit():
+ return True
+ if text.lower() in _num_words:
+ return True
+ if text.count("-") == 1:
+ _, num = text.split("-")
+ if num.isdigit() or num in _num_words:
+ return True
+ return False
+
+
+def is_currency(text):
+ if text in LIST_CURRENCY:
+ return True
+
+ for char in text:
+ if unicodedata.category(char) != "Sc":
+ return False
+ return True
+
+
+LEX_ATTRS = {IS_CURRENCY: is_currency, LIKE_NUM: like_num}
diff --git a/spacy/lang/ms/punctuation.py b/spacy/lang/ms/punctuation.py
new file mode 100644
index 000000000..a8d6c2e8e
--- /dev/null
+++ b/spacy/lang/ms/punctuation.py
@@ -0,0 +1,60 @@
+from ..char_classes import ALPHA, _currency, _units, merge_chars, split_chars
+from ..punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
+
+_units = (
+ _units + "s bit Gbps Mbps mbps Kbps kbps ƒ ppi px "
+ "Hz kHz MHz GHz mAh "
+ "ratus rb ribu ribuan "
+ "juta jt jutaan mill?iar million bil[l]?iun bilyun billion "
+)
+_currency = _currency + r" USD RM MYR Rp IDR RMB SGD S\$"
+_months = (
+ "Januari Februari Mac April Mei Jun Julai Ogos September "
+ "Oktober November Disember Januari Februari Mac Mei Jun "
+ "Julai Ogos Oktober Disember Jan Feb Mac Jun Julai Ogos Sept "
+ "Okt Nov Dis"
+)
+
+
+UNITS = merge_chars(_units)
+CURRENCY = merge_chars(_currency)
+HTML_PREFIX = r"<(b|strong|i|em|p|span|div|br)\s?/>|
- {Children.toArray(children).flat().filter(isRelevant)}
+
+
+ {Children.toArray(children).flat().filter(isRelevant)}
+