From 2bba2a35362cbe00449f6f452dd6166620cb92d6 Mon Sep 17 00:00:00 2001 From: intrafind <2115805+intrafindBreno@users.noreply.github.com> Date: Mon, 3 Jun 2019 18:32:47 +0200 Subject: [PATCH 01/18] Fix for #3811 (#3815) Corrected type of seed parameter. --- spacy/cli/pretrain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/pretrain.py b/spacy/cli/pretrain.py index b2c22d929..3b2981f0d 100644 --- a/spacy/cli/pretrain.py +++ b/spacy/cli/pretrain.py @@ -33,7 +33,7 @@ from .. import util batch_size=("Number of words per training batch", "option", "bs", int), max_length=("Max words per example.", "option", "xw", int), min_length=("Min words per example.", "option", "nw", int), - seed=("Seed for random number generators", "option", "s", float), + seed=("Seed for random number generators", "option", "s", int), n_iter=("Number of iterations to pretrain", "option", "i", int), n_save_every=("Save model every X batches.", "option", "se", int), ) From 436a578369217cce1960c6dd5ddeee2c0e4c5600 Mon Sep 17 00:00:00 2001 From: intrafind <2115805+intrafindBreno@users.noreply.github.com> Date: Mon, 3 Jun 2019 18:33:09 +0200 Subject: [PATCH 02/18] Create intrafindBreno.md (#3814) --- .github/contributors/intrafindBreno.md | 106 +++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 .github/contributors/intrafindBreno.md diff --git a/.github/contributors/intrafindBreno.md b/.github/contributors/intrafindBreno.md new file mode 100644 index 000000000..204d20c07 --- /dev/null +++ b/.github/contributors/intrafindBreno.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Breno Faria | +| Company name (if applicable) | IntraFind | +| Title or role (if applicable) | Product Lead | +| Date | 03.06.2019 | +| GitHub username | intrafindBreno | +| Website (optional) | | From eb12703d1044c52ea50df7fb6944bedf640af3da Mon Sep 17 00:00:00 2001 From: Ramanan Balakrishnan Date: Tue, 4 Jun 2019 14:45:35 +0530 Subject: [PATCH 03/18] minor fix to broken link in documentation (#3819) [ci skip] --- website/docs/usage/processing-pipelines.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/usage/processing-pipelines.md b/website/docs/usage/processing-pipelines.md index 871ca3db6..0fa243501 100644 --- a/website/docs/usage/processing-pipelines.md +++ b/website/docs/usage/processing-pipelines.md @@ -41,7 +41,7 @@ components. spaCy then does the following: `Language` class contains the shared vocabulary, tokenization rules and the language-specific annotation scheme. 2. Iterate over the **pipeline names** and create each component using - [`create_pipe`](/api/anguage#create_pipe), which looks them up in + [`create_pipe`](/api/language#create_pipe), which looks them up in `Language.factories`. 3. Add each pipeline component to the pipeline in order, using [`add_pipe`](/api/language#add_pipe). From 511977ae5e6e1eca3be7c1cbe02e9294ca5114dc Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Tue, 4 Jun 2019 11:15:51 +0200 Subject: [PATCH 04/18] Update universe [ci skip] --- website/meta/universe.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index b3f0ccc5f..851b6107f 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1092,7 +1092,7 @@ { "type": "education", "id": "podcast-nlp-highlights", - "title": "NLP Highlights 78: Where do corpora come from?", + "title": "NLP Highlights #78: Where do corpora come from?", "slogan": "January 2019", "description": "Most NLP projects rely crucially on the quality of annotations used for training and evaluating models. In this episode, Matt and Ines of Explosion AI tell us how Prodigy can improve data annotation and model development workflows. Prodigy is an annotation tool implemented as a python library, and it comes with a web application and a command line interface. A developer can define input data streams and design simple annotation interfaces. Prodigy can help break down complex annotation decisions into a series of binary decisions, and it provides easy integration with spaCy models. Developers can specify how models should be modified as new annotations come in in an active learning framework.", "soundcloud": "559200912", @@ -1107,7 +1107,7 @@ { "type": "education", "id": "podcast-init", - "title": "Podcast.__init__ 87: spaCy with Matthew Honnibal", + "title": "Podcast.__init__ #87: spaCy with Matthew Honnibal", "slogan": "December 2017", "description": "As the amount of text available on the internet and in businesses continues to increase, the need for fast and accurate language analysis becomes more prominent. This week Matthew Honnibal, the creator of SpaCy, talks about his experiences researching natural language processing and creating a library to make his findings accessible to industry.", "iframe": "https://www.pythonpodcast.com/wp-content/plugins/podlove-podcasting-plugin-for-wordpress/lib/modules/podlove_web_player/player_v4/dist/share.html?episode=https://www.pythonpodcast.com/?podlove_player4=176", @@ -1123,7 +1123,7 @@ { "type": "education", "id": "talk-python-podcast", - "title": "Talk Python 202: Building a software business", + "title": "Talk Python #202: Building a software business", "slogan": "March 2019", "description": "One core question around open source is how do you fund it? Well, there is always that PayPal donate button. But that's been a tremendous failure for many projects. Often the go-to answer is consulting. But what if you don't want to trade time for money? You could take things up a notch and change the equation, exchanging value for money. That's what Ines Montani and her co-founder did when they started Explosion AI with spaCy as the foundation.", "thumb": "https://i.imgur.com/q1twuK8.png", From 5d6b4bb3bdd856d2a1d7926f4b5cbdde3849a921 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Fri, 7 Jun 2019 11:14:32 +0200 Subject: [PATCH 05/18] Update srsly pin --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 42045a829..8cc52dfe4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ thinc>=7.0.2,<7.1.0 blis>=0.2.2,<0.3.0 murmurhash>=0.28.0,<1.1.0 wasabi>=0.2.0,<1.1.0 -srsly>=0.0.5,<1.1.0 +srsly>=0.0.6,<1.1.0 # Third party dependencies numpy>=1.15.0 requests>=2.13.0,<3.0.0 diff --git a/setup.py b/setup.py index 0c2b541bd..33623588c 100755 --- a/setup.py +++ b/setup.py @@ -233,7 +233,7 @@ def setup_package(): "plac<1.0.0,>=0.9.6", "requests>=2.13.0,<3.0.0", "wasabi>=0.2.0,<1.1.0", - "srsly>=0.0.5,<1.1.0", + "srsly>=0.0.6,<1.1.0", 'pathlib==1.0.1; python_version < "3.4"', ], setup_requires=["wheel"], From a931d72459ab83c4583a995b8b433bd2470087f7 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 7 Jun 2019 20:40:41 +0200 Subject: [PATCH 06/18] Add merge_subtokens as parser post-process. Re #3830 --- spacy/pipeline/pipes.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spacy/pipeline/pipes.pyx b/spacy/pipeline/pipes.pyx index 7043c1647..f16a1a21e 100644 --- a/spacy/pipeline/pipes.pyx +++ b/spacy/pipeline/pipes.pyx @@ -13,6 +13,7 @@ from thinc.v2v import Affine, Maxout, Softmax from thinc.misc import LayerNorm from thinc.neural.util import to_categorical, copy_array +from .functions import merge_subtokens from ..tokens.doc cimport Doc from ..syntax.nn_parser cimport Parser from ..syntax.ner cimport BiluoPushDown @@ -1000,7 +1001,7 @@ cdef class DependencyParser(Parser): @property def postprocesses(self): - return [nonproj.deprojectivize] + return [nonproj.deprojectivize, merge_subtokens] def add_multitask_objective(self, target): if target == "cloze": From d0d56635ce5a2561a1e4f48a289246a5f62c3157 Mon Sep 17 00:00:00 2001 From: Azagh3l Date: Tue, 11 Jun 2019 10:58:32 +0200 Subject: [PATCH 07/18] Create Azagh3l.md (#3836) --- .github/contributors/Azagh3l.md | 106 ++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 .github/contributors/Azagh3l.md diff --git a/.github/contributors/Azagh3l.md b/.github/contributors/Azagh3l.md new file mode 100644 index 000000000..b0bf670d0 --- /dev/null +++ b/.github/contributors/Azagh3l.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ranty Baptiste | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 11/06/2019 | +| GitHub username | Azagh3l | +| Website (optional) | | From eb3e4263eec926447d274299b9882173ec7db22a Mon Sep 17 00:00:00 2001 From: Azagh3l Date: Tue, 11 Jun 2019 10:59:16 +0200 Subject: [PATCH 08/18] Update lex_attrs.py (#3835) Corrected typos, added french (from France) versions of some numbers. --- spacy/lang/fr/lex_attrs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spacy/lang/fr/lex_attrs.py b/spacy/lang/fr/lex_attrs.py index 470398d71..e3ccd9fdd 100644 --- a/spacy/lang/fr/lex_attrs.py +++ b/spacy/lang/fr/lex_attrs.py @@ -8,7 +8,7 @@ _num_words = set( """ zero un deux trois quatre cinq six sept huit neuf dix onze douze treize quatorze quinze seize dix-sept dix-huit dix-neuf -vingt trente quanrante cinquante soixante septante quatre-vingt huitante nonante +vingt trente quarante cinquante soixante soixante-dix septante quatre-vingt huitante quatre-vingt-dix nonante cent mille mil million milliard billion quadrillion quintillion sextillion septillion octillion nonillion decillion """.split() @@ -17,8 +17,8 @@ sextillion septillion octillion nonillion decillion _ordinal_words = set( """ premier deuxième second troisième quatrième cinquième sixième septième huitième neuvième dixième -onzième douzième treizième quatorzième quinzième seizième dix-septième dix-huitième dix-neufième -vingtième trentième quanrantième cinquantième soixantième septantième quatre-vingtième huitantième nonantième +onzième douzième treizième quatorzième quinzième seizième dix-septième dix-huitième dix-neuvième +vingtième trentième quarantième cinquantième soixantième soixante-dixième septantième quatre-vingtième huitantième quatre-vingt-dixième nonantième centième millième millionnième milliardième billionnième quadrillionnième quintillionnième sextillionnième septillionnième octillionnième nonillionnième decillionnième """.split() From 9c064e6ad95ac2733fc27a874644b6ad8caecedf Mon Sep 17 00:00:00 2001 From: Motoki Wu Date: Wed, 12 Jun 2019 04:29:23 -0700 Subject: [PATCH 09/18] Add resume logic to spacy pretrain (#3652) * Added ability to resume training * Add to readmee * Remove duplicate entry --- spacy/cli/pretrain.py | 15 +++++++++++++++ website/docs/api/cli.md | 1 + 2 files changed, 16 insertions(+) diff --git a/spacy/cli/pretrain.py b/spacy/cli/pretrain.py index 3b2981f0d..18fe0598f 100644 --- a/spacy/cli/pretrain.py +++ b/spacy/cli/pretrain.py @@ -18,6 +18,7 @@ from ..attrs import ID, HEAD from .._ml import Tok2Vec, flatten, chain, create_default_optimizer from .._ml import masked_language_model from .. import util +from .train import _load_pretrained_tok2vec @plac.annotations( @@ -36,6 +37,12 @@ from .. import util seed=("Seed for random number generators", "option", "s", int), n_iter=("Number of iterations to pretrain", "option", "i", int), n_save_every=("Save model every X batches.", "option", "se", int), + init_tok2vec=( + "Path to pretrained weights for the token-to-vector parts of the models. See 'spacy pretrain'. Experimental.", + "option", + "t2v", + Path, + ), ) def pretrain( texts_loc, @@ -53,6 +60,7 @@ def pretrain( min_length=5, seed=0, n_save_every=None, + init_tok2vec=None, ): """ Pre-train the 'token-to-vector' (tok2vec) layer of pipeline components, @@ -70,6 +78,9 @@ def pretrain( errors around this need some improvement. """ config = dict(locals()) + for key in config: + if isinstance(config[key], Path): + config[key] = str(config[key]) msg = Printer() util.fix_random_seed(seed) @@ -112,6 +123,10 @@ def pretrain( subword_features=True, # Set to False for Chinese etc ), ) + # Load in pre-trained weights + if init_tok2vec is not None: + components = _load_pretrained_tok2vec(nlp, init_tok2vec) + msg.text("Loaded pretrained tok2vec for: {}".format(components)) optimizer = create_default_optimizer(model.ops) tracker = ProgressTracker(frequency=10000) msg.divider("Pre-training tok2vec layer") diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index 7788d7a8f..8b982309a 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -305,6 +305,7 @@ $ python -m spacy pretrain [texts_loc] [vectors_model] [output_dir] [--width] | `--n-iter`, `-i` | option | Number of iterations to pretrain. | | `--use-vectors`, `-uv` | flag | Whether to use the static vectors as input features. | | `--n-save_every`, `-se` | option | Save model every X batches. | +| `--init-tok2vec`, `-t2v` 2.1 | option | Path to pretrained weights for the token-to-vector parts of the models. See `spacy pretrain`. Experimental.| | **CREATES** | weights | The pre-trained weights that can be used to initialize `spacy train`. | ### JSONL format for raw text {#pretrain-jsonl} From aae9034492e2b8f558765bc3c07340b32915aa8c Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Wed, 12 Jun 2019 13:38:23 +0200 Subject: [PATCH 10/18] Tidy up [ci skip] --- spacy/lang/mr/__init__.py | 2 +- spacy/tests/regression/test_issue3803.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/spacy/lang/mr/__init__.py b/spacy/lang/mr/__init__.py index 538540935..fd95f9354 100644 --- a/spacy/lang/mr/__init__.py +++ b/spacy/lang/mr/__init__.py @@ -1,4 +1,4 @@ -#coding: utf8 +# coding: utf8 from __future__ import unicode_literals from .stop_words import STOP_WORDS diff --git a/spacy/tests/regression/test_issue3803.py b/spacy/tests/regression/test_issue3803.py index 4d9b664fa..37d15a5cf 100644 --- a/spacy/tests/regression/test_issue3803.py +++ b/spacy/tests/regression/test_issue3803.py @@ -1,8 +1,6 @@ # coding: utf8 from __future__ import unicode_literals -import pytest - from spacy.lang.es import Spanish From f35ce097762c3bfc7010cfe6ee8398c66d8223b0 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Wed, 12 Jun 2019 13:38:30 +0200 Subject: [PATCH 11/18] Add regression test for #3839 --- spacy/tests/regression/test_issue3839.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 spacy/tests/regression/test_issue3839.py diff --git a/spacy/tests/regression/test_issue3839.py b/spacy/tests/regression/test_issue3839.py new file mode 100644 index 000000000..fa915faf0 --- /dev/null +++ b/spacy/tests/regression/test_issue3839.py @@ -0,0 +1,23 @@ +# coding: utf8 +from __future__ import unicode_literals + +import pytest +from spacy.matcher import Matcher +from spacy.tokens import Doc + + +@pytest.mark.xfail +def test_issue3839(en_vocab): + """Test that match IDs returned by the matcher are correct, are in the string """ + doc = Doc(en_vocab, words=["terrific", "group", "of", "people"]) + matcher = Matcher(en_vocab) + match_id = "PATTERN" + pattern1 = [{"LOWER": "terrific"}, {"OP": "?"}, {"LOWER": "group"}] + pattern2 = [{"LOWER": "terrific"}, {"OP": "?"}, {"OP": "?"}, {"LOWER": "group"}] + matcher.add(match_id, None, pattern1) + matches = matcher(doc) + assert matches[0][0] == en_vocab.strings[match_id] + matcher = Matcher(en_vocab) + matcher.add(match_id, None, pattern2) + matches = matcher(doc) + assert matches[0][0] == en_vocab.strings[match_id] From 5accfbb9381172a83905bb14623784a3dd4f1473 Mon Sep 17 00:00:00 2001 From: Azagh3l Date: Fri, 14 Jun 2019 09:31:05 +0200 Subject: [PATCH 12/18] Update exemples.py (#3838) Added missing hyphen and accent. --- spacy/lang/fr/examples.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/lang/fr/examples.py b/spacy/lang/fr/examples.py index bf508022e..a874c22fc 100644 --- a/spacy/lang/fr/examples.py +++ b/spacy/lang/fr/examples.py @@ -11,7 +11,7 @@ Example sentences to test spaCy and its language models. sentences = [ - "Apple cherche à acheter une startup anglaise pour 1 milliard de dollars", + "Apple cherche à acheter une start-up anglaise pour 1 milliard de dollars", "Les voitures autonomes déplacent la responsabilité de l'assurance vers les constructeurs", "San Francisco envisage d'interdire les robots coursiers sur les trottoirs", "Londres est une grande ville du Royaume-Uni", @@ -21,6 +21,6 @@ sentences = [ "Nouvelles attaques de Trump contre le maire de Londres", "Où es-tu ?", "Qui est le président de la France ?", - "Où est la capitale des Etats-Unis ?", + "Où est la capitale des États-Unis ?", "Quand est né Barack Obama ?", ] From d8573ee715701cc7227378d08bb4b7e3bc62c030 Mon Sep 17 00:00:00 2001 From: BreakBB <33514570+BreakBB@users.noreply.github.com> Date: Sun, 16 Jun 2019 13:22:57 +0200 Subject: [PATCH 13/18] Update error raising for CLI pretrain to fix #3840 (#3843) * Add check for empty input file to CLI pretrain * Raise error if JSONL is not a dict or contains neither `tokens` nor `text` key * Skip empty values for correct pretrain keys and log a counter as warning * Add tests for CLI pretrain core function make_docs. * Add a short hint for the `tokens` key to the CLI pretrain docs * Add success message to CLI pretrain * Update model loading to fix the tests * Skip empty values and do not create docs out of it --- spacy/cli/pretrain.py | 28 +++++++++++++++++++++---- spacy/errors.py | 6 ++++++ spacy/tests/test_cli.py | 46 +++++++++++++++++++++++++++++++++++++++++ website/docs/api/cli.md | 2 +- 4 files changed, 77 insertions(+), 5 deletions(-) diff --git a/spacy/cli/pretrain.py b/spacy/cli/pretrain.py index 18fe0598f..be8733c62 100644 --- a/spacy/cli/pretrain.py +++ b/spacy/cli/pretrain.py @@ -13,6 +13,7 @@ from thinc.neural.util import prefer_gpu, get_array_module from wasabi import Printer import srsly +from ..errors import Errors from ..tokens import Doc from ..attrs import ID, HEAD from .._ml import Tok2Vec, flatten, chain, create_default_optimizer @@ -101,6 +102,8 @@ def pretrain( msg.fail("Input text file doesn't exist", texts_loc, exits=1) with msg.loading("Loading input texts..."): texts = list(srsly.read_jsonl(texts_loc)) + if not texts: + msg.fail("Input file is empty", texts_loc, exits=1) msg.good("Loaded input texts") random.shuffle(texts) else: # reading from stdin @@ -149,16 +152,18 @@ def pretrain( with (output_dir / "log.jsonl").open("a") as file_: file_.write(srsly.json_dumps(log) + "\n") + skip_counter = 0 for epoch in range(n_iter): for batch_id, batch in enumerate( util.minibatch_by_words(((text, None) for text in texts), size=batch_size) ): - docs = make_docs( + docs, count = make_docs( nlp, [text for (text, _) in batch], max_length=max_length, min_length=min_length, ) + skip_counter += count loss = make_update( model, docs, optimizer, objective=loss_func, drop=dropout ) @@ -174,6 +179,9 @@ def pretrain( if texts_loc != "-": # Reshuffle the texts if texts were loaded from a file random.shuffle(texts) + if skip_counter > 0: + msg.warn("Skipped {count} empty values".format(count=str(skip_counter))) + msg.good("Successfully finished pretrain") def make_update(model, docs, optimizer, drop=0.0, objective="L2"): @@ -195,12 +203,24 @@ def make_update(model, docs, optimizer, drop=0.0, objective="L2"): def make_docs(nlp, batch, min_length, max_length): docs = [] + skip_count = 0 for record in batch: + if not isinstance(record, dict): + raise TypeError(Errors.E137.format(type=type(record), line=record)) if "tokens" in record: - doc = Doc(nlp.vocab, words=record["tokens"]) - else: + words = record["tokens"] + if not words: + skip_count += 1 + continue + doc = Doc(nlp.vocab, words=words) + elif "text" in record: text = record["text"] + if not text: + skip_count += 1 + continue doc = nlp.make_doc(text) + else: + raise ValueError(Errors.E138.format(text=record)) if "heads" in record: heads = record["heads"] heads = numpy.asarray(heads, dtype="uint64") @@ -208,7 +228,7 @@ def make_docs(nlp, batch, min_length, max_length): doc = doc.from_array([HEAD], heads) if len(doc) >= min_length and len(doc) < max_length: docs.append(doc) - return docs + return docs, skip_count def get_vectors_loss(ops, docs, prediction, objective="L2"): diff --git a/spacy/errors.py b/spacy/errors.py index 3a1e05e05..fcc3132c6 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -393,6 +393,12 @@ class Errors(object): "`nlp.replace_pipe('{name}', nlp.create_pipe('{name}'))`") E136 = ("This additional feature requires the jsonschema library to be " "installed:\npip install jsonschema") + E137 = ("Expected 'dict' type, but got '{type}' from '{line}'. Make sure to provide a valid JSON " + "object as input with either the `text` or `tokens` key. For more info, see the docs:\n" + "https://spacy.io/api/cli#pretrain-jsonl") + E138 = ("Invalid JSONL format for raw text '{text}'. Make sure the input includes either the " + "`text` or `tokens` key. For more info, see the docs:\n" + "https://spacy.io/api/cli#pretrain-jsonl") @add_codes diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 2afa6a71b..f0c34276d 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -1,7 +1,11 @@ # coding: utf-8 from __future__ import unicode_literals +import pytest + +from spacy.lang.en import English from spacy.cli.converters import conllu2json +from spacy.cli.pretrain import make_docs def test_cli_converters_conllu2json(): @@ -26,3 +30,45 @@ def test_cli_converters_conllu2json(): assert [t["head"] for t in tokens] == [1, 2, -1, 0] assert [t["dep"] for t in tokens] == ["appos", "nsubj", "name", "ROOT"] assert [t["ner"] for t in tokens] == ["O", "B-PER", "L-PER", "O"] + + +def test_pretrain_make_docs(): + nlp = English() + + valid_jsonl_text = {"text": "Some text"} + docs, skip_count = make_docs(nlp, [valid_jsonl_text], 1, 10) + assert len(docs) == 1 + assert skip_count == 0 + + valid_jsonl_tokens = {"tokens": ["Some", "tokens"]} + docs, skip_count = make_docs(nlp, [valid_jsonl_tokens], 1, 10) + assert len(docs) == 1 + assert skip_count == 0 + + invalid_jsonl_type = 0 + with pytest.raises(TypeError): + make_docs(nlp, [invalid_jsonl_type], 1, 100) + + invalid_jsonl_key = {"invalid": "Does not matter"} + with pytest.raises(ValueError): + make_docs(nlp, [invalid_jsonl_key], 1, 100) + + empty_jsonl_text = {"text": ""} + docs, skip_count = make_docs(nlp, [empty_jsonl_text], 1, 10) + assert len(docs) == 0 + assert skip_count == 1 + + empty_jsonl_tokens = {"tokens": []} + docs, skip_count = make_docs(nlp, [empty_jsonl_tokens], 1, 10) + assert len(docs) == 0 + assert skip_count == 1 + + too_short_jsonl = {"text": "This text is not long enough"} + docs, skip_count = make_docs(nlp, [too_short_jsonl], 10, 15) + assert len(docs) == 0 + assert skip_count == 0 + + too_long_jsonl = {"text": "This text contains way too much tokens for this test"} + docs, skip_count = make_docs(nlp, [too_long_jsonl], 1, 5) + assert len(docs) == 0 + assert skip_count == 0 diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index 8b982309a..ac4c7eddb 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -291,7 +291,7 @@ $ python -m spacy pretrain [texts_loc] [vectors_model] [output_dir] [--width] | Argument | Type | Description | | ----------------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------- | -| `texts_loc` | positional | Path to JSONL file with raw texts to learn from, with text provided as the key `"text"`. [See here](#pretrain-jsonl) for details. | +| `texts_loc` | positional | Path to JSONL file with raw texts to learn from, with text provided as the key `"text"` or tokens as the key `tokens`. [See here](#pretrain-jsonl) for details. | | `vectors_model` | positional | Name or path to spaCy model with vectors to learn from. | | `output_dir` | positional | Directory to write models to on each epoch. | | `--width`, `-cw` | option | Width of CNN layers. | From 3f52e1233531bf09976f4bdb33dd001f9e023882 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Sun, 16 Jun 2019 20:24:06 +0900 Subject: [PATCH 14/18] Change vector training to work with latest gensim (fix #3749) (#3757) --- bin/train_word_vectors.py | 51 ++++++++++----------------------------- 1 file changed, 13 insertions(+), 38 deletions(-) diff --git a/bin/train_word_vectors.py b/bin/train_word_vectors.py index 8482a7a55..624e339a0 100644 --- a/bin/train_word_vectors.py +++ b/bin/train_word_vectors.py @@ -13,23 +13,21 @@ logger = logging.getLogger(__name__) class Corpus(object): - def __init__(self, directory, min_freq=10): + def __init__(self, directory, nlp): self.directory = directory - self.counts = PreshCounter() - self.strings = {} - self.min_freq = min_freq - - def count_doc(self, doc): - # Get counts for this document - for word in doc: - self.counts.inc(word.orth, 1) - return len(doc) + self.nlp = nlp def __iter__(self): for text_loc in iter_dir(self.directory): with text_loc.open("r", encoding="utf-8") as file_: text = file_.read() - yield text + + # This is to keep the input to the blank model (which doesn't + # sentencize) from being too long. It works particularly well with + # the output of [WikiExtractor](https://github.com/attardi/wikiextractor) + paragraphs = text.split('\n\n') + for par in paragraphs: + yield [word.orth_ for word in self.nlp(par)] def iter_dir(loc): @@ -62,12 +60,15 @@ def main( window=5, size=128, min_count=10, - nr_iter=2, + nr_iter=5, ): logging.basicConfig( format="%(asctime)s : %(levelname)s : %(message)s", level=logging.INFO ) + nlp = spacy.blank(lang) + corpus = Corpus(in_dir, nlp) model = Word2Vec( + sentences=corpus, size=size, window=window, min_count=min_count, @@ -75,33 +76,7 @@ def main( sample=1e-5, negative=negative, ) - nlp = spacy.blank(lang) - corpus = Corpus(in_dir) - total_words = 0 - total_sents = 0 - for text_no, text_loc in enumerate(iter_dir(corpus.directory)): - with text_loc.open("r", encoding="utf-8") as file_: - text = file_.read() - total_sents += text.count("\n") - doc = nlp(text) - total_words += corpus.count_doc(doc) - logger.info( - "PROGRESS: at batch #%i, processed %i words, keeping %i word types", - text_no, - total_words, - len(corpus.strings), - ) - model.corpus_count = total_sents - model.raw_vocab = defaultdict(int) - for orth, freq in corpus.counts: - if freq >= min_count: - model.raw_vocab[nlp.vocab.strings[orth]] = freq - model.scale_vocab() - model.finalize_vocab() - model.iter = nr_iter - model.train(corpus) model.save(out_loc) - if __name__ == "__main__": plac.call(main) From 46c78d0a4152cee3ca8f8946914d3cacf6774506 Mon Sep 17 00:00:00 2001 From: Suraj Rajan Date: Sun, 16 Jun 2019 16:55:32 +0530 Subject: [PATCH 15/18] Dependency tree pattern matcher (#3465) * Functional dependency tree pattern matcher * Tests fail due to inconsistent behaviour * Renamed dependencymatcher and added optimizations --- spacy/matcher/__init__.py | 4 +- spacy/matcher/dependencymatcher.pyx | 107 +++++++++++------------- spacy/tests/matcher/test_matcher_api.py | 51 ++++++----- 3 files changed, 74 insertions(+), 88 deletions(-) diff --git a/spacy/matcher/__init__.py b/spacy/matcher/__init__.py index d3923754b..91874ed43 100644 --- a/spacy/matcher/__init__.py +++ b/spacy/matcher/__init__.py @@ -3,6 +3,6 @@ from __future__ import unicode_literals from .matcher import Matcher from .phrasematcher import PhraseMatcher -from .dependencymatcher import DependencyTreeMatcher +from .dependencymatcher import DependencyMatcher -__all__ = ["Matcher", "PhraseMatcher", "DependencyTreeMatcher"] +__all__ = ["Matcher", "PhraseMatcher", "DependencyMatcher"] diff --git a/spacy/matcher/dependencymatcher.pyx b/spacy/matcher/dependencymatcher.pyx index 8fca95a2d..b58d36d62 100644 --- a/spacy/matcher/dependencymatcher.pyx +++ b/spacy/matcher/dependencymatcher.pyx @@ -12,13 +12,15 @@ from ..tokens.doc cimport Doc from .matcher import unpickle_matcher from ..errors import Errors +from libcpp cimport bool +import numpy DELIMITER = "||" INDEX_HEAD = 1 INDEX_RELOP = 0 -cdef class DependencyTreeMatcher: +cdef class DependencyMatcher: """Match dependency parse tree based on pattern rules.""" cdef Pool mem cdef readonly Vocab vocab @@ -32,11 +34,11 @@ cdef class DependencyTreeMatcher: cdef public object _tree def __init__(self, vocab): - """Create the DependencyTreeMatcher. + """Create the DependencyMatcher. vocab (Vocab): The vocabulary object, which must be shared with the documents the matcher will operate on. - RETURNS (DependencyTreeMatcher): The newly constructed object. + RETURNS (DependencyMatcher): The newly constructed object. """ size = 20 self.token_matcher = Matcher(vocab) @@ -199,7 +201,7 @@ cdef class DependencyTreeMatcher: return (self._callbacks[key], self._patterns[key]) def __call__(self, Doc doc): - matched_trees = [] + matched_key_trees = [] matches = self.token_matcher(doc) for key in list(self._patterns.keys()): _patterns_list = self._patterns[key] @@ -227,51 +229,36 @@ cdef class DependencyTreeMatcher: _nodes,_root ) length = len(_nodes) - if _root in id_to_position: - candidates = id_to_position[_root] - for candidate in candidates: - isVisited = {} - self.dfs( - candidate, - _root,_tree, - id_to_position, - doc, - isVisited, - _node_operator_map - ) - # To check if the subtree pattern is completely - # identified. This is a heuristic. This is done to - # reduce the complexity of exponential unordered subtree - # matching. Will give approximate matches in some cases. - if(len(isVisited) == length): - matched_trees.append((key,list(isVisited))) - for i, (ent_id, nodes) in enumerate(matched_trees): + + matched_trees = [] + self.recurse(_tree,id_to_position,_node_operator_map,0,[],matched_trees) + matched_key_trees.append((key,matched_trees)) + + for i, (ent_id, nodes) in enumerate(matched_key_trees): on_match = self._callbacks.get(ent_id) if on_match is not None: on_match(self, doc, i, matches) - return matched_trees + return matched_key_trees - def dfs(self,candidate,root,tree,id_to_position,doc,isVisited,_node_operator_map): - if (root in id_to_position and candidate in id_to_position[root]): - # Color the node since it is valid - isVisited[candidate] = True - if root in tree: - for root_child in tree[root]: - if ( - candidate in _node_operator_map - and root_child[INDEX_RELOP] in _node_operator_map[candidate] - ): - candidate_children = _node_operator_map[candidate][root_child[INDEX_RELOP]] - for candidate_child in candidate_children: - result = self.dfs( - candidate_child.i, - root_child[INDEX_HEAD], - tree, - id_to_position, - doc, - isVisited, - _node_operator_map - ) + def recurse(self,tree,id_to_position,_node_operator_map,int patternLength,visitedNodes,matched_trees): + cdef bool isValid; + if(patternLength == len(id_to_position.keys())): + isValid = True + for node in range(patternLength): + if(node in tree): + for idx, (relop,nbor) in enumerate(tree[node]): + computed_nbors = numpy.asarray(_node_operator_map[visitedNodes[node]][relop]) + isNbor = False + for computed_nbor in computed_nbors: + if(computed_nbor.i == visitedNodes[nbor]): + isNbor = True + isValid = isValid & isNbor + if(isValid): + matched_trees.append(visitedNodes) + return + allPatternNodes = numpy.asarray(id_to_position[patternLength]) + for patternNode in allPatternNodes: + self.recurse(tree,id_to_position,_node_operator_map,patternLength+1,visitedNodes+[patternNode],matched_trees) # Given a node and an edge operator, to return the list of nodes # from the doc that belong to node+operator. This is used to store @@ -299,8 +286,8 @@ cdef class DependencyTreeMatcher: switcher = { "<": self.dep, ">": self.gov, - ">>": self.dep_chain, - "<<": self.gov_chain, + "<<": self.dep_chain, + ">>": self.gov_chain, ".": self.imm_precede, "$+": self.imm_right_sib, "$-": self.imm_left_sib, @@ -313,7 +300,7 @@ cdef class DependencyTreeMatcher: return _node_operator_map def dep(self, doc, node): - return list(doc[node].head) + return [doc[node].head] def gov(self,doc,node): return list(doc[node].children) @@ -330,29 +317,29 @@ cdef class DependencyTreeMatcher: return [] def imm_right_sib(self, doc, node): - for idx in range(list(doc[node].head.children)): - if idx == node - 1: - return [doc[idx]] + for child in list(doc[node].head.children): + if child.i == node - 1: + return [doc[child.i]] return [] def imm_left_sib(self, doc, node): - for idx in range(list(doc[node].head.children)): - if idx == node + 1: - return [doc[idx]] + for child in list(doc[node].head.children): + if child.i == node + 1: + return [doc[child.i]] return [] def right_sib(self, doc, node): candidate_children = [] - for idx in range(list(doc[node].head.children)): - if idx < node: - candidate_children.append(doc[idx]) + for child in list(doc[node].head.children): + if child.i < node: + candidate_children.append(doc[child.i]) return candidate_children def left_sib(self, doc, node): candidate_children = [] - for idx in range(list(doc[node].head.children)): - if idx > node: - candidate_children.append(doc[idx]) + for child in list(doc[node].head.children): + if child.i > node: + candidate_children.append(doc[child.i]) return candidate_children def _normalize_key(self, key): diff --git a/spacy/tests/matcher/test_matcher_api.py b/spacy/tests/matcher/test_matcher_api.py index 6ece07482..54ddd6789 100644 --- a/spacy/tests/matcher/test_matcher_api.py +++ b/spacy/tests/matcher/test_matcher_api.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import pytest import re -from spacy.matcher import Matcher, DependencyTreeMatcher +from spacy.matcher import Matcher, DependencyMatcher from spacy.tokens import Doc, Token from ..util import get_doc @@ -285,45 +285,44 @@ def deps(): @pytest.fixture -def dependency_tree_matcher(en_vocab): +def dependency_matcher(en_vocab): def is_brown_yellow(text): return bool(re.compile(r"brown|yellow|over").match(text)) - IS_BROWN_YELLOW = en_vocab.add_flag(is_brown_yellow) + pattern1 = [ {"SPEC": {"NODE_NAME": "fox"}, "PATTERN": {"ORTH": "fox"}}, - { - "SPEC": {"NODE_NAME": "q", "NBOR_RELOP": ">", "NBOR_NAME": "fox"}, - "PATTERN": {"LOWER": "quick"}, - }, - { - "SPEC": {"NODE_NAME": "r", "NBOR_RELOP": ">", "NBOR_NAME": "fox"}, - "PATTERN": {IS_BROWN_YELLOW: True}, - }, + {"SPEC": {"NODE_NAME": "q", "NBOR_RELOP": ">", "NBOR_NAME": "fox"},"PATTERN": {"ORTH": "quick", "DEP": "amod"}}, + {"SPEC": {"NODE_NAME": "r", "NBOR_RELOP": ">", "NBOR_NAME": "fox"}, "PATTERN": {IS_BROWN_YELLOW: True}}, ] pattern2 = [ {"SPEC": {"NODE_NAME": "jumped"}, "PATTERN": {"ORTH": "jumped"}}, - { - "SPEC": {"NODE_NAME": "fox", "NBOR_RELOP": ">", "NBOR_NAME": "jumped"}, - "PATTERN": {"LOWER": "fox"}, - }, - { - "SPEC": {"NODE_NAME": "over", "NBOR_RELOP": ">", "NBOR_NAME": "fox"}, - "PATTERN": {IS_BROWN_YELLOW: True}, - }, + {"SPEC": {"NODE_NAME": "fox", "NBOR_RELOP": ">", "NBOR_NAME": "jumped"}, "PATTERN": {"ORTH": "fox"}}, + {"SPEC": {"NODE_NAME": "quick", "NBOR_RELOP": ".", "NBOR_NAME": "jumped"}, "PATTERN": {"ORTH": "fox"}} ] - matcher = DependencyTreeMatcher(en_vocab) + + pattern3 = [ + {"SPEC": {"NODE_NAME": "jumped"}, "PATTERN": {"ORTH": "jumped"}}, + {"SPEC": {"NODE_NAME": "fox", "NBOR_RELOP": ">", "NBOR_NAME": "jumped"}, "PATTERN": {"ORTH": "fox"}}, + {"SPEC": {"NODE_NAME": "r", "NBOR_RELOP": ">>", "NBOR_NAME": "fox"}, "PATTERN": {"ORTH": "brown"}} + ] + + matcher = DependencyMatcher(en_vocab) matcher.add("pattern1", None, pattern1) matcher.add("pattern2", None, pattern2) + matcher.add("pattern3", None, pattern3) + return matcher -def test_dependency_tree_matcher_compile(dependency_tree_matcher): - assert len(dependency_tree_matcher) == 2 +def test_dependency_matcher_compile(dependency_matcher): + assert len(dependency_matcher) == 3 -def test_dependency_tree_matcher(dependency_tree_matcher, text, heads, deps): - doc = get_doc(dependency_tree_matcher.vocab, text.split(), heads=heads, deps=deps) - matches = dependency_tree_matcher(doc) - assert len(matches) == 2 +def test_dependency_matcher(dependency_matcher, text, heads, deps): + doc = get_doc(dependency_matcher.vocab, text.split(), heads=heads, deps=deps) + matches = dependency_matcher(doc) + # assert matches[0][1] == [[3, 1, 2]] + # assert matches[1][1] == [[4, 3, 3]] + # assert matches[2][1] == [[4, 3, 2]] From 1e19f34e29c03dd2751804a437bbb7f62c50771c Mon Sep 17 00:00:00 2001 From: Kabir Khan Date: Sun, 16 Jun 2019 04:29:04 -0700 Subject: [PATCH 16/18] Add optional `id` property to EntityRuler patterns (#3591) * Adding support for entity_id in EntityRuler pipeline component * Adding Spacy Contributor aggreement * Updating EntityRuler to use string.format instead of f strings * Update Entity Ruler to support an 'id' attribute per pattern that explicitly identifies an entity. * Fixing tests * Remove custom extension entity_id and use built in ent_id token attribute. * Changing entity_id to ent_id for consistent naming * entity_ids => ent_ids * Removing kb, cleaning up tests, making util functions private, use rsplit instead of split --- .github/contributors/kabirkhan.md | 106 ++++++++++++++++++++++ spacy/pipeline/entityruler.py | 64 ++++++++++++- spacy/tests/pipeline/test_entity_ruler.py | 26 +++++- 3 files changed, 190 insertions(+), 6 deletions(-) create mode 100644 .github/contributors/kabirkhan.md diff --git a/.github/contributors/kabirkhan.md b/.github/contributors/kabirkhan.md new file mode 100644 index 000000000..4dfb3dfa1 --- /dev/null +++ b/.github/contributors/kabirkhan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Kabir Khan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-04-08 | +| GitHub username | kabirkhan | +| Website (optional) | | diff --git a/spacy/pipeline/entityruler.py b/spacy/pipeline/entityruler.py index cd399a4fe..54fd4a062 100644 --- a/spacy/pipeline/entityruler.py +++ b/spacy/pipeline/entityruler.py @@ -48,6 +48,7 @@ class EntityRuler(object): self.phrase_patterns = defaultdict(list) self.matcher = Matcher(nlp.vocab) self.phrase_matcher = PhraseMatcher(nlp.vocab) + self.ent_id_sep = cfg.get("ent_id_sep", "||") patterns = cfg.get("patterns") if patterns is not None: self.add_patterns(patterns) @@ -84,7 +85,16 @@ class EntityRuler(object): continue # check for end - 1 here because boundaries are inclusive if start not in seen_tokens and end - 1 not in seen_tokens: - new_entities.append(Span(doc, start, end, label=match_id)) + if self.ent_ids: + label_ = self.nlp.vocab.strings[match_id] + ent_label, ent_id = self._split_label(label_) + span = Span(doc, start, end, label=ent_label) + if ent_id: + for token in span: + token.ent_id_ = ent_id + else: + span = Span(doc, start, end, label=match_id) + new_entities.append(span) entities = [ e for e in entities if not (e.start < end and e.end > start) ] @@ -104,6 +114,21 @@ class EntityRuler(object): all_labels.update(self.phrase_patterns.keys()) return tuple(all_labels) + @property + def ent_ids(self): + """All entity ids present in the match patterns meta dicts. + + RETURNS (set): The string entity ids. + + DOCS: https://spacy.io/api/entityruler#labels + """ + all_ent_ids = set() + for l in self.labels: + if self.ent_id_sep in l: + _, ent_id = self._split_label(l) + all_ent_ids.add(ent_id) + return tuple(all_ent_ids) + @property def patterns(self): """Get all patterns that were added to the entity ruler. @@ -115,10 +140,19 @@ class EntityRuler(object): all_patterns = [] for label, patterns in self.token_patterns.items(): for pattern in patterns: - all_patterns.append({"label": label, "pattern": pattern}) + ent_label, ent_id = self._split_label(label) + p = {"label": ent_label, "pattern": pattern} + if ent_id: + p["id"] = ent_id + all_patterns.append(p) for label, patterns in self.phrase_patterns.items(): for pattern in patterns: - all_patterns.append({"label": label, "pattern": pattern.text}) + ent_label, ent_id = self._split_label(label) + p = {"label": ent_label, "pattern": pattern.text} + if ent_id: + p["id"] = ent_id + all_patterns.append(p) + return all_patterns def add_patterns(self, patterns): @@ -133,6 +167,8 @@ class EntityRuler(object): """ for entry in patterns: label = entry["label"] + if "id" in entry: + label = self._create_label(label, entry["id"]) pattern = entry["pattern"] if isinstance(pattern, basestring_): self.phrase_patterns[label].append(self.nlp(pattern)) @@ -145,6 +181,28 @@ class EntityRuler(object): for label, patterns in self.phrase_patterns.items(): self.phrase_matcher.add(label, None, *patterns) + def _split_label(self, label): + """Split Entity label into ent_label and ent_id if it contains self.ent_id_sep + + RETURNS (tuple): ent_label, ent_id + """ + if self.ent_id_sep in label: + ent_label, ent_id = label.rsplit(self.ent_id_sep, 1) + else: + ent_label = label + ent_id = None + + return ent_label, ent_id + + def _create_label(self, label, ent_id): + """Join Entity label with ent_id if the pattern has an `id` attribute + + RETURNS (str): The ent_label joined with configured `ent_id_sep` + """ + if isinstance(ent_id, basestring_): + label = "{}{}{}".format(label, self.ent_id_sep, ent_id) + return label + def from_bytes(self, patterns_bytes, **kwargs): """Load the entity ruler from a bytestring. diff --git a/spacy/tests/pipeline/test_entity_ruler.py b/spacy/tests/pipeline/test_entity_ruler.py index 8757c9af5..040d5ff22 100644 --- a/spacy/tests/pipeline/test_entity_ruler.py +++ b/spacy/tests/pipeline/test_entity_ruler.py @@ -19,6 +19,7 @@ def patterns(): {"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]}, {"label": "HELLO", "pattern": [{"ORTH": "HELLO"}]}, {"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]}, + {"label": "TECH_ORG", "pattern": "Apple", "id": "a1"}, ] @@ -34,7 +35,7 @@ def add_ent(): def test_entity_ruler_init(nlp, patterns): ruler = EntityRuler(nlp, patterns=patterns) assert len(ruler) == len(patterns) - assert len(ruler.labels) == 3 + assert len(ruler.labels) == 4 assert "HELLO" in ruler assert "BYE" in ruler nlp.add_pipe(ruler) @@ -77,14 +78,33 @@ def test_entity_ruler_existing_complex(nlp, patterns, add_ent): assert len(doc.ents[1]) == 2 +def test_entity_ruler_entity_id(nlp, patterns): + ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True) + nlp.add_pipe(ruler) + doc = nlp("Apple is a technology company") + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "TECH_ORG" + assert doc.ents[0].ent_id_ == "a1" + + +def test_entity_ruler_cfg_ent_id_sep(nlp, patterns): + ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True, ent_id_sep="**") + assert "TECH_ORG**a1" in ruler.phrase_patterns + nlp.add_pipe(ruler) + doc = nlp("Apple is a technology company") + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "TECH_ORG" + assert doc.ents[0].ent_id_ == "a1" + + def test_entity_ruler_serialize_bytes(nlp, patterns): ruler = EntityRuler(nlp, patterns=patterns) assert len(ruler) == len(patterns) - assert len(ruler.labels) == 3 + assert len(ruler.labels) == 4 ruler_bytes = ruler.to_bytes() new_ruler = EntityRuler(nlp) assert len(new_ruler) == 0 assert len(new_ruler.labels) == 0 new_ruler = new_ruler.from_bytes(ruler_bytes) assert len(ruler) == len(patterns) - assert len(ruler.labels) == 3 + assert len(ruler.labels) == 4 From 9041a72d7f755b99504a3430224b18f46c7a7f79 Mon Sep 17 00:00:00 2001 From: Greg Werner Date: Sun, 16 Jun 2019 08:32:56 -0400 Subject: [PATCH 17/18] Update tokenizer.md for construction example (#3790) * Update tokenizer.md for construction example Self contained example. You should really say what nlp is so that the example will work as is * Update CONTRIBUTOR_AGREEMENT.md * Restore contributor agreement * Adjust construction examples --- .github/contributors/demongolem.md | 106 +++++++++++++++++++++++++++++ website/docs/api/tokenizer.md | 5 +- 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 .github/contributors/demongolem.md diff --git a/.github/contributors/demongolem.md b/.github/contributors/demongolem.md new file mode 100644 index 000000000..29cb088c1 --- /dev/null +++ b/.github/contributors/demongolem.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Gregory Werner | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 5/29/2019 | +| GitHub username | demongolem | +| Website (optional) | | diff --git a/website/docs/api/tokenizer.md b/website/docs/api/tokenizer.md index 3bec5b165..66ac315c6 100644 --- a/website/docs/api/tokenizer.md +++ b/website/docs/api/tokenizer.md @@ -16,11 +16,14 @@ Create a `Tokenizer`, to create `Doc` objects given unicode text. > ```python > # Construction 1 > from spacy.tokenizer import Tokenizer +> from spacy.lang.en import English +> nlp = English() > tokenizer = Tokenizer(nlp.vocab) > > # Construction 2 > from spacy.lang.en import English -> tokenizer = English().Defaults.create_tokenizer(nlp) +> nlp = English() +> tokenizer = nlp.Defaults.create_tokenizer(nlp) > ``` | Name | Type | Description | From 81c12640ab4f16ad2a9c718416f32409d0c457d2 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Sun, 16 Jun 2019 14:33:20 +0200 Subject: [PATCH 18/18] Auto-format [ci skip] --- website/docs/api/tokenizer.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/docs/api/tokenizer.md b/website/docs/api/tokenizer.md index 66ac315c6..5bc0df625 100644 --- a/website/docs/api/tokenizer.md +++ b/website/docs/api/tokenizer.md @@ -64,11 +64,11 @@ Tokenize a stream of texts. > pass > ``` -| Name | Type | Description | -| ------------ | ----- | -------------------------------------------------------- | -| `texts` | - | A sequence of unicode texts. | -| `batch_size` | int | The number of texts to accumulate in an internal buffer. Defaults to `1000`.| -| **YIELDS** | `Doc` | A sequence of Doc objects, in order. | +| Name | Type | Description | +| ------------ | ----- | ---------------------------------------------------------------------------- | +| `texts` | - | A sequence of unicode texts. | +| `batch_size` | int | The number of texts to accumulate in an internal buffer. Defaults to `1000`. | +| **YIELDS** | `Doc` | A sequence of Doc objects, in order. | ## Tokenizer.find_infix {#find_infix tag="method"}