From f6af3cf8d9650e56da4ec941b4dbff77d61e3e98 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Sat, 19 Oct 2019 18:13:25 +0200 Subject: [PATCH 1/6] Add 3.8 classifier [ci skip] --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index d0fa9daf6..f90ff7fc7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ classifiers = Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 Topic :: Scientific/Engineering [options] From 2c96a5e5b0bc388f206033e3e45f5045c2ef33e1 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Sat, 19 Oct 2019 23:18:09 +0200 Subject: [PATCH 2/6] Remove lemma attrs on BaseDefaults (#4468) --- spacy/language.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spacy/language.py b/spacy/language.py index e9f5d8a3e..330852741 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -110,10 +110,6 @@ class BaseDefaults(object): tag_map = dict(TAG_MAP) tokenizer_exceptions = {} stop_words = set() - lemma_rules = {} - lemma_exc = {} - lemma_index = {} - lemma_lookup = {} morph_rules = {} lex_attr_getters = LEX_ATTRS syntax_iterators = {} From 7772d5d3c55c0cfaa44a3c895fc3b9c1034c71d0 Mon Sep 17 00:00:00 2001 From: Pepe Berba Date: Sun, 20 Oct 2019 07:28:18 +0800 Subject: [PATCH 3/6] Update `vocab.get_vector` docs to include features on Fasttext ngram (#4464) * Update `vocab.get_vector` * Added contrib agreement --- .github/contributors/pberba.md | 106 +++++++++++++++++++++++++++++++++ spacy/vocab.pyx | 8 +++ website/docs/api/vocab.md | 14 +++-- 3 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 .github/contributors/pberba.md diff --git a/.github/contributors/pberba.md b/.github/contributors/pberba.md new file mode 100644 index 000000000..34feb3eea --- /dev/null +++ b/.github/contributors/pberba.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Pepe Berba | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-10-18 | +| GitHub username | pberba | +| Website (optional) | | \ No newline at end of file diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx index 67317a9ac..4a21537cb 100644 --- a/spacy/vocab.pyx +++ b/spacy/vocab.pyx @@ -336,7 +336,15 @@ cdef class Vocab: """Retrieve a vector for a word in the vocabulary. Words can be looked up by string or int ID. If no vectors data is loaded, ValueError is raised. + + If `minn` is defined, then the resulting vector uses Fasttext's + subword features by average over ngrams of `orth`. + orth (int / unicode): The hash value of a word, or its unicode string. + minn (int): Minimum n-gram length used for Fasttext's ngram computation. + Defaults to the length of `orth`. + maxn (int): Maximum n-gram length used for Fasttext's ngram computation. + Defaults to the length of `orth`. RETURNS (numpy.ndarray): A word vector. Size and shape determined by the `vocab.vectors` instance. Usually, a numpy ndarray of shape (300,) and dtype float32. diff --git a/website/docs/api/vocab.md b/website/docs/api/vocab.md index ea0c2d219..b94789921 100644 --- a/website/docs/api/vocab.md +++ b/website/docs/api/vocab.md @@ -168,16 +168,22 @@ cosines are calculated in minibatches, to reduce memory usage. Retrieve a vector for a word in the vocabulary. Words can be looked up by string or hash value. If no vectors data is loaded, a `ValueError` is raised. +If `minn` is defined, then the resulting vector uses Fasttext's +subword features by average over ngrams of `orth`. (Introduced in spaCy `v2.1`) + > #### Example > > ```python > nlp.vocab.get_vector("apple") +> nlp.vocab.get_vector("apple", minn=1, maxn=5) > ``` -| Name | Type | Description | -| ----------- | ---------------------------------------- | ----------------------------------------------------------------------------- | -| `orth` | int / unicode | The hash value of a word, or its unicode string. | -| **RETURNS** | `numpy.ndarray[ndim=1, dtype='float32']` | A word vector. Size and shape are determined by the `Vocab.vectors` instance. | +| Name | Type | Description | +| ----------- | ---------------------------------------- | ---------------------------------------------------------------------------------------------- | +| `orth` | int / unicode | The hash value of a word, or its unicode string. | +| `minn` | int | Minimum n-gram length used for Fasttext's ngram computation. Defaults to the length of `orth`. | +| `maxn` | int | Maximum n-gram length used for Fasttext's ngram computation. Defaults to the length of `orth`. | +| **RETURNS** | `numpy.ndarray[ndim=1, dtype='float32']` | A word vector. Size and shape are determined by the `Vocab.vectors` instance. | ## Vocab.set_vector {#set_vector tag="method" new="2"} From a98d1cd58e64b55b43db40ad674041b84c127295 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Sun, 20 Oct 2019 19:01:45 +0200 Subject: [PATCH 4/6] Update Thinc version and remove GPU ops --- requirements.txt | 2 +- setup.cfg | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 574ff950d..1a9eecb86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # Our libraries cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 -thinc>=7.1.1,<7.2.0 +thinc>=7.2.0,<7.3.0 blis>=0.4.0,<0.5.0 murmurhash>=0.28.0,<1.1.0 wasabi>=0.2.0,<1.1.0 diff --git a/setup.cfg b/setup.cfg index f90ff7fc7..3b2d0e77e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,14 +38,14 @@ setup_requires = cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 murmurhash>=0.28.0,<1.1.0 - thinc>=7.1.1,<7.2.0 + thinc>=7.2.0,<7.3.0 install_requires = setuptools numpy>=1.15.0 murmurhash>=0.28.0,<1.1.0 cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 - thinc>=7.1.1,<7.2.0 + thinc>=7.2.0,<7.3.0 blis>=0.4.0,<0.5.0 plac<1.0.0,>=0.9.6 requests>=2.13.0,<3.0.0 @@ -58,22 +58,16 @@ install_requires = lookups = spacy_lookups_data>=0.0.5<0.2.0 cuda = - thinc_gpu_ops>=0.0.1,<0.1.0 cupy>=5.0.0b4 cuda80 = - thinc_gpu_ops>=0.0.1,<0.1.0 cupy-cuda80>=5.0.0b4 cuda90 = - thinc_gpu_ops>=0.0.1,<0.1.0 cupy-cuda90>=5.0.0b4 cuda91 = - thinc_gpu_ops>=0.0.1,<0.1.0 cupy-cuda91>=5.0.0b4 cuda92 = - thinc_gpu_ops>=0.0.1,<0.1.0 cupy-cuda92>=5.0.0b4 cuda100 = - thinc_gpu_ops>=0.0.1,<0.1.0 cupy-cuda100>=5.0.0b4 # Language tokenizers with external dependencies ja = From d5d55312b2c5184ec96f8f073eab1b045b441fe4 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 21 Oct 2019 12:04:46 +0200 Subject: [PATCH 5/6] prevent division by zero in most_similar method (#4488) --- spacy/tests/regression/test_issue3001-3500.py | 8 ++++++++ spacy/vectors.pyx | 8 ++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/spacy/tests/regression/test_issue3001-3500.py b/spacy/tests/regression/test_issue3001-3500.py index def95ac73..8ed243051 100644 --- a/spacy/tests/regression/test_issue3001-3500.py +++ b/spacy/tests/regression/test_issue3001-3500.py @@ -15,6 +15,7 @@ from spacy.util import decaying import numpy import re +from spacy.vectors import Vectors from ..util import get_doc @@ -293,6 +294,13 @@ def test_issue3410(): list(phrasematcher.pipe(docs, n_threads=4)) +def test_issue3412(): + data = numpy.asarray([[0, 0, 0], [1, 2, 3], [9, 8, 7]], dtype="f") + vectors = Vectors(data=data) + keys, best_rows, scores = vectors.most_similar(numpy.asarray([[9, 8, 7], [0, 0, 0]], dtype="f")) + assert(best_rows[0] == 2) + + def test_issue3447(): sizes = decaying(10.0, 1.0, 0.5) size = next(sizes) diff --git a/spacy/vectors.pyx b/spacy/vectors.pyx index 6ad1202de..0f015521a 100644 --- a/spacy/vectors.pyx +++ b/spacy/vectors.pyx @@ -321,14 +321,18 @@ cdef class Vectors: """ xp = get_array_module(self.data) - vectors = self.data / xp.linalg.norm(self.data, axis=1, keepdims=True) + norms = xp.linalg.norm(self.data, axis=1, keepdims=True) + norms[norms == 0] = 1 + vectors = self.data / norms best_rows = xp.zeros((queries.shape[0], n), dtype='i') scores = xp.zeros((queries.shape[0], n), dtype='f') # Work in batches, to avoid memory problems. for i in range(0, queries.shape[0], batch_size): batch = queries[i : i+batch_size] - batch /= xp.linalg.norm(batch, axis=1, keepdims=True) + batch_norms = xp.linalg.norm(batch, axis=1, keepdims=True) + batch_norms[batch_norms == 0] = 1 + batch /= batch_norms # batch e.g. (1024, 300) # vectors e.g. (10000, 300) # sims e.g. (1024, 10000) From 3195a8f170741d9eef0d3abdc017bc1567f0147d Mon Sep 17 00:00:00 2001 From: adrianeboyd Date: Mon, 21 Oct 2019 12:17:30 +0200 Subject: [PATCH 6/6] Add Entity Linking to menu (#4489) --- website/docs/usage/linguistic-features.md | 1 + 1 file changed, 1 insertion(+) diff --git a/website/docs/usage/linguistic-features.md b/website/docs/usage/linguistic-features.md index e8326cdc9..039534fb7 100644 --- a/website/docs/usage/linguistic-features.md +++ b/website/docs/usage/linguistic-features.md @@ -5,6 +5,7 @@ menu: - ['POS Tagging', 'pos-tagging'] - ['Dependency Parse', 'dependency-parse'] - ['Named Entities', 'named-entities'] + - ['Entity Linking', 'entity-linking'] - ['Tokenization', 'tokenization'] - ['Merging & Splitting', 'retokenization'] - ['Sentence Segmentation', 'sbd']