From bc9557b21ffde564e39a98a0819fd6a63f47e11a Mon Sep 17 00:00:00 2001 From: Leif Uwe Vogelsang Date: Thu, 23 Mar 2017 11:10:22 +0100 Subject: [PATCH 01/29] Norwegian language basics --- spacy/nb/__init__.py | 26 +++++ spacy/nb/language_data.py | 28 +++++ spacy/nb/morph_rules.py | 67 ++++++++++++ spacy/nb/stop_words.py | 40 +++++++ spacy/nb/tokenizer_exceptions.py | 175 +++++++++++++++++++++++++++++++ 5 files changed, 336 insertions(+) create mode 100644 spacy/nb/__init__.py create mode 100644 spacy/nb/language_data.py create mode 100644 spacy/nb/morph_rules.py create mode 100644 spacy/nb/stop_words.py create mode 100644 spacy/nb/tokenizer_exceptions.py diff --git a/spacy/nb/__init__.py b/spacy/nb/__init__.py new file mode 100644 index 000000000..6c1aab979 --- /dev/null +++ b/spacy/nb/__init__.py @@ -0,0 +1,26 @@ +# encoding: utf8 +from __future__ import unicode_literals, print_function + +from os import path + +from ..language import Language +from ..attrs import LANG + + +# Import language-specific data +from .language_data import * + + +# create Language subclass +class NorwegianBokmal(Language): + lang = 'nb' # ISO code + + class Defaults(Language.Defaults): + lex_attr_getters = dict(Language.Defaults.lex_attr_getters) + lex_attr_getters[LANG] = lambda text: 'nb' + + + # override defaults + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + #tag_map = TAG_MAP + stop_words = STOP_WORDS \ No newline at end of file diff --git a/spacy/nb/language_data.py b/spacy/nb/language_data.py new file mode 100644 index 000000000..9383f3a62 --- /dev/null +++ b/spacy/nb/language_data.py @@ -0,0 +1,28 @@ +# encoding: utf8 +from __future__ import unicode_literals + + +# import base language data +from .. import language_data as base + + +# import util functions +from ..language_data import update_exc, strings_to_exc, expand_exc + +# import language-specific data from files +#from .tag_map import TAG_MAP +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS, ORTH_ONLY +from .morph_rules import MORPH_RULES + +TOKENIZER_EXCEPTIONS = dict(TOKENIZER_EXCEPTIONS) +#TAG_MAP = dict(TAG_MAP) +STOP_WORDS = set(STOP_WORDS) + +# customize tokenizer exceptions +update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(ORTH_ONLY)) +update_exc(TOKENIZER_EXCEPTIONS, expand_exc(TOKENIZER_EXCEPTIONS, "'", "’")) +update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(base.EMOTICONS)) + +# export +__all__ = ["TOKENIZER_EXCEPTIONS", "STOP_WORDS", "MORPH_RULES"] \ No newline at end of file diff --git a/spacy/nb/morph_rules.py b/spacy/nb/morph_rules.py new file mode 100644 index 000000000..1f9f1b84e --- /dev/null +++ b/spacy/nb/morph_rules.py @@ -0,0 +1,67 @@ +# encoding: utf8 +# norwegian bokmål +from __future__ import unicode_literals + +from ..symbols import * +from ..language_data import PRON_LEMMA + +# Used the table of pronouns at https://no.wiktionary.org/wiki/Tillegg:Pronomen_i_norsk + +MORPH_RULES = { + "PRP": { + "jeg": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Sing", "Case": "Nom"}, + "meg": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Sing", "Case": "Acc"}, + "du": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Two", "Number": "Sing", "Case": "Nom"}, + "deg": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Two", "Number": "Sing", "Case": "Acc"}, + "han": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Gender": "Masc", "Case": "Nom"}, + "ham": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Gender": "Masc", "Case": "Acc"}, + "han": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Gender": "Masc", "Case": "Acc"}, + "hun": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Gender": "Fem", "Case": "Nom"}, + "henne": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Gender": "Fem", "Case": "Acc"}, + "den": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Gender": "Neut"}, + "det": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Gender": "Neut"}, + "seg": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Reflex": "Yes"}, + "vi": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Plur", "Case": "Nom"}, + "oss": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Plur", "Case": "Acc"}, + "dere": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Two", "Number": "Plur", "Case": "Nom"}, + "de": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Plur", "Case": "Nom"}, + "dem": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Plur", "Case": "Acc"}, + "seg": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Plur", "Reflex": "Yes"}, + + "min": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Sing", "Poss": "Yes", "Gender": "Masc"}, + "mi": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Sing", "Poss": "Yes", "Gender": "Fem"}, + "mitt": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Sing", "Poss": "Yes", "Gender": "Neu"}, + "mine": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Plur", "Poss": "Yes"}, + "din": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Two", "Number": "Sing", "Poss": "Yes", "Gender": "Masc"}, + "di": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Two", "Number": "Sing", "Poss": "Yes", "Gender": "Fem"}, + "ditt": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Two", "Number": "Sing", "Poss": "Yes", "Gender": "Neu"}, + "dine": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Two", "Number": "Plur", "Poss": "Yes"}, + "hans": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Poss": "Yes", "Gender": "Masc"}, + "hennes": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Poss": "Yes", "Gender": "Fem"}, + "dens": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Poss": "Yes", "Gender": "Neu"}, + "dets": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Three", "Number": "Sing", "Poss": "Yes", "Gender": "Neu"}, + "vår": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Plur", "Poss": "Yes"}, + "vårt": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Plur", "Poss": "Yes"}, + "våre": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Plur", "Poss": "Yes", "Gender":"Neu"}, + "deres": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Plur", "Poss": "Yes", "Gender":"Neu", "Reflex":"Yes"}, + "sin": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Sing", "Poss": "Yes", "Gender":"Masc", "Reflex":"Yes"}, + "si": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Sing", "Poss": "Yes", "Gender":"Fem", "Reflex":"Yes"}, + "sitt": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Sing", "Poss": "Yes", "Gender":"Neu", "Reflex":"Yes"}, + "sine": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "One", "Number": "Plur", "Poss": "Yes", "Reflex":"Yes"}, + }, + + "VBZ": { + "er": {LEMMA: "be", "VerbForm": "Fin", "Person": "One", "Tense": "Pres", "Mood": "Ind"}, + "er": {LEMMA: "be", "VerbForm": "Fin", "Person": "Two", "Tense": "Pres", "Mood": "Ind"}, + "er": {LEMMA: "be", "VerbForm": "Fin", "Person": "Three", "Tense": "Pres", "Mood": "Ind"}, + }, + + "VBP": { + "er": {LEMMA: "be", "VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"} + }, + + "VBD": { + "var": {LEMMA: "be", "VerbForm": "Fin", "Tense": "Past", "Number": "Sing"}, + "vært": {LEMMA: "be", "VerbForm": "Fin", "Tense": "Past", "Number": "Plur"} + } +} \ No newline at end of file diff --git a/spacy/nb/stop_words.py b/spacy/nb/stop_words.py new file mode 100644 index 000000000..62d1a6028 --- /dev/null +++ b/spacy/nb/stop_words.py @@ -0,0 +1,40 @@ +# encoding: utf8 +from __future__ import unicode_literals + + +STOP_WORDS = set("""alle at av + +bare begge ble blei bli blir blitt både båe + +da de deg dei deim deira deires dem den denne der dere deres det dette di din disse ditt du dykk dykkar då + +eg ein eit eitt eller elles en enn er et ett etter + +for fordi fra før + +ha hadde han hans har hennar henne hennes her hjå ho hoe honom hoss hossen hun hva hvem hver hvilke hvilken hvis hvor hvordan hvorfor + +i ikke ikkje ikkje ingen ingi inkje inn inni + +ja jeg + +kan kom korleis korso kun kunne kva kvar kvarhelst kven kvi kvifor + +man mange me med medan meg meget mellom men mi min mine mitt mot mykje + +ned no noe noen noka noko nokon nokor nokre nå når + +og også om opp oss over + +på + +samme seg selv si sia sidan siden sin sine sitt sjøl skal skulle slik so som som somme somt så sånn + +til + +um upp ut uten + +var vart varte ved vere verte vi vil ville vore vors vort vår være være vært + +å +""".split()) \ No newline at end of file diff --git a/spacy/nb/tokenizer_exceptions.py b/spacy/nb/tokenizer_exceptions.py new file mode 100644 index 000000000..44fc76532 --- /dev/null +++ b/spacy/nb/tokenizer_exceptions.py @@ -0,0 +1,175 @@ +# encoding: utf8 +# Norwegian bokmaål +from __future__ import unicode_literals + +from ..symbols import * +from ..language_data import PRON_LEMMA + + +TOKENIZER_EXCEPTIONS = { + "jan.": [ + {ORTH: "jan.", LEMMA: "januar"} + ], + + "feb.": [ + {ORTH: "feb.", LEMMA: "februar"} + ], + + "jul.": [ + {ORTH: "jul.", LEMMA: "juli"} + ] +} + + +ORTH_ONLY = ["adm.dir.", + "a.m.", + "Aq.", + "b.c.", + "bl.a.", + "bla.", + "bm.", + "bto.", + "ca.", + "cand.mag.", + "c.c.", + "co.", + "d.d.", + "dept.", + "d.m.", + "dr.philos.", + "dvs.", + "d.y.", + "E. coli", + "eg.", + "ekskl.", + "e.Kr.", + "el.", + "e.l.", + "et.", + "etg.", + "ev.", + "evt.", + "f.", + "f.eks.", + "fhv.", + "fk.", + "f.Kr.", + "f.o.m.", + "foreg.", + "fork.", + "fv.", + "fvt.", + "g.", + "gt.", + "gl.", + "gno.", + "gnr.", + "grl.", + "hhv.", + "hoh.", + "hr.", + "h.r.adv.", + "ifb.", + "ifm.", + "iht.", + "inkl.", + "istf.", + "jf.", + "jr.", + "jun.", + "kfr.", + "kgl.res.", + "kl.", + "komm.", + "kst.", + "lø.", + "ma.", + "mag.art.", + "m.a.o.", + "md.", + "mfl.", + "mill.", + "min.", + "m.m.", + "mnd.", + "moh.", + "Mr.", + "muh.", + "mv.", + "mva.", + "ndf.", + "no.", + "nov.", + "nr.", + "nto.", + "nyno.", + "n.å.", + "o.a.", + "off.", + "ofl.", + "okt.", + "o.l.", + "on.", + "op.", + "osv.", + "ovf.", + "p.", + "p.a.", + "Pb.", + "pga.", + "ph.d.", + "pkt.", + "p.m.", + "pr.", + "pst.", + "p.t.", + "red.anm.", + "ref.", + "res.", + "res.kap.", + "resp.", + "rv.", + "s.", + "s.d.", + "sen.", + "sep.", + "siviling.", + "sms.", + "spm.", + "sr.", + "sst.", + "st.", + "stip.", + "stk.", + "st.meld.", + "st.prp.", + "stud.", + "s.u.", + "sv.", + "sø.", + "s.å.", + "såk.", + "temp.", + "ti.", + "tils.", + "tilsv.", + "tl;dr", + "tlf.", + "to.", + "t.o.m.", + "ult.", + "utg.", + "v.", + "vedk.", + "vedr.", + "vg.", + "vgs.", + "vha.", + "vit.ass.", + "vn.", + "vol.", + "vs.", + "vsa.", + "årg.", + "årh." +] \ No newline at end of file From 3796c668d9c0d33c472262e18669d680c4b2409f Mon Sep 17 00:00:00 2001 From: Leif Uwe Vogelsang Date: Tue, 28 Mar 2017 14:10:20 +0200 Subject: [PATCH 02/29] more norwegian --- spacy/__init__.py | 43 +++++++++++++++++++++-- spacy/nb/__init__.py | 7 ++-- spacy/nb/stop_words.py | 79 +++++++++++++++++++++++------------------- 3 files changed, 87 insertions(+), 42 deletions(-) diff --git a/spacy/__init__.py b/spacy/__init__.py index efd6c00c0..7b2769ecd 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -3,19 +3,52 @@ from __future__ import unicode_literals from . import util from .deprecated import resolve_model_name -from .cli import info +from .cli.info import info -from . import en, de, zh, es, it, hu, fr, pt, nl, sv, fi, bn, he + +from . import en, de, zh, es, it, hu, fr, pt, nl, sv, fi, bn, he, nb _languages = (en.English, de.German, es.Spanish, pt.Portuguese, fr.French, it.Italian, hu.Hungarian, zh.Chinese, nl.Dutch, sv.Swedish, - fi.Finnish, bn.Bengali, he.Hebrew) + fi.Finnish, bn.Bengali, he.Hebrew, nb.Norwegian) for _lang in _languages: util.set_lang_class(_lang.lang, _lang) +from . import en +from . import de +from . import zh +from . import es +from . import it +from . import hu +from . import fr +from . import pt +from . import nl +from . import sv +from . import fi +from . import bn +from . import nb + +from .about import * + + +set_lang_class(en.English.lang, en.English) +set_lang_class(de.German.lang, de.German) +set_lang_class(es.Spanish.lang, es.Spanish) +set_lang_class(pt.Portuguese.lang, pt.Portuguese) +set_lang_class(fr.French.lang, fr.French) +set_lang_class(it.Italian.lang, it.Italian) +set_lang_class(hu.Hungarian.lang, hu.Hungarian) +set_lang_class(zh.Chinese.lang, zh.Chinese) +set_lang_class(nl.Dutch.lang, nl.Dutch) +set_lang_class(sv.Swedish.lang, sv.Swedish) +set_lang_class(fi.Finnish.lang, fi.Finnish) +set_lang_class(bn.Bengali.lang, bn.Bengali) +set_lang_class(nb.Norwegian.lang, nb.Norwegian) +>>>>>>> more norwegian + def load(name, **overrides): if overrides.get('path') in (None, False, True): @@ -38,3 +71,7 @@ def load(name, **overrides): overrides['meta'] = meta overrides['path'] = model_path return cls(**overrides) + + +def info(name, markdown): + info(name, markdown) diff --git a/spacy/nb/__init__.py b/spacy/nb/__init__.py index 6c1aab979..6389356a7 100644 --- a/spacy/nb/__init__.py +++ b/spacy/nb/__init__.py @@ -15,12 +15,11 @@ from .language_data import * class NorwegianBokmal(Language): lang = 'nb' # ISO code - class Defaults(Language.Defaults): + class Defaults(Language.Defaults): lex_attr_getters = dict(Language.Defaults.lex_attr_getters) lex_attr_getters[LANG] = lambda text: 'nb' - - + # override defaults tokenizer_exceptions = TOKENIZER_EXCEPTIONS #tag_map = TAG_MAP - stop_words = STOP_WORDS \ No newline at end of file + stop_words = STOP_WORDS \ No newline at end of file diff --git a/spacy/nb/stop_words.py b/spacy/nb/stop_words.py index 62d1a6028..a4d10ad76 100644 --- a/spacy/nb/stop_words.py +++ b/spacy/nb/stop_words.py @@ -1,40 +1,49 @@ # encoding: utf8 from __future__ import unicode_literals - -STOP_WORDS = set("""alle at av - -bare begge ble blei bli blir blitt både båe - -da de deg dei deim deira deires dem den denne der dere deres det dette di din disse ditt du dykk dykkar då - -eg ein eit eitt eller elles en enn er et ett etter - -for fordi fra før - -ha hadde han hans har hennar henne hennes her hjå ho hoe honom hoss hossen hun hva hvem hver hvilke hvilken hvis hvor hvordan hvorfor - -i ikke ikkje ikkje ingen ingi inkje inn inni - + +STOP_WORDS = set(""" +alle allerede alt and andre annen annet at av + +bak bare bedre beste blant ble bli blir blitt bris by både + +da dag de del dem den denne der dermed det dette disse drept du + +eller en enn er et ett etter + +fem fikk fire fjor flere folk for fortsatt fotball fra fram frankrike fredag funnet få får fått før først første + +gang gi gikk gjennom gjorde gjort gjør gjøre god godt grunn gå går + +ha hadde ham han hans har hele helt henne hennes her hun hva hvor hvordan hvorfor + +i ifølge igjen ikke ingen inn + ja jeg - -kan kom korleis korso kun kunne kva kvar kvarhelst kven kvi kvifor - -man mange me med medan meg meget mellom men mi min mine mitt mot mykje - -ned no noe noen noka noko nokon nokor nokre nå når - -og også om opp oss over - -på - -samme seg selv si sia sidan siden sin sine sitt sjøl skal skulle slik so som som somme somt så sånn - -til - -um upp ut uten - -var vart varte ved vere verte vi vil ville vore vors vort vår være være vært - -å + +kamp kampen kan kl klart kom komme kommer kontakt kort kroner kunne kveld kvinner + +la laget land landet langt leder ligger like litt løpet lørdag + +man mandag mange mannen mars med meg mellom men mener menn mennesker mens mer millioner minutter mot msci mye må mål måtte + +ned neste noe noen nok norge norsk norske ntb ny nye nå når + +og også om onsdag opp opplyser oslo oss over + +personer plass poeng politidistrikt politiet president prosent på + +regjeringen runde rundt russland + +sa saken samme sammen samtidig satt se seg seks selv senere september ser sett siden sier sin sine siste sitt skal skriver skulle slik som sted stedet stor store står sverige svært så søndag + +ta tatt tid tidligere til tilbake tillegg tirsdag to tok torsdag tre tror tyskland + +under usa ut uten utenfor + +vant var ved veldig vi videre viktig vil ville viser vår være vært + +å år + +ønsker """.split()) \ No newline at end of file From ff900ffd7cf4b038b7091960dccf23a8bb905c93 Mon Sep 17 00:00:00 2001 From: luvogels Date: Sat, 8 Apr 2017 17:06:12 +0200 Subject: [PATCH 03/29] Update setup.py added nb --- setup.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index f7cd0ddcb..d8fbe9bb4 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ PACKAGES = [ 'spacy.fi', 'spacy.bn', 'spacy.he', + 'spacy.nb', 'spacy.en.lemmatizer', 'spacy.cli.converters', 'spacy.language_data', From e821c484895d42be9a90743ffd8d31a0469f841b Mon Sep 17 00:00:00 2001 From: Leif Uwe Vogelsang Date: Thu, 23 Mar 2017 11:10:22 +0100 Subject: [PATCH 04/29] Norwegian language basics --- spacy/nb/stop_words.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/spacy/nb/stop_words.py b/spacy/nb/stop_words.py index a4d10ad76..ef6f13717 100644 --- a/spacy/nb/stop_words.py +++ b/spacy/nb/stop_words.py @@ -1,6 +1,7 @@ # encoding: utf8 from __future__ import unicode_literals +<<<<<<< HEAD STOP_WORDS = set(""" alle allerede alt and andre annen annet at av @@ -46,4 +47,42 @@ vant var ved veldig vi videre viktig vil ville viser vår være vært å år ønsker +======= + +STOP_WORDS = set("""alle at av + +bare begge ble blei bli blir blitt både båe + +da de deg dei deim deira deires dem den denne der dere deres det dette di din disse ditt du dykk dykkar då + +eg ein eit eitt eller elles en enn er et ett etter + +for fordi fra før + +ha hadde han hans har hennar henne hennes her hjå ho hoe honom hoss hossen hun hva hvem hver hvilke hvilken hvis hvor hvordan hvorfor + +i ikke ikkje ikkje ingen ingi inkje inn inni + +ja jeg + +kan kom korleis korso kun kunne kva kvar kvarhelst kven kvi kvifor + +man mange me med medan meg meget mellom men mi min mine mitt mot mykje + +ned no noe noen noka noko nokon nokor nokre nå når + +og også om opp oss over + +på + +samme seg selv si sia sidan siden sin sine sitt sjøl skal skulle slik so som som somme somt så sånn + +til + +um upp ut uten + +var vart varte ved vere verte vi vil ville vore vors vort vår være være vært + +å +>>>>>>> Norwegian language basics """.split()) \ No newline at end of file From 538a8d6b129a86cf35c84bbce5137df7fca41f02 Mon Sep 17 00:00:00 2001 From: Leif Uwe Vogelsang Date: Mon, 10 Apr 2017 17:45:25 +0200 Subject: [PATCH 05/29] Resolved merge conflict by incorporating both suggestions. --- spacy/__init__.py | 5 ++++- spacy/nb/stop_words.py | 41 +---------------------------------------- 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/spacy/__init__.py b/spacy/__init__.py index 7b2769ecd..0e7e8603f 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals from . import util from .deprecated import resolve_model_name from .cli.info import info +<<<<<<< HEAD from . import en, de, zh, es, it, hu, fr, pt, nl, sv, fi, bn, he, nb @@ -29,6 +30,7 @@ from . import nl from . import sv from . import fi from . import bn +from . import he from . import nb from .about import * @@ -47,7 +49,8 @@ set_lang_class(sv.Swedish.lang, sv.Swedish) set_lang_class(fi.Finnish.lang, fi.Finnish) set_lang_class(bn.Bengali.lang, bn.Bengali) set_lang_class(nb.Norwegian.lang, nb.Norwegian) ->>>>>>> more norwegian +set_lang_class(he.Hebrew.lang, he.Hebrew) +set_lang_class(nb.Norwegian.lang, nb.Norwegian) def load(name, **overrides): diff --git a/spacy/nb/stop_words.py b/spacy/nb/stop_words.py index ef6f13717..56e0ef3bc 100644 --- a/spacy/nb/stop_words.py +++ b/spacy/nb/stop_words.py @@ -1,7 +1,5 @@ # encoding: utf8 from __future__ import unicode_literals - -<<<<<<< HEAD STOP_WORDS = set(""" alle allerede alt and andre annen annet at av @@ -47,42 +45,5 @@ vant var ved veldig vi videre viktig vil ville viser vår være vært å år ønsker -======= - -STOP_WORDS = set("""alle at av - -bare begge ble blei bli blir blitt både båe - -da de deg dei deim deira deires dem den denne der dere deres det dette di din disse ditt du dykk dykkar då - -eg ein eit eitt eller elles en enn er et ett etter - -for fordi fra før - -ha hadde han hans har hennar henne hennes her hjå ho hoe honom hoss hossen hun hva hvem hver hvilke hvilken hvis hvor hvordan hvorfor - -i ikke ikkje ikkje ingen ingi inkje inn inni - -ja jeg - -kan kom korleis korso kun kunne kva kvar kvarhelst kven kvi kvifor - -man mange me med medan meg meget mellom men mi min mine mitt mot mykje - -ned no noe noen noka noko nokon nokor nokre nå når - -og også om opp oss over - -på - -samme seg selv si sia sidan siden sin sine sitt sjøl skal skulle slik so som som somme somt så sånn - -til - -um upp ut uten - -var vart varte ved vere verte vi vil ville vore vors vort vår være være vært - -å ->>>>>>> Norwegian language basics + """.split()) \ No newline at end of file From 03abd0c8e698742f93909b760c57adddb040cfbd Mon Sep 17 00:00:00 2001 From: luvogels Date: Tue, 11 Apr 2017 18:29:09 +0200 Subject: [PATCH 06/29] Update __init__.py --- spacy/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spacy/__init__.py b/spacy/__init__.py index 0e7e8603f..19cc61c06 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -48,7 +48,6 @@ set_lang_class(nl.Dutch.lang, nl.Dutch) set_lang_class(sv.Swedish.lang, sv.Swedish) set_lang_class(fi.Finnish.lang, fi.Finnish) set_lang_class(bn.Bengali.lang, bn.Bengali) -set_lang_class(nb.Norwegian.lang, nb.Norwegian) set_lang_class(he.Hebrew.lang, he.Hebrew) set_lang_class(nb.Norwegian.lang, nb.Norwegian) From 55e8cade363c128d082507a4e39b06bdc4ca5cf4 Mon Sep 17 00:00:00 2001 From: luvogels Date: Tue, 11 Apr 2017 18:30:15 +0200 Subject: [PATCH 07/29] Update __init__.py --- spacy/nb/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/spacy/nb/__init__.py b/spacy/nb/__init__.py index 6389356a7..0392f6520 100644 --- a/spacy/nb/__init__.py +++ b/spacy/nb/__init__.py @@ -12,14 +12,14 @@ from .language_data import * # create Language subclass -class NorwegianBokmal(Language): +class Norwegian(Language): lang = 'nb' # ISO code class Defaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: 'nb' + lex_attr_getters = dict(Language.Defaults.lex_attr_getters) + lex_attr_getters[LANG] = lambda text: 'nb' - # override defaults - tokenizer_exceptions = TOKENIZER_EXCEPTIONS - #tag_map = TAG_MAP - stop_words = STOP_WORDS \ No newline at end of file + # override defaults + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + #tag_map = TAG_MAP + stop_words = STOP_WORDS From c7cec7e5e23f50ab4044364fc525ff48ab52da84 Mon Sep 17 00:00:00 2001 From: luvogels Date: Wed, 12 Apr 2017 17:59:55 +0200 Subject: [PATCH 08/29] Update __init__.py --- spacy/nb/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/spacy/nb/__init__.py b/spacy/nb/__init__.py index 0392f6520..7d4b471a2 100644 --- a/spacy/nb/__init__.py +++ b/spacy/nb/__init__.py @@ -15,11 +15,11 @@ from .language_data import * class Norwegian(Language): lang = 'nb' # ISO code - class Defaults(Language.Defaults): - lex_attr_getters = dict(Language.Defaults.lex_attr_getters) - lex_attr_getters[LANG] = lambda text: 'nb' + class Defaults(Language.Defaults): + lex_attr_getters = dict(Language.Defaults.lex_attr_getters) + lex_attr_getters[LANG] = lambda text: 'nb' - # override defaults - tokenizer_exceptions = TOKENIZER_EXCEPTIONS - #tag_map = TAG_MAP - stop_words = STOP_WORDS + # override defaults + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + #tag_map = TAG_MAP + stop_words = STOP_WORDS From cbfe4920bb88623803f7210dfa90e90d6cc8b08a Mon Sep 17 00:00:00 2001 From: luvogels Date: Wed, 26 Apr 2017 18:02:34 +0200 Subject: [PATCH 09/29] Added contributor agreement and pull request doc --- .github/Alpha_support_Norwegian bokmål.md | 20 ++++ .github/contributors/luvogels.md | 106 ++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 .github/Alpha_support_Norwegian bokmål.md create mode 100644 .github/contributors/luvogels.md diff --git a/.github/Alpha_support_Norwegian bokmål.md b/.github/Alpha_support_Norwegian bokmål.md new file mode 100644 index 000000000..0917810ab --- /dev/null +++ b/.github/Alpha_support_Norwegian bokmål.md @@ -0,0 +1,20 @@ + + +## Description + + +Added alpha support for Norwegian bokmål. + +## Types of changes + +- [ ] **Bug fix** (non-breaking change fixing an issue) +- [x] **New feature** (non-breaking change adding functionality to spaCy) +- [ ] **Breaking change** (fix or feature causing change to spaCy's existing functionality) +- [ ] **Documentation** (addition to documentation of spaCy) + +## Checklist: + +- [ ] My change requires a change to spaCy's documentation. +- [ ] I have updated the documentation accordingly. +- [ ] I have added tests to cover my changes. +- [ ] All new and existing tests passed. diff --git a/.github/contributors/luvogels.md b/.github/contributors/luvogels.md new file mode 100644 index 000000000..c915d48bf --- /dev/null +++ b/.github/contributors/luvogels.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Shuvanon Razik | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 3/12/2017 | +| GitHub username | shuvanon | +| Website (optional) | | From 460094bf092714216273e46c33f2dde8b3a73a25 Mon Sep 17 00:00:00 2001 From: Leif Uwe Vogelsang Date: Wed, 26 Apr 2017 18:27:55 +0200 Subject: [PATCH 10/29] Update __init__.py --- spacy/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/spacy/__init__.py b/spacy/__init__.py index 19cc61c06..ce68dfc1a 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -4,8 +4,6 @@ from __future__ import unicode_literals from . import util from .deprecated import resolve_model_name from .cli.info import info -<<<<<<< HEAD - from . import en, de, zh, es, it, hu, fr, pt, nl, sv, fi, bn, he, nb From 8de59ce3b96a1a5d94e8bf72963e6d632b96564f Mon Sep 17 00:00:00 2001 From: luvogels Date: Wed, 26 Apr 2017 19:10:18 +0200 Subject: [PATCH 11/29] Added tokenizer tests --- spacy/__init__.py | 40 +------------------------------- spacy/tests/nb/__init__.py | 0 spacy/tests/nb/test_tokenizer.py | 17 ++++++++++++++ 3 files changed, 18 insertions(+), 39 deletions(-) create mode 100644 spacy/tests/nb/__init__.py create mode 100644 spacy/tests/nb/test_tokenizer.py diff --git a/spacy/__init__.py b/spacy/__init__.py index 19cc61c06..9bbbd8f3a 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals from . import util from .deprecated import resolve_model_name from .cli.info import info -<<<<<<< HEAD from . import en, de, zh, es, it, hu, fr, pt, nl, sv, fi, bn, he, nb @@ -18,39 +17,6 @@ _languages = (en.English, de.German, es.Spanish, pt.Portuguese, fr.French, for _lang in _languages: util.set_lang_class(_lang.lang, _lang) -from . import en -from . import de -from . import zh -from . import es -from . import it -from . import hu -from . import fr -from . import pt -from . import nl -from . import sv -from . import fi -from . import bn -from . import he -from . import nb - -from .about import * - - -set_lang_class(en.English.lang, en.English) -set_lang_class(de.German.lang, de.German) -set_lang_class(es.Spanish.lang, es.Spanish) -set_lang_class(pt.Portuguese.lang, pt.Portuguese) -set_lang_class(fr.French.lang, fr.French) -set_lang_class(it.Italian.lang, it.Italian) -set_lang_class(hu.Hungarian.lang, hu.Hungarian) -set_lang_class(zh.Chinese.lang, zh.Chinese) -set_lang_class(nl.Dutch.lang, nl.Dutch) -set_lang_class(sv.Swedish.lang, sv.Swedish) -set_lang_class(fi.Finnish.lang, fi.Finnish) -set_lang_class(bn.Bengali.lang, bn.Bengali) -set_lang_class(he.Hebrew.lang, he.Hebrew) -set_lang_class(nb.Norwegian.lang, nb.Norwegian) - def load(name, **overrides): if overrides.get('path') in (None, False, True): @@ -72,8 +38,4 @@ def load(name, **overrides): cls = util.get_lang_class(lang) overrides['meta'] = meta overrides['path'] = model_path - return cls(**overrides) - - -def info(name, markdown): - info(name, markdown) + return cls(**overrides) \ No newline at end of file diff --git a/spacy/tests/nb/__init__.py b/spacy/tests/nb/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/spacy/tests/nb/test_tokenizer.py b/spacy/tests/nb/test_tokenizer.py new file mode 100644 index 000000000..b55901339 --- /dev/null +++ b/spacy/tests/nb/test_tokenizer.py @@ -0,0 +1,17 @@ +# coding: utf8 +from __future__ import unicode_literals + +import pytest + + +NB_TOKEN_EXCEPTION_TESTS = [ + ('Smørsausen brukes bl.a. til fisk', ['Smørsausen', 'brukes', 'bl.a.', 'til', 'fisk']), + ('Jeg kommer først kl. 13 pga. diverse forsinkelser', ['Jeg', 'kommer', 'først', 'kl.', '13', 'pga.', 'diverse', 'forsinkelser']) +] + + +@pytest.mark.parametrize('text,expected_tokens', NB_TOKEN_EXCEPTION_TESTS) +def test_tokenizer_handles_exception_cases(nb_tokenizer, text, expected_tokens): + tokens = nb_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list From 4eacd72bc365763810a430cbafc1ef5ec4332dcc Mon Sep 17 00:00:00 2001 From: ines Date: Wed, 26 Apr 2017 20:49:48 +0200 Subject: [PATCH 12/29] Move list of models to own file --- website/docs/usage/_models-list.jade | 27 +++++++++++++++++++++++++++ website/docs/usage/models.jade | 23 +---------------------- 2 files changed, 28 insertions(+), 22 deletions(-) create mode 100644 website/docs/usage/_models-list.jade diff --git a/website/docs/usage/_models-list.jade b/website/docs/usage/_models-list.jade new file mode 100644 index 000000000..866dc1c42 --- /dev/null +++ b/website/docs/usage/_models-list.jade @@ -0,0 +1,27 @@ +//- 💫 DOCS > USAGE > MODELS LIST + +include ../../_includes/_mixins + +p + | Model differences are mostly statistical. In general, we do expect larger + | models to be "better" and more accurate overall. Ultimately, it depends on + | your use case and requirements, and we recommend starting with the default + | models (marked with a star below). + ++aside + | Models are now available as #[code .tar.gz] archives #[+a(gh("spacy-models")) from GitHub], + | attached to individual releases. They can be downloaded and loaded manually, + | or using spaCy's #[code download] and #[code link] commands. All models + | follow the naming convention of #[code [language]_[type]_[genre]_[size]]. + | #[br]#[br] + + +button(gh("spacy-models"), true, "primary").u-text-tag + | View model releases + ++table(["Name", "Language", "Voc", "Dep", "Ent", "Vec", "Size", "License"]) + +model-row("en_core_web_sm", "English", [1, 1, 1, 1], "50 MB", "CC BY-SA", true) + +model-row("en_core_web_md", "English", [1, 1, 1, 1], "1 GB", "CC BY-SA") + +model-row("en_depent_web_md", "English", [1, 1, 1, 0], "328 MB", "CC BY-SA") + +model-row("en_vectors_glove_md", "English", [0, 0, 0, 1], "727 MB", "CC BY-SA") + +model-row("de_core_news_md", "German", [1, 1, 1, 1], "645 MB", "CC BY-SA", true, true) + +model-row("fr_depvec_web_lg", "French", [1, 1, 0, 1], "1.33 GB", "CC BY-NC", true, true) diff --git a/website/docs/usage/models.jade b/website/docs/usage/models.jade index 69142b351..b25b462d8 100644 --- a/website/docs/usage/models.jade +++ b/website/docs/usage/models.jade @@ -33,28 +33,7 @@ p +h(2, "available") Available models -p - | Model differences are mostly statistical. In general, we do expect larger - | models to be "better" and more accurate overall. Ultimately, it depends on - | your use case and requirements, and we recommend starting with the default - | models (marked with a star below). - -+aside - | Models are now available as #[code .tar.gz] archives #[+a(gh("spacy-models")) from GitHub], - | attached to individual releases. They can be downloaded and loaded manually, - | or using spaCy's #[code download] and #[code link] commands. All models - | follow the naming convention of #[code [language]_[type]_[genre]_[size]]. - | #[br]#[br] - - +button(gh("spacy-models"), true, "primary").u-text-tag - | View model releases - -+table(["Name", "Language", "Voc", "Dep", "Ent", "Vec", "Size", "License"]) - +model-row("en_core_web_sm", "English", [1, 1, 1, 1], "50 MB", "CC BY-SA", true) - +model-row("en_core_web_md", "English", [1, 1, 1, 1], "1 GB", "CC BY-SA") - +model-row("en_depent_web_md", "English", [1, 1, 1, 0], "328 MB", "CC BY-SA") - +model-row("en_vectors_glove_md", "English", [0, 0, 0, 1], "727 MB", "CC BY-SA") - +model-row("de_core_news_md", "German", [1, 1, 1, 1], "645 MB", "CC BY-SA", true, true) +include _models-list +h(2, "download") Downloading models From 375edf0bb566062f0a86b582ee9f98b6f57f1c5a Mon Sep 17 00:00:00 2001 From: ines Date: Wed, 26 Apr 2017 20:50:02 +0200 Subject: [PATCH 13/29] Add list of models and include French --- website/docs/api/language-models.jade | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/website/docs/api/language-models.jade b/website/docs/api/language-models.jade index 083a501be..6a5ed9a9c 100644 --- a/website/docs/api/language-models.jade +++ b/website/docs/api/language-models.jade @@ -7,6 +7,7 @@ p spaCy currently supports the following languages and capabilities: +aside-code("Download language models", "bash"). python -m spacy download en python -m spacy download de + python -m spacy download fr +table([ "Language", "Token", "SBD", "Lemma", "POS", "NER", "Dep", "Vector", "Sentiment"]) +row @@ -19,6 +20,14 @@ p spaCy currently supports the following languages and capabilities: each icon in [ "pro", "pro", "con", "pro", "pro", "pro", "pro", "con" ] +cell.u-text-center #[+procon(icon)] + +row + +cell French #[code de] + each icon in [ "pro", "pro", "con", "pro", "con", "pro", "pro", "con" ] + +cell.u-text-center #[+procon(icon)] + ++h(2, "available") Available models + +include ../usage/_models-list +h(2, "alpha-support") Alpha support @@ -27,7 +36,7 @@ p | the existing language data and extending the tokenization patterns. +table([ "Language", "Source" ]) - each language, code in { zh: "Chinese", es: "Spanish", it: "Italian", fr: "French", pt: "Portuguese", nl: "Dutch", sv: "Swedish", fi: "Finnish", hu: "Hungarian", bn: "Bengali", he: "Hebrew" } + each language, code in { zh: "Chinese", es: "Spanish", it: "Italian", pt: "Portuguese", nl: "Dutch", sv: "Swedish", fi: "Finnish", hu: "Hungarian", bn: "Bengali", he: "Hebrew" } +row +cell #{language} #[code=code] +cell From 05bcd61fcff932194e1f7af78c003155f0f58727 Mon Sep 17 00:00:00 2001 From: ines Date: Wed, 26 Apr 2017 20:51:38 +0200 Subject: [PATCH 14/29] Update README.rst --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 3979b2e37..1045e919d 100644 --- a/README.rst +++ b/README.rst @@ -320,6 +320,7 @@ and ``--model`` are optional and enable additional tests: =========== ============== =========== Version Date Description =========== ============== =========== +`v1.8.2`_ ``2017-04-26`` French model and small improvements `v1.8.1`_ ``2017-04-23`` Saving, loading and training bug fixes `v1.8.0`_ ``2017-04-16`` Better NER training, saving and loading `v1.7.5`_ ``2017-04-07`` Bug fixes and new CLI commands @@ -352,6 +353,7 @@ Version Date Description `v0.93`_ ``2015-09-22`` Bug fixes to word vectors =========== ============== =========== +.. _v1.8.2: https://github.com/explosion/spaCy/releases/tag/v1.8.2 .. _v1.8.1: https://github.com/explosion/spaCy/releases/tag/v1.8.1 .. _v1.8.0: https://github.com/explosion/spaCy/releases/tag/v1.8.0 .. _v1.7.5: https://github.com/explosion/spaCy/releases/tag/v1.7.5 From 100846bed332d9f1baff0426adf7e8426ae21ef9 Mon Sep 17 00:00:00 2001 From: ines Date: Wed, 26 Apr 2017 21:40:17 +0200 Subject: [PATCH 15/29] Fix typo in model list --- website/docs/usage/_models-list.jade | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/usage/_models-list.jade b/website/docs/usage/_models-list.jade index 866dc1c42..942de28c4 100644 --- a/website/docs/usage/_models-list.jade +++ b/website/docs/usage/_models-list.jade @@ -22,6 +22,6 @@ p +model-row("en_core_web_sm", "English", [1, 1, 1, 1], "50 MB", "CC BY-SA", true) +model-row("en_core_web_md", "English", [1, 1, 1, 1], "1 GB", "CC BY-SA") +model-row("en_depent_web_md", "English", [1, 1, 1, 0], "328 MB", "CC BY-SA") - +model-row("en_vectors_glove_md", "English", [0, 0, 0, 1], "727 MB", "CC BY-SA") + +model-row("en_vectors_glove_md", "English", [1, 0, 0, 1], "727 MB", "CC BY-SA") +model-row("de_core_news_md", "German", [1, 1, 1, 1], "645 MB", "CC BY-SA", true, true) +model-row("fr_depvec_web_lg", "French", [1, 1, 0, 1], "1.33 GB", "CC BY-NC", true, true) From d12a0b643146e58b6b76dbb3514dc2aa62eb0f41 Mon Sep 17 00:00:00 2001 From: luvogels Date: Wed, 26 Apr 2017 23:21:41 +0200 Subject: [PATCH 16/29] Hooked up tokenizer tests --- spacy/tests/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index 37d3180d0..b8ada1d9a 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -13,6 +13,8 @@ from ..hu import Hungarian from ..fi import Finnish from ..bn import Bengali from ..he import Hebrew +from ..nb import Norwegian + from ..tokens import Doc from ..strings import StringStore @@ -26,7 +28,7 @@ import pytest LANGUAGES = [English, German, Spanish, Italian, French, Portuguese, Dutch, - Swedish, Hungarian, Finnish, Bengali] + Swedish, Hungarian, Finnish, Bengali, Norwegian] @pytest.fixture(params=LANGUAGES) @@ -88,6 +90,9 @@ def bn_tokenizer(): def he_tokenizer(): return Hebrew.Defaults.create_tokenizer() +@pytest.fixture +def nb_tokenizer(): + return Norwegian.Defaults.create_tokenizer() @pytest.fixture def stringstore(): From e136c51393e8f87b9bf9a4bbf1ea69da604e85a2 Mon Sep 17 00:00:00 2001 From: Leif Uwe Vogelsang Date: Wed, 26 Apr 2017 23:24:11 +0200 Subject: [PATCH 17/29] =?UTF-8?q?Update=20Alpha=5Fsupport=5FNorwegian=20bo?= =?UTF-8?q?km=C3=A5l.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/Alpha_support_Norwegian bokmål.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/Alpha_support_Norwegian bokmål.md b/.github/Alpha_support_Norwegian bokmål.md index 0917810ab..a3752db42 100644 --- a/.github/Alpha_support_Norwegian bokmål.md +++ b/.github/Alpha_support_Norwegian bokmål.md @@ -16,5 +16,5 @@ Added alpha support for Norwegian bokmål. - [ ] My change requires a change to spaCy's documentation. - [ ] I have updated the documentation accordingly. -- [ ] I have added tests to cover my changes. -- [ ] All new and existing tests passed. +- [x] I have added tests to cover my changes. +- [x] All new and existing tests passed. From 13ce4c96b1bfe2f484423e02afb7a49463b650a0 Mon Sep 17 00:00:00 2001 From: Leif Uwe Vogelsang Date: Thu, 27 Apr 2017 10:42:07 +0200 Subject: [PATCH 18/29] Update luvogels.md --- .github/contributors/luvogels.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/contributors/luvogels.md b/.github/contributors/luvogels.md index c915d48bf..c967c1cd2 100644 --- a/.github/contributors/luvogels.md +++ b/.github/contributors/luvogels.md @@ -98,9 +98,9 @@ mark both statements: | Field | Entry | |------------------------------- | -------------------- | -| Name | Shuvanon Razik | +| Name | Leif Uwe Vogelsang | | Company name (if applicable) | | | Title or role (if applicable) | | -| Date | 3/12/2017 | -| GitHub username | shuvanon | +| Date | 4/27/2017 | +| GitHub username | luvogels | | Website (optional) | | From 03d2b0cc058803b373c7fadd1c95e57ad24bf6dc Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Thu, 27 Apr 2017 11:14:26 +0200 Subject: [PATCH 19/29] Add newline --- spacy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/__init__.py b/spacy/__init__.py index 3106b49c5..f71d3addd 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -37,4 +37,4 @@ def load(name, **overrides): cls = util.get_lang_class(lang) overrides['meta'] = meta overrides['path'] = model_path - return cls(**overrides) \ No newline at end of file + return cls(**overrides) From ccf13ecc21ffd39ef226044c9218ccbca2c221ec Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Thu, 27 Apr 2017 11:14:42 +0200 Subject: [PATCH 20/29] Add newline --- spacy/nb/language_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/nb/language_data.py b/spacy/nb/language_data.py index 9383f3a62..248b09fc7 100644 --- a/spacy/nb/language_data.py +++ b/spacy/nb/language_data.py @@ -25,4 +25,4 @@ update_exc(TOKENIZER_EXCEPTIONS, expand_exc(TOKENIZER_EXCEPTIONS, "'", "’")) update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(base.EMOTICONS)) # export -__all__ = ["TOKENIZER_EXCEPTIONS", "STOP_WORDS", "MORPH_RULES"] \ No newline at end of file +__all__ = ["TOKENIZER_EXCEPTIONS", "STOP_WORDS", "MORPH_RULES"] From 4cd9269aef8a4a16331991c013f45e6d3bd1ddd9 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Thu, 27 Apr 2017 11:15:04 +0200 Subject: [PATCH 21/29] Add newline --- spacy/nb/morph_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/nb/morph_rules.py b/spacy/nb/morph_rules.py index 1f9f1b84e..38498513b 100644 --- a/spacy/nb/morph_rules.py +++ b/spacy/nb/morph_rules.py @@ -64,4 +64,4 @@ MORPH_RULES = { "var": {LEMMA: "be", "VerbForm": "Fin", "Tense": "Past", "Number": "Sing"}, "vært": {LEMMA: "be", "VerbForm": "Fin", "Tense": "Past", "Number": "Plur"} } -} \ No newline at end of file +} From 5942adccc278a710944d14db134f2a40cc8ed841 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Thu, 27 Apr 2017 11:15:19 +0200 Subject: [PATCH 22/29] Add newline --- spacy/nb/stop_words.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/nb/stop_words.py b/spacy/nb/stop_words.py index 56e0ef3bc..721ba2e47 100644 --- a/spacy/nb/stop_words.py +++ b/spacy/nb/stop_words.py @@ -46,4 +46,4 @@ vant var ved veldig vi videre viktig vil ville viser vår være vært ønsker -""".split()) \ No newline at end of file +""".split()) From c9e592ae6c4d68c3d0bc02539ac7de500688bd74 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Thu, 27 Apr 2017 11:15:41 +0200 Subject: [PATCH 23/29] Add newline --- spacy/nb/tokenizer_exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/nb/tokenizer_exceptions.py b/spacy/nb/tokenizer_exceptions.py index 44fc76532..ea7658c87 100644 --- a/spacy/nb/tokenizer_exceptions.py +++ b/spacy/nb/tokenizer_exceptions.py @@ -172,4 +172,4 @@ ORTH_ONLY = ["adm.dir.", "vsa.", "årg.", "årh." -] \ No newline at end of file +] From 6930ed719d6b28ed95d8502926be70f7ae8384f3 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Thu, 27 Apr 2017 11:17:06 +0200 Subject: [PATCH 24/29] Update CONTRIBUTORS.md --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index ce4ca62db..69a562e48 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -29,6 +29,7 @@ This is a list of everyone who has made significant contributions to spaCy, in a * Juan Miguel Cejuela, [@juanmirocks](https://github.com/juanmirocks) * Kendrick Tan, [@kendricktan](https://github.com/kendricktan) * Kyle P. Johnson, [@kylepjohnson](https://github.com/kylepjohnson) +* Leif Uwe Vogelsang, [@luvogels](https://github.com/luvogels) * Liling Tan, [@alvations](https://github.com/alvations) * Magnus Burton, [@magnusburton](https://github.com/magnusburton) * Mark Amery, [@ExplodingCabbage](https://github.com/ExplodingCabbage) From bc88f9865e574d842e45b8d27fe89bca16b8763d Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Thu, 27 Apr 2017 11:17:30 +0200 Subject: [PATCH 25/29] Remove file (already covered in PR) --- .github/Alpha_support_Norwegian bokmål.md | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 .github/Alpha_support_Norwegian bokmål.md diff --git a/.github/Alpha_support_Norwegian bokmål.md b/.github/Alpha_support_Norwegian bokmål.md deleted file mode 100644 index a3752db42..000000000 --- a/.github/Alpha_support_Norwegian bokmål.md +++ /dev/null @@ -1,20 +0,0 @@ - - -## Description - - -Added alpha support for Norwegian bokmål. - -## Types of changes - -- [ ] **Bug fix** (non-breaking change fixing an issue) -- [x] **New feature** (non-breaking change adding functionality to spaCy) -- [ ] **Breaking change** (fix or feature causing change to spaCy's existing functionality) -- [ ] **Documentation** (addition to documentation of spaCy) - -## Checklist: - -- [ ] My change requires a change to spaCy's documentation. -- [ ] I have updated the documentation accordingly. -- [x] I have added tests to cover my changes. -- [x] All new and existing tests passed. From 2f918e3004879b430b52afa501936b7a255f37aa Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Thu, 27 Apr 2017 11:18:41 +0200 Subject: [PATCH 26/29] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 1045e919d..baf0654e6 100644 --- a/README.rst +++ b/README.rst @@ -5,8 +5,8 @@ spaCy is a library for advanced natural language processing in Python and Cython. spaCy is built on the very latest research, but it isn't researchware. It was designed from day one to be used in real products. spaCy currently supports English and German, as well as tokenization for Chinese, Spanish, Italian, French, -Portuguese, Dutch, Swedish, Finnish, Hungarian, Bengali and Hebrew. It's commercial -open-source software, released under the MIT license. +Portuguese, Dutch, Swedish, Finnish, Norwegian, Hungarian, Bengali and Hebrew. It's +commercial open-source software, released under the MIT license. 📊 **Help us improve the library!** `Take the spaCy user survey `_. From 034ec5710b01feed0ac782dc975d0eecb31990c6 Mon Sep 17 00:00:00 2001 From: ines Date: Thu, 27 Apr 2017 11:24:18 +0200 Subject: [PATCH 27/29] Fix typo and add Norwegian to alpha languages --- website/docs/api/language-models.jade | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/api/language-models.jade b/website/docs/api/language-models.jade index 6a5ed9a9c..a2ad9b9eb 100644 --- a/website/docs/api/language-models.jade +++ b/website/docs/api/language-models.jade @@ -21,7 +21,7 @@ p spaCy currently supports the following languages and capabilities: +cell.u-text-center #[+procon(icon)] +row - +cell French #[code de] + +cell French #[code fr] each icon in [ "pro", "pro", "con", "pro", "con", "pro", "pro", "con" ] +cell.u-text-center #[+procon(icon)] @@ -36,7 +36,7 @@ p | the existing language data and extending the tokenization patterns. +table([ "Language", "Source" ]) - each language, code in { zh: "Chinese", es: "Spanish", it: "Italian", pt: "Portuguese", nl: "Dutch", sv: "Swedish", fi: "Finnish", hu: "Hungarian", bn: "Bengali", he: "Hebrew" } + each language, code in { zh: "Chinese", es: "Spanish", it: "Italian", pt: "Portuguese", nl: "Dutch", sv: "Swedish", fi: "Finnish", nb: "Norwegian Bokmål", hu: "Hungarian", bn: "Bengali", he: "Hebrew" } +row +cell #{language} #[code=code] +cell From 7a894c9ef0e693ef78c3979624fa2fec4238d14c Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Thu, 27 Apr 2017 11:25:30 +0200 Subject: [PATCH 28/29] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index baf0654e6..9b8438ce8 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ spaCy: Industrial-strength NLP spaCy is a library for advanced natural language processing in Python and Cython. spaCy is built on the very latest research, but it isn't researchware. It was designed from day one to be used in real products. spaCy currently supports -English and German, as well as tokenization for Chinese, Spanish, Italian, French, +English, German and French, as well as tokenization for Chinese, Spanish, Italian, Portuguese, Dutch, Swedish, Finnish, Norwegian, Hungarian, Bengali and Hebrew. It's commercial open-source software, released under the MIT license. From 5aa49971f9693f6b26379c9effcae278a044226a Mon Sep 17 00:00:00 2001 From: ines Date: Thu, 27 Apr 2017 12:08:45 +0200 Subject: [PATCH 29/29] Add French example to models docs --- website/docs/usage/models.jade | 1 + 1 file changed, 1 insertion(+) diff --git a/website/docs/usage/models.jade b/website/docs/usage/models.jade index b25b462d8..9bb75ba9a 100644 --- a/website/docs/usage/models.jade +++ b/website/docs/usage/models.jade @@ -61,6 +61,7 @@ p # out-of-the-box: download best-matching default model python -m spacy download en python -m spacy download de + python -m spacy download fr # download best-matching version of specific model for your spaCy installation python -m spacy download en_core_web_md