mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-16 06:37:04 +03:00
703 lines
20 KiB
Plaintext
703 lines
20 KiB
Plaintext
//- 💫 DOCS > API > LANGUAGE
|
|
|
|
include ../_includes/_mixins
|
|
|
|
p
|
|
| Usually you'll load this once per process as #[code nlp] and pass the
|
|
| instance around your application. The #[code Language] class is created
|
|
| when you call #[+api("spacy#load") #[code spacy.load()]] and contains
|
|
| the shared vocabulary and #[+a("/usage/adding-languages") language data],
|
|
| optional model data loaded from a #[+a("/models") model package] or
|
|
| a path, and a #[+a("/usage/processing-pipelines") processing pipeline]
|
|
| containing components like the tagger or parser that are called on a
|
|
| document in order. You can also add your own processing pipeline
|
|
| components that take a #[code Doc] object, modify it and return it.
|
|
|
|
+h(2, "init") Language.__init__
|
|
+tag method
|
|
|
|
p Initialise a #[code Language] object.
|
|
|
|
+aside-code("Example").
|
|
from spacy.vocab import Vocab
|
|
from spacy.language import Language
|
|
nlp = Language(Vocab())
|
|
|
|
from spacy.lang.en import English
|
|
nlp = English()
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code vocab]
|
|
+cell #[code Vocab]
|
|
+cell
|
|
| A #[code Vocab] object. If #[code True], a vocab is created via
|
|
| #[code Language.Defaults.create_vocab].
|
|
|
|
+row
|
|
+cell #[code make_doc]
|
|
+cell callable
|
|
+cell
|
|
| A function that takes text and returns a #[code Doc] object.
|
|
| Usually a #[code Tokenizer].
|
|
|
|
+row
|
|
+cell #[code meta]
|
|
+cell dict
|
|
+cell
|
|
| Custom meta data for the #[code Language] class. Is written to by
|
|
| models to add model meta data.
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell #[code Language]
|
|
+cell The newly constructed object.
|
|
|
|
+h(2, "call") Language.__call__
|
|
+tag method
|
|
|
|
p
|
|
| Apply the pipeline to some text. The text can span multiple sentences,
|
|
| and can contain arbtrary whitespace. Alignment into the original string
|
|
| is preserved.
|
|
|
|
+aside-code("Example").
|
|
doc = nlp(u'An example sentence. Another sentence.')
|
|
assert (doc[0].text, doc[0].head.tag_) == ('An', 'NN')
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code text]
|
|
+cell unicode
|
|
+cell The text to be processed.
|
|
|
|
+row
|
|
+cell #[code disable]
|
|
+cell list
|
|
+cell
|
|
| Names of pipeline components to
|
|
| #[+a("/usage/processing-pipelines#disabling") disable].
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell #[code Doc]
|
|
+cell A container for accessing the annotations.
|
|
|
|
+infobox("Changed in v2.0", "⚠️")
|
|
| Pipeline components to prevent from being loaded can now be added as
|
|
| a list to #[code disable], instead of specifying one keyword argument
|
|
| per component.
|
|
|
|
+code-wrapper
|
|
+code-new doc = nlp(u"I don't want parsed", disable=['parser'])
|
|
+code-old doc = nlp(u"I don't want parsed", parse=False)
|
|
|
|
+h(2, "pipe") Language.pipe
|
|
+tag method
|
|
|
|
p
|
|
| Process texts as a stream, and yield #[code Doc] objects in order.
|
|
| Supports GIL-free multi-threading.
|
|
|
|
+infobox("Important note for spaCy v2.0.x", "⚠️")
|
|
| By default, multiple threads will be launched for matrix multiplication,
|
|
| which may be inefficient on multi-core machines. Setting
|
|
| #[code OPENBLAS_NUM_THREADS=1] should fix this problem. spaCy v2.1.x
|
|
| will be switching to single-thread by default.
|
|
|
|
+aside-code("Example").
|
|
texts = [u'One document.', u'...', u'Lots of documents']
|
|
for doc in nlp.pipe(texts, batch_size=50, n_threads=4):
|
|
assert doc.is_parsed
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code texts]
|
|
+cell -
|
|
+cell A sequence of unicode objects.
|
|
|
|
+row
|
|
+cell #[code as_tuples]
|
|
+cell bool
|
|
+cell
|
|
| If set to #[code True], inputs should be a sequence of
|
|
| #[code (text, context)] tuples. Output will then be a sequence of
|
|
| #[code (doc, context)] tuples. Defaults to #[code False].
|
|
|
|
+row
|
|
+cell #[code n_threads]
|
|
+cell int
|
|
+cell
|
|
| The number of worker threads to use. If #[code -1], OpenMP will
|
|
| decide how many to use at run time. Default is #[code 2].
|
|
|
|
+row
|
|
+cell #[code batch_size]
|
|
+cell int
|
|
+cell The number of texts to buffer.
|
|
|
|
+row
|
|
+cell #[code disable]
|
|
+cell list
|
|
+cell
|
|
| Names of pipeline components to
|
|
| #[+a("/usage/processing-pipelines#disabling") disable].
|
|
|
|
+row("foot")
|
|
+cell yields
|
|
+cell #[code Doc]
|
|
+cell Documents in the order of the original text.
|
|
|
|
+h(2, "update") Language.update
|
|
+tag method
|
|
|
|
p Update the models in the pipeline.
|
|
|
|
+aside-code("Example").
|
|
for raw_text, entity_offsets in train_data:
|
|
doc = nlp.make_doc(raw_text)
|
|
gold = GoldParse(doc, entities=entity_offsets)
|
|
nlp.update([doc], [gold], drop=0.5, sgd=optimizer)
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code docs]
|
|
+cell iterable
|
|
+cell
|
|
| A batch of #[code Doc] objects or unicode. If unicode, a
|
|
| #[code Doc] object will be created from the text.
|
|
|
|
+row
|
|
+cell #[code golds]
|
|
+cell iterable
|
|
+cell
|
|
| A batch of #[code GoldParse] objects or dictionaries.
|
|
| Dictionaries will be used to create
|
|
| #[+api("goldparse") #[code GoldParse]] objects. For the available
|
|
| keys and their usage, see
|
|
| #[+api("goldparse#init") #[code GoldParse.__init__]].
|
|
|
|
+row
|
|
+cell #[code drop]
|
|
+cell float
|
|
+cell The dropout rate.
|
|
|
|
+row
|
|
+cell #[code sgd]
|
|
+cell callable
|
|
+cell An optimizer.
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell dict
|
|
+cell Results from the update.
|
|
|
|
+h(2, "begin_training") Language.begin_training
|
|
+tag method
|
|
|
|
p
|
|
| Allocate models, pre-process training data and acquire an optimizer.
|
|
|
|
+aside-code("Example").
|
|
optimizer = nlp.begin_training(gold_tuples)
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code gold_tuples]
|
|
+cell iterable
|
|
+cell Gold-standard training data.
|
|
|
|
+row
|
|
+cell #[code **cfg]
|
|
+cell -
|
|
+cell Config parameters.
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell callable
|
|
+cell An optimizer.
|
|
|
|
+h(2, "use_params") Language.use_params
|
|
+tag contextmanager
|
|
+tag method
|
|
|
|
p
|
|
| Replace weights of models in the pipeline with those provided in the
|
|
| params dictionary. Can be used as a contextmanager, in which case, models
|
|
| go back to their original weights after the block.
|
|
|
|
+aside-code("Example").
|
|
with nlp.use_params(optimizer.averages):
|
|
nlp.to_disk('/tmp/checkpoint')
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code params]
|
|
+cell dict
|
|
+cell A dictionary of parameters keyed by model ID.
|
|
|
|
+row
|
|
+cell #[code **cfg]
|
|
+cell -
|
|
+cell Config parameters.
|
|
|
|
+h(2, "preprocess_gold") Language.preprocess_gold
|
|
+tag method
|
|
|
|
p
|
|
| Can be called before training to pre-process gold data. By default, it
|
|
| handles nonprojectivity and adds missing tags to the tag map.
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code docs_golds]
|
|
+cell iterable
|
|
+cell Tuples of #[code Doc] and #[code GoldParse] objects.
|
|
|
|
+row("foot")
|
|
+cell yields
|
|
+cell tuple
|
|
+cell Tuples of #[code Doc] and #[code GoldParse] objects.
|
|
|
|
+h(2, "create_pipe") Language.create_pipe
|
|
+tag method
|
|
+tag-new(2)
|
|
|
|
p Create a pipeline component from a factory.
|
|
|
|
+aside-code("Example").
|
|
parser = nlp.create_pipe('parser')
|
|
nlp.add_pipe(parser)
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code name]
|
|
+cell unicode
|
|
+cell
|
|
| Factory name to look up in
|
|
| #[+api("language#class-attributes") #[code Language.factories]].
|
|
|
|
+row
|
|
+cell #[code config]
|
|
+cell dict
|
|
+cell Configuration parameters to initialise component.
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell callable
|
|
+cell The pipeline component.
|
|
|
|
+h(2, "add_pipe") Language.add_pipe
|
|
+tag method
|
|
+tag-new(2)
|
|
|
|
p
|
|
| Add a component to the processing pipeline. Valid components are
|
|
| callables that take a #[code Doc] object, modify it and return it. Only
|
|
| one of #[code before], #[code after], #[code first] or #[code last] can
|
|
| be set. Default behaviour is #[code last=True].
|
|
|
|
+aside-code("Example").
|
|
def component(doc):
|
|
# modify Doc and return it
|
|
return doc
|
|
|
|
nlp.add_pipe(component, before='ner')
|
|
nlp.add_pipe(component, name='custom_name', last=True)
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code component]
|
|
+cell callable
|
|
+cell The pipeline component.
|
|
|
|
+row
|
|
+cell #[code name]
|
|
+cell unicode
|
|
+cell
|
|
| Name of pipeline component. Overwrites existing
|
|
| #[code component.name] attribute if available. If no #[code name]
|
|
| is set and the component exposes no name attribute,
|
|
| #[code component.__name__] is used. An error is raised if the
|
|
| name already exists in the pipeline.
|
|
|
|
+row
|
|
+cell #[code before]
|
|
+cell unicode
|
|
+cell Component name to insert component directly before.
|
|
|
|
+row
|
|
+cell #[code after]
|
|
+cell unicode
|
|
+cell Component name to insert component directly after:
|
|
|
|
+row
|
|
+cell #[code first]
|
|
+cell bool
|
|
+cell Insert component first / not first in the pipeline.
|
|
|
|
+row
|
|
+cell #[code last]
|
|
+cell bool
|
|
+cell Insert component last / not last in the pipeline.
|
|
|
|
+h(2, "has_pipe") Language.has_pipe
|
|
+tag method
|
|
+tag-new(2)
|
|
|
|
p
|
|
| Check whether a component is present in the pipeline. Equivalent to
|
|
| #[code name in nlp.pipe_names].
|
|
|
|
+aside-code("Example").
|
|
nlp.add_pipe(lambda doc: doc, name='component')
|
|
assert 'component' in nlp.pipe_names
|
|
assert nlp.has_pipe('component')
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code name]
|
|
+cell unicode
|
|
+cell Name of the pipeline component to check.
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell bool
|
|
+cell Whether a component of that name exists in the pipeline.
|
|
|
|
+h(2, "get_pipe") Language.get_pipe
|
|
+tag method
|
|
+tag-new(2)
|
|
|
|
p Get a pipeline component for a given component name.
|
|
|
|
+aside-code("Example").
|
|
parser = nlp.get_pipe('parser')
|
|
custom_component = nlp.get_pipe('custom_component')
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code name]
|
|
+cell unicode
|
|
+cell Name of the pipeline component to get.
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell callable
|
|
+cell The pipeline component.
|
|
|
|
+h(2, "replace_pipe") Language.replace_pipe
|
|
+tag method
|
|
+tag-new(2)
|
|
|
|
p Replace a component in the pipeline.
|
|
|
|
+aside-code("Example").
|
|
nlp.replace_pipe('parser', my_custom_parser)
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code name]
|
|
+cell unicode
|
|
+cell Name of the component to replace.
|
|
|
|
+row
|
|
+cell #[code component]
|
|
+cell callable
|
|
+cell The pipeline component to inser.
|
|
|
|
|
|
+h(2, "rename_pipe") Language.rename_pipe
|
|
+tag method
|
|
+tag-new(2)
|
|
|
|
p
|
|
| Rename a component in the pipeline. Useful to create custom names for
|
|
| pre-defined and pre-loaded components. To change the default name of
|
|
| a component added to the pipeline, you can also use the #[code name]
|
|
| argument on #[+api("language#add_pipe") #[code add_pipe]].
|
|
|
|
+aside-code("Example").
|
|
nlp.rename_pipe('parser', 'spacy_parser')
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code old_name]
|
|
+cell unicode
|
|
+cell Name of the component to rename.
|
|
|
|
+row
|
|
+cell #[code new_name]
|
|
+cell unicode
|
|
+cell New name of the component.
|
|
|
|
+h(2, "remove_pipe") Language.remove_pipe
|
|
+tag method
|
|
+tag-new(2)
|
|
|
|
p
|
|
| Remove a component from the pipeline. Returns the removed component name
|
|
| and component function.
|
|
|
|
+aside-code("Example").
|
|
name, component = nlp.remove_pipe('parser')
|
|
assert name == 'parser'
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code name]
|
|
+cell unicode
|
|
+cell Name of the component to remove.
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell tuple
|
|
+cell A #[code (name, component)] tuple of the removed component.
|
|
|
|
+h(2, "disable_pipes") Language.disable_pipes
|
|
+tag contextmanager
|
|
+tag-new(2)
|
|
|
|
p
|
|
| Disable one or more pipeline components. If used as a context manager,
|
|
| the pipeline will be restored to the initial state at the end of the
|
|
| block. Otherwise, a #[code DisabledPipes] object is returned, that has a
|
|
| #[code .restore()] method you can use to undo your changes.
|
|
|
|
+aside-code("Example").
|
|
with nlp.disable_pipes('tagger', 'parser'):
|
|
optimizer = nlp.begin_training(gold_tuples)
|
|
|
|
disabled = nlp.disable_pipes('tagger', 'parser')
|
|
optimizer = nlp.begin_training(gold_tuples)
|
|
disabled.restore()
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code *disabled]
|
|
+cell unicode
|
|
+cell Names of pipeline components to disable.
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell #[code DisabledPipes]
|
|
+cell
|
|
| The disabled pipes that can be restored by calling the object's
|
|
| #[code .restore()] method.
|
|
|
|
+h(2, "to_disk") Language.to_disk
|
|
+tag method
|
|
+tag-new(2)
|
|
|
|
p
|
|
| Save the current state to a directory. If a model is loaded, this will
|
|
| #[strong include the model].
|
|
|
|
+aside-code("Example").
|
|
nlp.to_disk('/path/to/models')
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code path]
|
|
+cell unicode or #[code Path]
|
|
+cell
|
|
| A path to a directory, which will be created if it doesn't exist.
|
|
| Paths may be either strings or #[code Path]-like objects.
|
|
|
|
+row
|
|
+cell #[code disable]
|
|
+cell list
|
|
+cell
|
|
| Names of pipeline components to
|
|
| #[+a("/usage/processing-pipelines#disabling") disable]
|
|
| and prevent from being saved.
|
|
|
|
+h(2, "from_disk") Language.from_disk
|
|
+tag method
|
|
+tag-new(2)
|
|
|
|
p
|
|
| Loads state from a directory. Modifies the object in place and returns
|
|
| it. If the saved #[code Language] object contains a model, the
|
|
| model will be loaded. Note that this method is commonly used via the
|
|
| subclasses like #[code English] or #[code German] to make
|
|
| language-specific functionality like the
|
|
| #[+a("/usage/adding-languages#lex-attrs") lexical attribute getters]
|
|
| available to the loaded object.
|
|
|
|
+aside-code("Example").
|
|
from spacy.language import Language
|
|
nlp = Language().from_disk('/path/to/model')
|
|
|
|
# using language-specific subclass
|
|
from spacy.lang.en import English
|
|
nlp = English().from_disk('/path/to/en_model')
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code path]
|
|
+cell unicode or #[code Path]
|
|
+cell
|
|
| A path to a directory. Paths may be either strings or
|
|
| #[code Path]-like objects.
|
|
|
|
+row
|
|
+cell #[code disable]
|
|
+cell list
|
|
+cell
|
|
| Names of pipeline components to
|
|
| #[+a("/usage/processing-pipelines#disabling") disable].
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell #[code Language]
|
|
+cell The modified #[code Language] object.
|
|
|
|
+infobox("Changed in v2.0", "⚠️")
|
|
| As of spaCy v2.0, the #[code save_to_directory] method has been
|
|
| renamed to #[code to_disk], to improve consistency across classes.
|
|
| Pipeline components to prevent from being loaded can now be added as
|
|
| a list to #[code disable], instead of specifying one keyword argument
|
|
| per component.
|
|
|
|
+code-wrapper
|
|
+code-new nlp = English().from_disk(disable=['tagger', 'ner'])
|
|
+code-old nlp = spacy.load('en', tagger=False, entity=False)
|
|
|
|
+h(2, "to_bytes") Language.to_bytes
|
|
+tag method
|
|
|
|
p Serialize the current state to a binary string.
|
|
|
|
+aside-code("Example").
|
|
nlp_bytes = nlp.to_bytes()
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code disable]
|
|
+cell list
|
|
+cell
|
|
| Names of pipeline components to
|
|
| #[+a("/usage/processing-pipelines#disabling") disable]
|
|
| and prevent from being serialized.
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell bytes
|
|
+cell The serialized form of the #[code Language] object.
|
|
|
|
+h(2, "from_bytes") Language.from_bytes
|
|
+tag method
|
|
|
|
p
|
|
| Load state from a binary string. Note that this method is commonly used
|
|
| via the subclasses like #[code English] or #[code German] to make
|
|
| language-specific functionality like the
|
|
| #[+a("/usage/adding-languages#lex-attrs") lexical attribute getters]
|
|
| available to the loaded object.
|
|
|
|
+aside-code("Example").
|
|
from spacy.lang.en import English
|
|
nlp_bytes = nlp.to_bytes()
|
|
nlp2 = English()
|
|
nlp2.from_bytes(nlp_bytes)
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code bytes_data]
|
|
+cell bytes
|
|
+cell The data to load from.
|
|
|
|
+row
|
|
+cell #[code disable]
|
|
+cell list
|
|
+cell
|
|
| Names of pipeline components to
|
|
| #[+a("/usage/processing-pipelines#disabling") disable].
|
|
|
|
+row("foot")
|
|
+cell returns
|
|
+cell #[code Language]
|
|
+cell The #[code Language] object.
|
|
|
|
+infobox("Changed in v2.0", "⚠️")
|
|
| Pipeline components to prevent from being loaded can now be added as
|
|
| a list to #[code disable], instead of specifying one keyword argument
|
|
| per component.
|
|
|
|
+code-wrapper
|
|
+code-new nlp = English().from_bytes(bytes, disable=['tagger', 'ner'])
|
|
+code-old nlp = English().from_bytes('en', tagger=False, entity=False)
|
|
|
|
+h(2, "attributes") Attributes
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code vocab]
|
|
+cell #[code Vocab]
|
|
+cell A container for the lexical types.
|
|
|
|
+row
|
|
+cell #[code tokenizer]
|
|
+cell #[code Tokenizer]
|
|
+cell The tokenizer.
|
|
|
|
+row
|
|
+cell #[code make_doc]
|
|
+cell #[code lambda text: Doc]
|
|
+cell Create a #[code Doc] object from unicode text.
|
|
|
|
+row
|
|
+cell #[code pipeline]
|
|
+cell list
|
|
+cell
|
|
| List of #[code (name, component)] tuples describing the current
|
|
| processing pipeline, in order.
|
|
|
|
+row
|
|
+cell #[code pipe_names]
|
|
+tag-new(2)
|
|
+cell list
|
|
+cell List of pipeline component names, in order.
|
|
|
|
+row
|
|
+cell #[code meta]
|
|
+cell dict
|
|
+cell
|
|
| Custom meta data for the Language class. If a model is loaded,
|
|
| contains meta data of the model.
|
|
|
|
+row
|
|
+cell #[code path]
|
|
+tag-new(2)
|
|
+cell #[code Path]
|
|
+cell
|
|
| Path to the model data directory, if a model is loaded. Otherwise
|
|
| #[code None].
|
|
|
|
+h(2, "class-attributes") Class attributes
|
|
|
|
+table(["Name", "Type", "Description"])
|
|
+row
|
|
+cell #[code Defaults]
|
|
+cell class
|
|
+cell
|
|
| Settings, data and factory methods for creating the
|
|
| #[code nlp] object and processing pipeline.
|
|
|
|
+row
|
|
+cell #[code lang]
|
|
+cell unicode
|
|
+cell
|
|
| Two-letter language ID, i.e.
|
|
| #[+a("https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes") ISO code].
|
|
|
|
+row
|
|
+cell #[code factories]
|
|
+tag-new(2)
|
|
+cell dict
|
|
+cell
|
|
| Factories that create pre-defined pipeline components, e.g. the
|
|
| tagger, parser or entity recognizer, keyed by their component
|
|
| name.
|