spaCy/website/api/language.jade

//- 💫 DOCS > API > LANGUAGE

include ../_includes/_mixins

p
    |  Usually you'll load this once per process as #[code nlp] and pass the
    |  instance around your application. The #[code Language] class is created
    |  when you call #[+api("spacy#load") #[code spacy.load()]] and contains
    |  the shared vocabulary and #[+a("/usage/adding-languages") language data],
    |  optional model data loaded from a #[+a("/models") model package] or
    |  a path, and a #[+a("/usage/processing-pipelines") processing pipeline]
    |  containing components like the tagger or parser that are called on a
    |  document in order. You can also add your own processing pipeline
    |  components that take a #[code Doc] object, modify it and return it.

+h(2, "init") Language.__init__
    +tag method

p Initialise a #[code Language] object.

+aside-code("Example").
    from spacy.vocab import Vocab
    from spacy.language import Language
    nlp = Language(Vocab())

    from spacy.lang.en import English
    nlp = English()

+table(["Name", "Type", "Description"])
    +row
        +cell #[code vocab]
        +cell #[code Vocab]
        +cell
            |  A #[code Vocab] object. If #[code True], a vocab is created via
            |  #[code Language.Defaults.create_vocab].

    +row
        +cell #[code make_doc]
        +cell callable
        +cell
            |  A function that takes text and returns a #[code Doc] object.
            |  Usually a #[code Tokenizer].

    +row
        +cell #[code meta]
        +cell dict
        +cell
            |  Custom meta data for the #[code Language] class. Is written to by
            |  models to add model meta data.

    +row("foot")
        +cell returns
        +cell #[code Language]
        +cell The newly constructed object.

+h(2, "call") Language.__call__
    +tag method

p
    |  Apply the pipeline to some text. The text can span multiple sentences,
    |  and can contain arbtrary whitespace. Alignment into the original string
    |  is preserved.

+aside-code("Example").
    doc = nlp(u'An example sentence. Another sentence.')
    assert (doc[0].text, doc[0].head.tag_) == ('An', 'NN')

+table(["Name", "Type", "Description"])
    +row
        +cell #[code text]
        +cell unicode
        +cell The text to be processed.

    +row
        +cell #[code disable]
        +cell list
        +cell
            |  Names of pipeline components to
            |  #[+a("/usage/processing-pipelines#disabling") disable].

    +row("foot")
        +cell returns
        +cell #[code Doc]
        +cell A container for accessing the annotations.

+infobox("Changed in v2.0", "⚠️")
    |  Pipeline components to prevent from being loaded can now be added as
    |  a list to #[code disable], instead of specifying one keyword argument
    |  per component.

    +code-wrapper
        +code-new doc = nlp(u"I don't want parsed", disable=['parser'])
        +code-old doc = nlp(u"I don't want parsed", parse=False)

+h(2, "pipe") Language.pipe
    +tag method

p
    |  Process texts as a stream, and yield #[code Doc] objects in order.
    |  Supports GIL-free multi-threading.

+infobox("Important note for spaCy v2.0.x", "⚠️")
    |  By default, multiple threads will be launched for matrix multiplication,
    |  which may be inefficient on multi-core machines. Setting
    |  #[code OPENBLAS_NUM_THREADS=1] should fix this problem. spaCy v2.1.x
    |  will be switching to single-thread by default.

+aside-code("Example").
    texts = [u'One document.', u'...', u'Lots of documents']
    for doc in nlp.pipe(texts, batch_size=50, n_threads=4):
        assert doc.is_parsed

+table(["Name", "Type", "Description"])
    +row
        +cell #[code texts]
        +cell -
        +cell A sequence of unicode objects.

    +row
        +cell #[code as_tuples]
        +cell bool
        +cell
            |  If set to #[code True], inputs should be a sequence of
            |  #[code (text, context)] tuples. Output will then be a sequence of
            |  #[code (doc, context)] tuples. Defaults to #[code False].

    +row
        +cell #[code n_threads]
        +cell int
        +cell
            |  The number of worker threads to use. If #[code -1], OpenMP will
            |  decide how many to use at run time. Default is #[code 2].

    +row
        +cell #[code batch_size]
        +cell int
        +cell The number of texts to buffer.

    +row
        +cell #[code disable]
        +cell list
        +cell
            |  Names of pipeline components to
            |  #[+a("/usage/processing-pipelines#disabling") disable].

    +row("foot")
        +cell yields
        +cell #[code Doc]
        +cell Documents in the order of the original text.

+h(2, "update") Language.update
    +tag method

p Update the models in the pipeline.

+aside-code("Example").
    for raw_text, entity_offsets in train_data:
        doc = nlp.make_doc(raw_text)
        gold = GoldParse(doc, entities=entity_offsets)
        nlp.update([doc], [gold], drop=0.5, sgd=optimizer)

+table(["Name", "Type", "Description"])
    +row
        +cell #[code docs]
        +cell iterable
        +cell
            |  A batch of #[code Doc] objects or unicode. If unicode, a
            |  #[code Doc] object will be created from the text.

    +row
        +cell #[code golds]
        +cell iterable
        +cell
            |  A batch of #[code GoldParse] objects or dictionaries.
            |  Dictionaries will be used to create
            |  #[+api("goldparse") #[code GoldParse]] objects. For the available
            |  keys and their usage, see
            |  #[+api("goldparse#init") #[code GoldParse.__init__]].

    +row
        +cell #[code drop]
        +cell float
        +cell The dropout rate.

    +row
        +cell #[code sgd]
        +cell callable
        +cell An optimizer.

    +row("foot")
        +cell returns
        +cell dict
        +cell Results from the update.

+h(2, "begin_training") Language.begin_training
    +tag method

p
    |  Allocate models, pre-process training data and acquire an optimizer.

+aside-code("Example").
    optimizer = nlp.begin_training(gold_tuples)

+table(["Name", "Type", "Description"])
    +row
        +cell #[code gold_tuples]
        +cell iterable
        +cell Gold-standard training data.

    +row
        +cell #[code **cfg]
        +cell -
        +cell Config parameters.

    +row("foot")
        +cell returns
        +cell callable
        +cell An optimizer.

+h(2, "use_params") Language.use_params
    +tag contextmanager
    +tag method

p
    |  Replace weights of models in the pipeline with those provided in the
    |  params dictionary. Can be used as a contextmanager, in which case, models
    |  go back to their original weights after the block.

+aside-code("Example").
    with nlp.use_params(optimizer.averages):
        nlp.to_disk('/tmp/checkpoint')

+table(["Name", "Type", "Description"])
    +row
        +cell #[code params]
        +cell dict
        +cell A dictionary of parameters keyed by model ID.

    +row
        +cell #[code **cfg]
        +cell -
        +cell Config parameters.

+h(2, "preprocess_gold") Language.preprocess_gold
    +tag method

p
    |  Can be called before training to pre-process gold data. By default, it
    |  handles nonprojectivity and adds missing tags to the tag map.

+table(["Name", "Type", "Description"])
    +row
        +cell #[code docs_golds]
        +cell iterable
        +cell Tuples of #[code Doc] and #[code GoldParse] objects.

    +row("foot")
        +cell yields
        +cell tuple
        +cell Tuples of #[code Doc] and #[code GoldParse] objects.

+h(2, "create_pipe") Language.create_pipe
    +tag method
    +tag-new(2)

p Create a pipeline component from a factory.

+aside-code("Example").
    parser = nlp.create_pipe('parser')
    nlp.add_pipe(parser)

+table(["Name", "Type", "Description"])
    +row
        +cell #[code name]
        +cell unicode
        +cell
            |  Factory name to look up in
            |  #[+api("language#class-attributes") #[code Language.factories]].

    +row
        +cell #[code config]
        +cell dict
        +cell Configuration parameters to initialise component.

    +row("foot")
        +cell returns
        +cell callable
        +cell The pipeline component.

+h(2, "add_pipe") Language.add_pipe
    +tag method
    +tag-new(2)

p
    |  Add a component to the processing pipeline. Valid components are
    |  callables that take a #[code Doc] object, modify it and return it. Only
    |  one of #[code before], #[code after], #[code first] or #[code last] can
    |  be set. Default behaviour is #[code last=True].

+aside-code("Example").
    def component(doc):
        # modify Doc and return it
        return doc

    nlp.add_pipe(component, before='ner')
    nlp.add_pipe(component, name='custom_name', last=True)

+table(["Name", "Type", "Description"])
    +row
        +cell #[code component]
        +cell callable
        +cell The pipeline component.

    +row
        +cell #[code name]
        +cell unicode
        +cell
            |  Name of pipeline component. Overwrites existing
            |  #[code component.name] attribute if available. If no #[code name]
            |  is set and the component exposes no name attribute,
            |  #[code component.__name__] is used. An error is raised if the
            |  name already exists in the pipeline.

    +row
        +cell #[code before]
        +cell unicode
        +cell Component name to insert component directly before.

    +row
        +cell #[code after]
        +cell unicode
        +cell Component name to insert component directly after:

    +row
        +cell #[code first]
        +cell bool
        +cell Insert component first / not first in the pipeline.

    +row
        +cell #[code last]
        +cell bool
        +cell Insert component last / not last in the pipeline.

+h(2, "has_pipe") Language.has_pipe
    +tag method
    +tag-new(2)

p
    |  Check whether a component is present in the pipeline. Equivalent to
    |  #[code name in nlp.pipe_names].

+aside-code("Example").
    nlp.add_pipe(lambda doc: doc, name='component')
    assert 'component' in nlp.pipe_names
    assert nlp.has_pipe('component')

+table(["Name", "Type", "Description"])
    +row
        +cell #[code name]
        +cell unicode
        +cell Name of the pipeline component to check.

    +row("foot")
        +cell returns
        +cell bool
        +cell Whether a component of that name exists in the pipeline.

+h(2, "get_pipe") Language.get_pipe
    +tag method
    +tag-new(2)

p Get a pipeline component for a given component name.

+aside-code("Example").
    parser = nlp.get_pipe('parser')
    custom_component = nlp.get_pipe('custom_component')

+table(["Name", "Type", "Description"])
    +row
        +cell #[code name]
        +cell unicode
        +cell Name of the pipeline component to get.

    +row("foot")
        +cell returns
        +cell callable
        +cell The pipeline component.

+h(2, "replace_pipe") Language.replace_pipe
    +tag method
    +tag-new(2)

p Replace a component in the pipeline.

+aside-code("Example").
    nlp.replace_pipe('parser', my_custom_parser)

+table(["Name", "Type", "Description"])
    +row
        +cell #[code name]
        +cell unicode
        +cell Name of the component to replace.

    +row
        +cell #[code component]
        +cell callable
        +cell The pipeline component to inser.


+h(2, "rename_pipe") Language.rename_pipe
    +tag method
    +tag-new(2)

p
    |  Rename a component in the pipeline. Useful to create custom names for
    |  pre-defined and pre-loaded components. To change the default name of
    |  a component added to the pipeline, you can also use the #[code name]
    |  argument on #[+api("language#add_pipe") #[code add_pipe]].

+aside-code("Example").
    nlp.rename_pipe('parser', 'spacy_parser')

+table(["Name", "Type", "Description"])
    +row
        +cell #[code old_name]
        +cell unicode
        +cell Name of the component to rename.

    +row
        +cell #[code new_name]
        +cell unicode
        +cell New name of the component.

+h(2, "remove_pipe") Language.remove_pipe
    +tag method
    +tag-new(2)

p
    |  Remove a component from the pipeline. Returns the removed component name
    |  and component function.

+aside-code("Example").
    name, component = nlp.remove_pipe('parser')
    assert name == 'parser'

+table(["Name", "Type", "Description"])
    +row
        +cell #[code name]
        +cell unicode
        +cell Name of the component to remove.

    +row("foot")
        +cell returns
        +cell tuple
        +cell A #[code (name, component)] tuple of the removed component.

+h(2, "disable_pipes") Language.disable_pipes
    +tag contextmanager
    +tag-new(2)

p
    |  Disable one or more pipeline components. If used as a context manager,
    |  the pipeline will be restored to the initial state at the end of the
    |  block. Otherwise, a #[code DisabledPipes] object is returned, that has a
    |  #[code .restore()] method you can use to undo your changes.

+aside-code("Example").
    with nlp.disable_pipes('tagger', 'parser'):
        optimizer = nlp.begin_training(gold_tuples)

    disabled = nlp.disable_pipes('tagger', 'parser')
    optimizer = nlp.begin_training(gold_tuples)
    disabled.restore()

+table(["Name", "Type", "Description"])
    +row
        +cell #[code *disabled]
        +cell unicode
        +cell Names of pipeline components to disable.

    +row("foot")
        +cell returns
        +cell #[code DisabledPipes]
        +cell
            |  The disabled pipes that can be restored by calling the object's
            |  #[code .restore()] method.

+h(2, "to_disk") Language.to_disk
    +tag method
    +tag-new(2)

p
    |  Save the current state to a directory. If a model is loaded, this will
    |  #[strong include the model].

+aside-code("Example").
    nlp.to_disk('/path/to/models')

+table(["Name", "Type", "Description"])
    +row
        +cell #[code path]
        +cell unicode or #[code Path]
        +cell
            |  A path to a directory, which will be created if it doesn't exist.
            |  Paths may be either strings or #[code Path]-like objects.

    +row
        +cell #[code disable]
        +cell list
        +cell
            |  Names of pipeline components to
            |  #[+a("/usage/processing-pipelines#disabling") disable]
            |  and prevent from being saved.

+h(2, "from_disk") Language.from_disk
    +tag method
    +tag-new(2)

p
    |  Loads state from a directory. Modifies the object in place and returns
    |  it. If the saved #[code Language] object contains a model, the
    |  model will be loaded. Note that this method is commonly used via the
    |  subclasses like #[code English] or #[code German] to make
    |  language-specific functionality like the
    |  #[+a("/usage/adding-languages#lex-attrs") lexical attribute getters]
    |  available to the loaded object.

+aside-code("Example").
    from spacy.language import Language
    nlp = Language().from_disk('/path/to/model')

    # using language-specific subclass
    from spacy.lang.en import English
    nlp = English().from_disk('/path/to/en_model')

+table(["Name", "Type", "Description"])
    +row
        +cell #[code path]
        +cell unicode or #[code Path]
        +cell
            |  A path to a directory. Paths may be either strings or
            |  #[code Path]-like objects.

    +row
        +cell #[code disable]
        +cell list
        +cell
            |  Names of pipeline components to
            |  #[+a("/usage/processing-pipelines#disabling") disable].

    +row("foot")
        +cell returns
        +cell #[code Language]
        +cell The modified #[code Language] object.

+infobox("Changed in v2.0", "⚠️")
    |  As of spaCy v2.0, the #[code save_to_directory] method has been
    |  renamed to #[code to_disk], to improve consistency across classes.
    |  Pipeline components to prevent from being loaded can now be added as
    |  a list to #[code disable], instead of specifying one keyword argument
    |  per component.

    +code-wrapper
        +code-new nlp = English().from_disk(disable=['tagger', 'ner'])
        +code-old nlp = spacy.load('en', tagger=False, entity=False)

+h(2, "to_bytes") Language.to_bytes
    +tag method

p Serialize the current state to a binary string.

+aside-code("Example").
    nlp_bytes = nlp.to_bytes()

+table(["Name", "Type", "Description"])
    +row
        +cell #[code disable]
        +cell list
        +cell
            |  Names of pipeline components to
            |  #[+a("/usage/processing-pipelines#disabling") disable]
            |  and prevent from being serialized.

    +row("foot")
        +cell returns
        +cell bytes
        +cell The serialized form of the #[code Language] object.

+h(2, "from_bytes") Language.from_bytes
    +tag method

p
    |  Load state from a binary string. Note that this method is commonly used
    |  via the subclasses like #[code English] or #[code German] to make
    |  language-specific functionality like the
    |  #[+a("/usage/adding-languages#lex-attrs") lexical attribute getters]
    |  available to the loaded object.

+aside-code("Example").
    from spacy.lang.en import English
    nlp_bytes = nlp.to_bytes()
    nlp2 = English()
    nlp2.from_bytes(nlp_bytes)

+table(["Name", "Type", "Description"])
    +row
        +cell #[code bytes_data]
        +cell bytes
        +cell The data to load from.

    +row
        +cell #[code disable]
        +cell list
        +cell
            |  Names of pipeline components to
            |  #[+a("/usage/processing-pipelines#disabling") disable].

    +row("foot")
        +cell returns
        +cell #[code Language]
        +cell The #[code Language] object.

+infobox("Changed in v2.0", "⚠️")
    |  Pipeline components to prevent from being loaded can now be added as
    |  a list to #[code disable], instead of specifying one keyword argument
    |  per component.

    +code-wrapper
        +code-new nlp = English().from_bytes(bytes, disable=['tagger', 'ner'])
        +code-old nlp = English().from_bytes('en', tagger=False, entity=False)

+h(2, "attributes") Attributes

+table(["Name", "Type", "Description"])
    +row
        +cell #[code vocab]
        +cell #[code Vocab]
        +cell A container for the lexical types.

    +row
        +cell #[code tokenizer]
        +cell #[code Tokenizer]
        +cell The tokenizer.

    +row
        +cell #[code make_doc]
        +cell #[code lambda text: Doc]
        +cell Create a #[code Doc] object from unicode text.

    +row
        +cell #[code pipeline]
        +cell list
        +cell
            |  List of #[code (name, component)] tuples describing the current
            |  processing pipeline, in order.

    +row
        +cell #[code pipe_names]
            +tag-new(2)
        +cell list
        +cell List of pipeline component names, in order.

    +row
        +cell #[code meta]
        +cell dict
        +cell
            |  Custom meta data for the Language class. If a model is loaded,
            |  contains meta data of the model.

    +row
        +cell #[code path]
            +tag-new(2)
        +cell #[code Path]
        +cell
            |  Path to the model data directory, if a model is loaded. Otherwise
            |  #[code None].

+h(2, "class-attributes") Class attributes

+table(["Name", "Type", "Description"])
    +row
        +cell #[code Defaults]
        +cell class
        +cell
            |  Settings, data and factory methods for creating the
            |  #[code nlp] object and processing pipeline.

    +row
        +cell #[code lang]
        +cell unicode
        +cell
            |  Two-letter language ID, i.e.
            |  #[+a("https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes") ISO code].

    +row
        +cell #[code factories]
            +tag-new(2)
        +cell dict
        +cell
            |  Factories that create pre-defined pipeline components, e.g. the
            |  tagger, parser or entity recognizer, keyed by their component
            |  name.