Update examples and API docs

website/docs/api/doc.jade

@@ -109,9 +109,8 @@ p
     |  easily accessed.
 
 +aside-code("Example").
-    doc = nlp(u'Give it back! He pleaded.')
+    doc = nlp(u'Give it back')
-    for token in doc:
+    assert [t.text for t in doc] == [u'Give', u'it', u'back']
-        print(token.text, token.tag_)
 
 p
     |  This is the main way of accessing #[+api("token") #[code Token]] objects,
@@ -143,7 +142,7 @@ p Get the number of tokens in the document.
 
 +h(2, "similarity") Doc.similarity
     +tag method
-    +tag requires model
+    +tag requires model: vectors
 
 p
     |  Make a semantic similarity estimate. The default estimate is cosine
@@ -178,11 +177,10 @@ p
     |  of the given attribute ID.
 
 +aside-code("Example").
-    from spacy import attrs
+    from spacy.attrs import ORTH
     doc = nlp(u'apple apple orange banana')
-    tokens.count_by(attrs.ORTH)
+    assert doc.count_by(ORTH) == {7024L: 1, 119552L: 1, 2087L: 2}
-    # {12800L: 1, 11880L: 2, 7561L: 1}
+    doc.to_array([attrs.ORTH])
-    tokens.to_array([attrs.ORTH])
     # array([[11880], [11880], [7561], [12800]])
 
 +table(["Name", "Type", "Description"])
@@ -237,6 +235,7 @@ p
     np_array = doc.to_array([LOWER, POS, ENT_TYPE, IS_ALPHA])
     doc2 = Doc(doc.vocab)
     doc2.from_array([LOWER, POS, ENT_TYPE, IS_ALPHA], np_array)
+    assert doc.text == doc2.text
 
 +table(["Name", "Type", "Description"])
     +row
@@ -307,8 +306,7 @@ p
 +aside-code("Example").
     doc = nlp(u'Los Angeles start.')
     doc.merge(0, len('Los Angeles'), 'NNP', 'Los Angeles', 'GPE')
-    print([token.text for token in doc])
+    assert [t.text for t in doc] == [u'Los Angeles', u'start', u'.']
-    # ['Los Angeles', 'start', '.']
 
 +table(["Name", "Type", "Description"])
     +row
@@ -338,7 +336,7 @@ p
 
 +h(2, "print_tree") Doc.print_tree
     +tag method
-    +tag requires model
+    +tag requires model: parse
 
 p
     |  Returns the parse trees in JSON (dict) format. Especially useful for
@@ -371,7 +369,7 @@ p
 
 +h(2, "ents") Doc.ents
     +tag property
-    +tag requires model
+    +tag requires model: NER
 
 p
     |  Iterate over the entities in the document. Yields named-entity
@@ -393,7 +391,7 @@ p
 
 +h(2, "noun_chunks") Doc.noun_chunks
     +tag property
-    +tag requires model
+    +tag requires model: parse
 
 p
     |  Iterate over the base noun phrases in the document. Yields base
@@ -416,7 +414,7 @@ p
 
 +h(2, "sents") Doc.sents
     +tag property
-    +tag requires model
+    +tag requires model: parse
 
 p
     |  Iterate over the sentences in the document. Sentence spans have no label.
@@ -438,7 +436,7 @@ p
 
 +h(2, "has_vector") Doc.has_vector
     +tag property
-    +tag requires model
+    +tag requires model: vectors
 
 p
     |  A boolean value indicating whether a word vector is associated with the
@@ -456,7 +454,7 @@ p
 
 +h(2, "vector") Doc.vector
     +tag property
-    +tag requires model
+    +tag requires model: vectors
 
 p
     |  A real-valued meaning representation. Defaults to an average of the
@@ -464,8 +462,8 @@ p
 
 +aside-code("Example").
     apples = nlp(u'I like apples')
-    (apples.vector.dtype, apples.vector.shape)
+    assert doc.vector.dtype == 'float32'
-    # (dtype('float32'), (300,))
+    assert doc.vector.shape == (300,)
 
 +table(["Name", "Type", "Description"])
     +footrow
@@ -475,11 +473,18 @@ p
 
 +h(2, "vector_norm") Doc.vector_norm
     +tag property
-    +tag requires model
+    +tag requires model: vectors
 
 p
     |  The L2 norm of the document's vector representation.
 
++aside-code("Example").
+    doc1 = nlp(u'I like apples')
+    doc2 = nlp(u'I like oranges')
+    doc1.vector_norm # 4.54232424414368
+    doc2.vector_norm # 3.304373298575751
+    assert doc1.vector_norm != doc2.vector_norm
+
 +table(["Name", "Type", "Description"])
     +footrow
         +cell returns

website/docs/api/span.jade

@@ -12,8 +12,7 @@ p Create a Span object from the #[code slice doc[start : end]].
 +aside-code("Example").
     doc = nlp(u'Give it back! He pleaded.')
     span = doc[1:4]
-    print([token.text for token in span])
+    assert [t.text for t in span] ==  [u'it', u'back', u'!']
-    # ['it', 'back', '!']
 
 +table(["Name", "Type", "Description"])
     +row
@@ -93,8 +92,7 @@ p Iterate over #[code Token] objects.
 +aside-code("Example").
     doc = nlp(u'Give it back! He pleaded.')
     span = doc[1:4]
-    print([token.text for token in span])
+    assert [t.text for t in span] == ['it', 'back', '!']
-    # ['it', 'back', '!']
 
 +table(["Name", "Type", "Description"])
     +footrow
@@ -120,16 +118,18 @@ p Get the number of tokens in the span.
 
 +h(2, "similarity") Span.similarity
     +tag method
-    +tag requires model
+    +tag requires model: vectors
 
 p
     |  Make a semantic similarity estimate. The default estimate is cosine
     |  similarity using an average of word vectors.
 
 +aside-code("Example").
-    apples, and, oranges = nlp(u'apples and oranges')
+    doc = nlp(u'green apples and red oranges')
-    apples_oranges = apples.similarity(oranges)
+    green_apples = doc[:2]
-    oranges_apples = oranges.similarity(apples)
+    red_oranges = doc[3:]
+    apples_oranges = green_apples.similarity(red_oranges)
+    oranges_apples = red_oranges.similarity(green_apples)
     assert apples_oranges == oranges_apples
 
 +table(["Name", "Type", "Description"])
@@ -165,17 +165,18 @@ p Retokenize the document, such that the span is merged into a single token.
 
 +h(2, "root") Span.root
     +tag property
+    +tag requires model: parse
 
 p
     |  The token within the span that's highest in the parse tree. If there's a
     |  tie, the earlist is prefered.
 
 +aside-code("Example").
-    tokens = nlp(u'I like New York in Autumn.')
+    doc = nlp(u'I like New York in Autumn.')
-    i, like, new, york, in_, autumn, dot = range(len(tokens))
+    i, like, new, york, in_, autumn, dot = range(len(doc))
-    assert tokens[new].head.text == 'York'
+    assert doc[new].head.text == 'York'
-    assert tokens[york].head.text == 'like'
+    assert doc[york].head.text == 'like'
-    new_york = tokens[new:york+1]
+    new_york = doc[new:york+1]
     assert new_york.root.text == 'York'
 
 +table(["Name", "Type", "Description"])
@@ -186,9 +187,15 @@ p
 
 +h(2, "lefts") Span.lefts
     +tag property
+    +tag requires model: parse
 
 p Tokens that are to the left of the span, whose head is within the span.
 
++aside-code("Example").
+    doc = nlp(u'I like New York in Autumn.')
+    lefts = [t.text for t in doc[3:7].lefts]
+    assert lefts == [u'New']
+
 +table(["Name", "Type", "Description"])
     +footrow
         +cell yields
@@ -197,9 +204,15 @@ p Tokens that are to the left of the span, whose head is within the span.
 
 +h(2, "rights") Span.rights
     +tag property
+    +tag requires model: parse
 
 p Tokens that are to the right of the span, whose head is within the span.
 
++aside-code("Example").
+    doc = nlp(u'I like New York in Autumn.')
+    rights = [t.text for t in doc[2:4].rights]
+    assert rights == [u'in']
+
 +table(["Name", "Type", "Description"])
     +footrow
         +cell yields
@@ -208,9 +221,15 @@ p Tokens that are to the right of the span, whose head is within the span.
 
 +h(2, "subtree") Span.subtree
     +tag property
+    +tag requires model: parse
 
 p Tokens that descend from tokens in the span, but fall outside it.
 
++aside-code("Example").
+    doc = nlp(u'Give it back! He pleaded.')
+    subtree = [t.text for t in doc[:3].subtree]
+    assert subtree == [u'Give', u'it', u'back', u'!']
+
 +table(["Name", "Type", "Description"])
     +footrow
         +cell yields
@@ -219,15 +238,15 @@ p Tokens that descend from tokens in the span, but fall outside it.
 
 +h(2, "has_vector") Span.has_vector
     +tag property
-    +tag requires model
+    +tag requires model: vectors
 
 p
     |  A boolean value indicating whether a word vector is associated with the
     |  object.
 
 +aside-code("Example").
-    apple = nlp(u'apple')
+    doc = nlp(u'I like apples')
-    assert apple.has_vector
+    assert doc[1:].has_vector
 
 +table(["Name", "Type", "Description"])
     +footrow
@@ -237,16 +256,16 @@ p
 
 +h(2, "vector") Span.vector
     +tag property
-    +tag requires model
+    +tag requires model: vectors
 
 p
     |  A real-valued meaning representation. Defaults to an average of the
     |  token vectors.
 
 +aside-code("Example").
-    apple = nlp(u'apple')
+    doc = nlp(u'I like apples')
-    (apple.vector.dtype, apple.vector.shape)
+    assert doc[1:].vector.dtype == 'float32'
-    # (dtype('float32'), (300,))
+    assert doc[1:].vector.shape == (300,)
 
 +table(["Name", "Type", "Description"])
     +footrow
@@ -256,11 +275,17 @@ p
 
 +h(2, "vector_norm") Span.vector_norm
     +tag property
-    +tag requires model
+    +tag requires model: vectors
 
 p
     |  The L2 norm of the span's vector representation.
 
++aside-code("Example").
+    doc = nlp(u'I like apples')
+    doc[1:].vector_norm # 4.800883928527915
+    doc[2:].vector_norm # 6.895897646384268
+    assert doc[1:].vector_norm != doc[2:].vector_norm
+
 +table(["Name", "Type", "Description"])
     +footrow
         +cell returns

website/docs/api/token.jade

@@ -12,6 +12,7 @@ p Construct a #[code Token] object.
 +aside-code("Example").
     doc = nlp(u'Give it back! He pleaded.')
     token = doc[0]
+    assert token.text == u'Give'
 
 +table(["Name", "Type", "Description"])
     +row
@@ -59,8 +60,7 @@ p Check the value of a boolean flag.
     from spacy.attrs import IS_TITLE
     doc = nlp(u'Give it back! He pleaded.')
     token = doc[0]
-    token.check_flag(IS_TITLE)
+    assert token.check_flag(IS_TITLE) == True
-    # True
 
 +table(["Name", "Type", "Description"])
     +row
@@ -73,35 +73,14 @@ p Check the value of a boolean flag.
         +cell bool
         +cell Whether the flag is set.
 
-+h(2, "nbor") Token.nbor
-    +tag method
-
-p Get a neighboring token.
-
-+aside-code("Example").
-    doc = nlp(u'Give it back! He pleaded.')
-    token = doc[0]
-    token.nbor()
-    # it
-
-+table(["Name", "Type", "Description"])
-    +row
-        +cell #[code i]
-        +cell int
-        +cell The relative position of the token to get. Defaults to #[code 1].
-
-    +footrow
-        +cell returns
-        +cell #[code Token]
-        +cell The token at position #[code self.doc[self.i+i]].
-
 +h(2, "similarity") Token.similarity
     +tag method
+    +tag requires model: vectors
 
 p Compute a semantic similarity estimate. Defaults to cosine over vectors.
 
 +aside-code("Example").
-    apples, and, oranges = nlp(u'apples and oranges')
+    apples, _, oranges = nlp(u'apples and oranges')
     apples_oranges = apples.similarity(oranges)
     oranges_apples = oranges.similarity(apples)
     assert apples_oranges == oranges_apples
@@ -119,13 +98,41 @@ p Compute a semantic similarity estimate. Defaults to cosine over vectors.
         +cell float
         +cell A scalar similarity score. Higher is more similar.
 
++h(2, "nbor") Token.nbor
+    +tag method
+
+p Get a neighboring token.
+
++aside-code("Example").
+    doc = nlp(u'Give it back! He pleaded.')
+    give_nbor = doc[0].nbor()
+    assert give_nbor.text == u'it'
+
++table(["Name", "Type", "Description"])
+    +row
+        +cell #[code i]
+        +cell int
+        +cell The relative position of the token to get. Defaults to #[code 1].
+
+    +footrow
+        +cell returns
+        +cell #[code Token]
+        +cell The token at position #[code self.doc[self.i+i]].
+
 +h(2, "is_ancestor") Token.is_ancestor
     +tag method
+    +tag requires model: parse
 
 p
     |  Check whether this token is a parent, grandparent, etc. of another
     |  in the dependency tree.
 
++aside-code("Example").
+    doc = nlp(u'Give it back! He pleaded.')
+    give = doc[0]
+    it = doc[1]
+    assert give.is_ancestor(it)
+
 +table(["Name", "Type", "Description"])
     +row
         +cell descendant
@@ -137,60 +144,38 @@ p
         +cell bool
         +cell Whether this token is the ancestor of the descendant.
 
-+h(2, "has_vector") Token.has_vector
++h(2, "ancestors") Token.ancestors
     +tag property
-    +tag requires model
+    +tag requires model: parse
 
-p
+p The rightmost token of this token's syntactic descendants.
-    |  A boolean value indicating whether a word vector is associated with the
-    |  token.
 
 +aside-code("Example").
-    apple = nlp(u'apple')
+    doc = nlp(u'Give it back! He pleaded.')
-    assert apple.has_vector
+    it_ancestors = doc[1].ancestors
+    assert [t.text for t in it_ancestors] == [u'Give']
+    he_ancestors = doc[4].ancestors
+    assert [t.text for t in he_ancestors] == [u'pleaded']
 
 +table(["Name", "Type", "Description"])
     +footrow
-        +cell returns
+        +cell yields
-        +cell bool
+        +cell #[code Token]
-        +cell Whether the token has a vector data attached.
+        +cell
-
+            |  A sequence of ancestor tokens such that
-+h(2, "vector") Token.vector
+            |  #[code ancestor.is_ancestor(self)].
-    +tag property
-    +tag requires model
-
-p
-    |  A real-valued meaning representation.
-
-+aside-code("Example").
-    apple = nlp(u'apple')
-    (apple.vector.dtype, apple.vector.shape)
-    # (dtype('float32'), (300,))
-
-+table(["Name", "Type", "Description"])
-    +footrow
-        +cell returns
-        +cell #[code numpy.ndarray[ndim=1, dtype='float32']]
-        +cell A 1D numpy array representing the token's semantics.
-
-+h(2, "vector_norm") Span.vector_norm
-    +tag property
-    +tag requires model
-
-p
-    |  The L2 norm of the token's vector representation.
-
-+table(["Name", "Type", "Description"])
-    +footrow
-        +cell returns
-        +cell float
-        +cell The L2 norm of the vector representation.
 
 +h(2, "conjuncts") Token.conjuncts
     +tag property
+    +tag requires model: parse
 
 p A sequence of coordinated tokens, including the token itself.
 
++aside-code("Example").
+    doc = nlp(u'I like apples and oranges')
+    apples_conjuncts = doc[2].conjuncts
+    assert [t.text for t in apples_conjuncts] == [u'oranges']
+
 +table(["Name", "Type", "Description"])
     +footrow
         +cell yields
@@ -199,9 +184,15 @@ p A sequence of coordinated tokens, including the token itself.
 
 +h(2, "children") Token.children
     +tag property
+    +tag requires model: parse
 
 p A sequence of the token's immediate syntactic children.
 
++aside-code("Example").
+    doc = nlp(u'Give it back! He pleaded.')
+    give_children = doc[0].children
+    assert [t.text for t in give_children] == [u'it', u'back', u'!']
+
 +table(["Name", "Type", "Description"])
     +footrow
         +cell yields
@@ -210,27 +201,79 @@ p A sequence of the token's immediate syntactic children.
 
 +h(2, "subtree") Token.subtree
     +tag property
+    +tag requires model: parse
 
 p A sequence of all the token's syntactic descendents.
 
++aside-code("Example").
+    doc = nlp(u'Give it back! He pleaded.')
+    give_subtree = doc[0].subtree
+    assert [t.text for t in give_subtree] == [u'Give', u'it', u'back', u'!']
+
 +table(["Name", "Type", "Description"])
     +footrow
         +cell yields
         +cell #[code Token]
         +cell A descendant token such that #[code self.is_ancestor(descendant)].
 
-+h(2, "ancestors") Token.ancestors
++h(2, "has_vector") Token.has_vector
     +tag property
+    +tag requires model: vectors
 
-p The rightmost token of this token's syntactic descendants.
+p
+    |  A boolean value indicating whether a word vector is associated with the
+    |  token.
+
++aside-code("Example").
+    doc = nlp(u'I like apples')
+    apples = doc[2]
+    assert apples.has_vector
 
 +table(["Name", "Type", "Description"])
     +footrow
-        +cell yields
+        +cell returns
-        +cell #[code Token]
+        +cell bool
-        +cell
+        +cell Whether the token has a vector data attached.
-            |  A sequence of ancestor tokens such that
+
-            |  #[code ancestor.is_ancestor(self)].
++h(2, "vector") Token.vector
+    +tag property
+    +tag requires model: vectors
+
+p
+    |  A real-valued meaning representation.
+
++aside-code("Example").
+    doc = nlp(u'I like apples')
+    apples = doc[2]
+    assert apples.vector.dtype == 'float32'
+    assert apples.vector.shape == (300,)
+
++table(["Name", "Type", "Description"])
+    +footrow
+        +cell returns
+        +cell #[code numpy.ndarray[ndim=1, dtype='float32']]
+        +cell A 1D numpy array representing the token's semantics.
+
++h(2, "vector_norm") Span.vector_norm
+    +tag property
+    +tag requires model: vectors
+
+p
+    |  The L2 norm of the token's vector representation.
+
++aside-code("Example").
+    doc = nlp(u'I like apples and pasta')
+    apples = doc[2]
+    pasta = doc[4]
+    apples.vector_norm # 6.89589786529541
+    pasta.vector_norm # 7.759851932525635
+    assert apples.vector_norm != pasta.vector_norm
+
++table(["Name", "Type", "Description"])
+    +footrow
+        +cell returns
+        +cell float
+        +cell The L2 norm of the vector representation.
 
 +h(2, "attributes") Attributes