Merge branch 'master' into spacy.io

This commit is contained in:
Ines Montani 2019-03-22 18:36:28 +01:00
commit a9ad735241

View File

@ -404,7 +404,7 @@ class BadHTMLMerger(object):
for match_id, start, end in matches: for match_id, start, end in matches:
spans.append(doc[start:end]) spans.append(doc[start:end])
with doc.retokenize() as retokenizer: with doc.retokenize() as retokenizer:
for span in spans: for span in hashtags:
retokenizer.merge(span) retokenizer.merge(span)
for token in span: for token in span:
token._.bad_html = True # Mark token as bad HTML token._.bad_html = True # Mark token as bad HTML