From 493d5ffb50815420155bbc0de98d9f4c307f7abf Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 2 Nov 2014 21:24:09 +1100 Subject: [PATCH] * Add test for '' in punct --- tests/test_post_punct.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_post_punct.py b/tests/test_post_punct.py index 5ee0eb066..da80f5636 100644 --- a/tests/test_post_punct.py +++ b/tests/test_post_punct.py @@ -39,3 +39,9 @@ def test_three_same_close(close_puncts): assert len(tokens) == 4 assert tokens[0].string == word_str assert tokens[1].string == p + + +def test_double_end_quote(): + assert len(EN.tokenize("Hello''")) == 2 + assert len(EN.tokenize("''")) == 1 +