diff --git a/spacy/tests/regression/test_issue3869.py b/spacy/tests/regression/test_issue3869.py index 72a485042..d76da6989 100644 --- a/spacy/tests/regression/test_issue3869.py +++ b/spacy/tests/regression/test_issue3869.py @@ -11,13 +11,17 @@ from spacy.lang.en import English "sentence", [ 'The story was to the effect that a young American student recently called on Professor Christlieb with a letter of introduction.', - 'The next month Barry Siddall joined Stoke City on a free transfer, after Chris Pearce had established himself as the Vale\'s #1.' + 'The next month Barry Siddall joined Stoke City on a free transfer, after Chris Pearce had established himself as the Vale\'s #1.', + 'The next month Barry Siddall joined Stoke City on a free transfer, after Chris Pearce had established himself as the Vale\'s number one', + 'Indeed, making the one who remains do all the work has installed him into a position of such insolent tyranny, it will take a month at least to reduce him to his proper proportions.', + "It was a missed assignment, but it shouldn't have resulted in a turnover ..." ], ) def test_issue3869(sentence): """Test that the Doc's count_by function works consistently""" nlp = English() + print() doc = nlp(sentence) count = 0 diff --git a/spacy/tokens/doc.pxd b/spacy/tokens/doc.pxd index 7cdc2316a..cc05cb495 100644 --- a/spacy/tokens/doc.pxd +++ b/spacy/tokens/doc.pxd @@ -8,6 +8,7 @@ from ..typedefs cimport attr_t from ..attrs cimport attr_id_t + cdef attr_t get_token_attr(const TokenC* token, attr_id_t feat_name) nogil diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index c77e5c44e..657b9a1d6 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -13,6 +13,7 @@ from libc.math cimport sqrt import numpy import numpy.linalg import struct +from libc.stdint cimport int64_t import srsly from thinc.neural.util import get_array_module, copy_array @@ -710,22 +711,52 @@ cdef class Doc: cdef int i cdef attr_t attr cdef size_t count + cdef int64_t this_value + + print("COUNTING") if counts is None: counts = PreshCounter() output_dict = True + print("counts None") else: output_dict = False # Take this check out of the loop, for a bit of extra speed if exclude is None: + print("exclude None") for i in range(self.length): - counts.inc(get_token_attr(&self.c[i], attr_id), 1) + print() + print("token", self[i]) + this_value = get_token_attr(&self.c[i], attr_id) + print("token attr value", this_value) + print("type attr value", type(this_value)) + + print(i, "key this_value before", counts.c_map.cells[this_value].key) + print(i, "value this_value before", counts.c_map.cells[this_value].value) + counts.inc(this_value, 1) + print(i, "key this_value after", counts.c_map.cells[this_value].key) + print(i, "value this_value after", counts.c_map.cells[this_value].value) + + print(i, "key 0", counts.c_map.cells[0].key) + print(i, "value 0", counts.c_map.cells[0].value) + print(i, "key 1", counts.c_map.cells[1].key) + print(i, "value 1", counts.c_map.cells[1].value) else: for i in range(self.length): if not exclude(self[i]): attr = get_token_attr(&self.c[i], attr_id) counts.inc(attr, 1) if output_dict: + print("output_dict") + print(counts.length) + print(counts.total) + print("key 0", counts.c_map.cells[0].key) + print("value 0", counts.c_map.cells[0].value) + print("key 1", counts.c_map.cells[1].key) + print("value 1", counts.c_map.cells[1].value) + print() + print(dict(counts)) + print() return dict(counts) def _realloc(self, new_size):