explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-24 17:06:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

tracked the bug down to PreshCounter.inc - still unclear what goes wrong

Browse Source
This commit is contained in:
svlandeg 2019-07-11 01:53:06 +02:00
parent a89fecce97
commit e080412385
3 changed files with 38 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spacy/tests/regression/test_issue3869.py
View File

@ -11,13 +11,17 @@ from spacy.lang.en import English
"sentence",
[
'The story was to the effect that a young American student recently called on Professor Christlieb with a letter of introduction.',
'The next month Barry Siddall joined Stoke City on a free transfer, after Chris Pearce had established himself as the Vale\'s #1.'
'The next month Barry Siddall joined Stoke City on a free transfer, after Chris Pearce had established himself as the Vale\'s #1.',
'The next month Barry Siddall joined Stoke City on a free transfer, after Chris Pearce had established himself as the Vale\'s number one',
'Indeed, making the one who remains do all the work has installed him into a position of such insolent tyranny, it will take a month at least to reduce him to his proper proportions.',
"It was a missed assignment, but it shouldn't have resulted in a turnover ..."
],
)
def test_issue3869(sentence):
"""Test that the Doc's count_by function works consistently"""
nlp = English()
print()
doc = nlp(sentence)
count = 0

1
spacy/tokens/doc.pxd
View File

@ -8,6 +8,7 @@ from ..typedefs cimport attr_t
from ..attrs cimport attr_id_t
cdef attr_t get_token_attr(const TokenC* token, attr_id_t feat_name) nogil

33
spacy/tokens/doc.pyx
View File

@ -13,6 +13,7 @@ from libc.math cimport sqrt
import numpy
import numpy.linalg
import struct
from libc.stdint cimport int64_t
import srsly
from thinc.neural.util import get_array_module, copy_array
@ -710,22 +711,52 @@ cdef class Doc:
cdef int i
cdef attr_t attr
cdef size_t count
cdef int64_t this_value
print("COUNTING")
if counts is None:
counts = PreshCounter()
output_dict = True
print("counts None")
else:
output_dict = False
# Take this check out of the loop, for a bit of extra speed
if exclude is None:
print("exclude None")
for i in range(self.length):
counts.inc(get_token_attr(&self.c[i], attr_id), 1)
print()
print("token", self[i])
this_value = get_token_attr(&self.c[i], attr_id)
print("token attr value", this_value)
print("type attr value", type(this_value))
print(i, "key this_value before", counts.c_map.cells[this_value].key)
print(i, "value this_value before", <int64_t>counts.c_map.cells[this_value].value)
counts.inc(this_value, 1)
print(i, "key this_value after", counts.c_map.cells[this_value].key)
print(i, "value this_value after", <int64_t>counts.c_map.cells[this_value].value)
print(i, "key 0", counts.c_map.cells[0].key)
print(i, "value 0", <int64_t>counts.c_map.cells[0].value)
print(i, "key 1", counts.c_map.cells[1].key)
print(i, "value 1", <int64_t>counts.c_map.cells[1].value)
else:
for i in range(self.length):
if not exclude(self[i]):
attr = get_token_attr(&self.c[i], attr_id)
counts.inc(attr, 1)
if output_dict:
print("output_dict")
print(counts.length)
print(counts.total)
print("key 0", counts.c_map.cells[0].key)
print("value 0", <int64_t>counts.c_map.cells[0].value)
print("key 1", counts.c_map.cells[1].key)
print("value 1", <int64_t>counts.c_map.cells[1].value)
print()
print(dict(counts))
print()
return dict(counts)
def _realloc(self, new_size):