explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-11 17:56:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Move fix_text to spacy.compat (see #1002)

Browse Source
This commit is contained in:
ines 2017-04-20 15:47:17 +02:00
parent 60b5243bee
commit 25c70b4cc5
2 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spacy/cli/model.py
View File

@ -6,9 +6,9 @@ import math
from ast import literal_eval from ast import literal_eval
from pathlib import Path from pathlib import Path
from preshed.counter import PreshCounter from preshed.counter import PreshCounter
import ftfy
from ..vocab import write_binary_vectors from ..vocab import write_binary_vectors
from ..compat import fix_text
from .. import util from .. import util
@ -77,7 +77,7 @@ def read_clusters(clusters_path):
for line in f: for line in f:
try: try:
cluster, word, freq = line.split() cluster, word, freq = line.split()
word = ftfy.fix_text(word) word = fix_text(word)
except ValueError: except ValueError:
continue continue
# If the clusterer has only seen the word a few times, its # If the clusterer has only seen the word a few times, its

4
spacy/compat.py
View File

@ -2,6 +2,7 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import six import six
import ftfy
import sys import sys
import ujson import ujson
@ -38,6 +39,9 @@ elif is_python3:
json_dumps = lambda data: ujson.dumps(data, indent=2) json_dumps = lambda data: ujson.dumps(data, indent=2)
fix_text = lambda text: ftfy.fix_text(text)
def symlink_to(orig, dest): def symlink_to(orig, dest):
if is_python2 and is_windows: if is_python2 and is_windows:
import subprocess import subprocess