mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-11 04:08:09 +03:00
f37863093a
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉 See here: https://github.com/explosion/srsly Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place. At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel. srsly currently includes forks of the following packages: ujson msgpack msgpack-numpy cloudpickle * WIP: replace json/ujson with srsly * Replace ujson in examples Use regular json instead of srsly to make code easier to read and follow * Update requirements * Fix imports * Fix typos * Replace msgpack with srsly * Fix warning
98 lines
2.9 KiB
Python
98 lines
2.9 KiB
Python
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
|
|
import bz2
|
|
import regex as re
|
|
import srsly
|
|
import sys
|
|
import random
|
|
import datetime
|
|
import plac
|
|
from pathlib import Path
|
|
|
|
_unset = object()
|
|
|
|
|
|
class Reddit(object):
|
|
"""Stream cleaned comments from Reddit."""
|
|
|
|
pre_format_re = re.compile(r"^[\`\*\~]")
|
|
post_format_re = re.compile(r"[\`\*\~]$")
|
|
url_re = re.compile(r"\[([^]]+)\]\(%%URL\)")
|
|
link_re = re.compile(r"\[([^]]+)\]\(https?://[^\)]+\)")
|
|
|
|
def __init__(self, file_path, meta_keys={"subreddit": "section"}):
|
|
"""
|
|
file_path (unicode / Path): Path to archive or directory of archives.
|
|
meta_keys (dict): Meta data key included in the Reddit corpus, mapped
|
|
to display name in Prodigy meta.
|
|
RETURNS (Reddit): The Reddit loader.
|
|
"""
|
|
self.meta = meta_keys
|
|
file_path = Path(file_path)
|
|
if not file_path.exists():
|
|
raise IOError("Can't find file path: {}".format(file_path))
|
|
if not file_path.is_dir():
|
|
self.files = [file_path]
|
|
else:
|
|
self.files = list(file_path.iterdir())
|
|
|
|
def __iter__(self):
|
|
for file_path in self.iter_files():
|
|
with bz2.open(str(file_path)) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
comment = srsly.json_loads(line)
|
|
if self.is_valid(comment):
|
|
text = self.strip_tags(comment["body"])
|
|
yield {"text": text}
|
|
|
|
def get_meta(self, item):
|
|
return {name: item.get(key, "n/a") for key, name in self.meta.items()}
|
|
|
|
def iter_files(self):
|
|
for file_path in self.files:
|
|
yield file_path
|
|
|
|
def strip_tags(self, text):
|
|
text = self.link_re.sub(r"\1", text)
|
|
text = text.replace(">", ">").replace("<", "<")
|
|
text = self.pre_format_re.sub("", text)
|
|
text = self.post_format_re.sub("", text)
|
|
text = re.sub(r"\s+", " ", text)
|
|
return text.strip()
|
|
|
|
def is_valid(self, comment):
|
|
return (
|
|
comment["body"] is not None
|
|
and comment["body"] != "[deleted]"
|
|
and comment["body"] != "[removed]"
|
|
)
|
|
|
|
|
|
def main(path):
|
|
reddit = Reddit(path)
|
|
for comment in reddit:
|
|
print(srsly.json_dumps(comment))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import socket
|
|
|
|
try:
|
|
BrokenPipeError
|
|
except NameError:
|
|
BrokenPipeError = socket.error
|
|
try:
|
|
plac.call(main)
|
|
except BrokenPipeError:
|
|
import os, sys
|
|
|
|
# Python flushes standard streams on exit; redirect remaining output
|
|
# to devnull to avoid another BrokenPipeError at shutdown
|
|
devnull = os.open(os.devnull, os.O_WRONLY)
|
|
os.dup2(devnull, sys.stdout.fileno())
|
|
sys.exit(1) # Python exits with error code 1 on EPIPE
|