From 09a02276566b1637f3dfefa05be5dd50d9628902 Mon Sep 17 00:00:00 2001
From: Matthew Honnibal <honnibal+gh@gmail.com>
Date: Thu, 15 Nov 2018 23:18:35 +0000
Subject: [PATCH] Temporarily add a script to load reddit

---
 bin/load_reddit.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 bin/load_reddit.py

diff --git a/bin/load_reddit.py b/bin/load_reddit.py
new file mode 100644
index 000000000..1b534bcc7
--- /dev/null
+++ b/bin/load_reddit.py
@@ -0,0 +1,87 @@
+# coding: utf8
+from __future__ import unicode_literals
+
+import bz2
+import regex as re
+import ujson
+import sys
+import random
+import datetime
+import plac
+from pathlib import Path
+
+_unset = object()
+
+
+class Reddit(object):
+    """Stream cleaned comments from Reddit."""
+    pre_format_re = re.compile(r'^[\`\*\~]')
+    post_format_re = re.compile(r'[\`\*\~]$')
+    url_re = re.compile(r'\[([^]]+)\]\(%%URL\)')
+    link_re = re.compile(r'\[([^]]+)\]\(https?://[^\)]+\)')
+
+    def __init__(self, file_path, meta_keys={'subreddit': 'section'}):
+        """
+        file_path (unicode / Path): Path to archive or directory of archives.
+        meta_keys (dict): Meta data key included in the Reddit corpus, mapped
+            to display name in Prodigy meta.
+        RETURNS (Reddit): The Reddit loader.
+        """
+        self.meta = meta_keys
+        file_path = Path(file_path)
+        if not file_path.exists():
+            raise IOError("Can't find file path: {}".format(file_path))
+        if not file_path.is_dir():
+            self.files = [file_path]
+        else:
+            self.files = list(file_path.iterdir())
+
+    def __iter__(self):
+        for file_path in self.iter_files():
+            with bz2.open(str(file_path)) as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    comment = ujson.loads(line)
+                    if self.is_valid(comment):
+                        text = self.strip_tags(comment['body'])
+                        yield {'text': text}
+
+    def get_meta(self, item):
+        return {name: item.get(key, 'n/a') for key, name in self.meta.items()}
+
+    def iter_files(self):
+        for file_path in self.files:
+            yield file_path
+
+    def strip_tags(self, text):
+        text = self.link_re.sub(r'\1', text)
+        text = text.replace('&gt;', '>').replace('&lt;', '<')
+        text = self.pre_format_re.sub('', text)
+        text = self.post_format_re.sub('', text)
+        text = re.sub(r'\s+', ' ', text)
+        return text.strip()
+
+    def is_valid(self, comment):
+        return comment['body'] is not None \
+            and comment['body'] != '[deleted]' \
+            and comment['body'] != '[removed]'
+
+
+def main(path):
+    reddit = Reddit(path)
+    for comment in reddit:
+        print(ujson.dumps(comment))
+
+
+if __name__ == '__main__':
+    try:
+        plac.call(main)
+    except BrokenPipeError: 
+        import os, sys
+        # Python flushes standard streams on exit; redirect remaining output
+        # to devnull to avoid another BrokenPipeError at shutdown
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, sys.stdout.fileno())
+        sys.exit(1)  # Python exits with error code 1 on EPIPE