interning strings to fix memory consumption

dockerfiles/Dockerfile

@@ -64,5 +64,6 @@ RUN apt-get install --yes \
 RUN pip install records
 RUN pip install babel
 RUN pip install tabulate
+RUN pip install pympler
 
 CMD bash 

text/torchtext/data/example.py

@@ -1,5 +1,6 @@
 import csv
 import json
+import sys
 
 import six
 
@@ -58,7 +59,7 @@ class Example(object):
             if field is not None:
                 if isinstance(val, six.string_types):
                     val = val.rstrip('\n')
-                setattr(ex, name, field.preprocess(val))
+                setattr(ex, name, [sys.intern(x) for x in field.preprocess(val)])
         return ex
 
     @classmethod