From de01e427fd1d7337b19c7bdb0749dc301af70106 Mon Sep 17 00:00:00 2001
From: Matthew Honnibal <honnibal+gh@gmail.com>
Date: Fri, 30 Sep 2016 20:19:42 +0200
Subject: [PATCH] Revert "Changes to strings.pyx for new StringStore scheme"

This reverts commit 22d4752d64d9ced99e24f8ed20bea942fae27c5b.
---
 spacy/strings.pxd |  2 +-
 spacy/strings.pyx | 26 ++++++++++----------------
 2 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/spacy/strings.pxd b/spacy/strings.pxd
index 5c5fe9c35..df61712c8 100644
--- a/spacy/strings.pxd
+++ b/spacy/strings.pxd
@@ -28,6 +28,6 @@ cdef class StringStore:
 
     cpdef int remove_oov_map(self, Pool mem) except -1
 
-    cpdef attr_t intern(self, basestring base_string, Pool mem=*) except -1
+    cdef hash_t intern(self, unicode py_string, Pool mem=*) except UINT64_MAX
     cdef const Utf8Str* _intern_utf8(self, const unsigned char* utf8_string,
                                      int length) except NULL
diff --git a/spacy/strings.pyx b/spacy/strings.pyx
index 63dd3206f..509b475e9 100644
--- a/spacy/strings.pyx
+++ b/spacy/strings.pyx
@@ -86,7 +86,7 @@ cdef class StringStore:
         self.size = 1
         if strings is not None:
             for string in strings:
-                self.intern(string)
+                _ = self[string]
 
     property size:
         def __get__(self):
@@ -115,7 +115,7 @@ cdef class StringStore:
             byte_string = <bytes>string_or_id
             if len(byte_string) == 0:
                 return 0
-            key = <attr_t>_hash_utf8(byte_string, len(byte_string))
+            key = _hash_utf8(byte_string, len(byte_string))
             utf8str = <Utf8Str*>self._map.get(key)
             if utf8str is NULL:
                 raise KeyError(byte_string)
@@ -124,7 +124,7 @@ cdef class StringStore:
         elif isinstance(string_or_id, unicode):
             if len(<unicode>string_or_id) == 0:
                 return 0
-            key = <attr_t>hash_string(string_or_id)
+            key = hash_string(string_or_id)
             utf8str = <Utf8Str*>self._map.get(key)
             if utf8str is NULL:
                 raise KeyError(string_or_id)
@@ -136,7 +136,7 @@ cdef class StringStore:
     def __contains__(self, unicode string not None):
         if len(string) == 0:
             return True
-        cdef attr_t key = <attr_t>hash_string(string)
+        cdef hash_t key = hash_string(string)
         return self._map.get(key) is not NULL
 
     def __iter__(self):
@@ -154,17 +154,12 @@ cdef class StringStore:
             strings.append(py_string)
         return (StringStore, (strings,), None, None, None)
 
-    cpdef attr_t intern(self, basestring base_string, Pool mem=None) except -1:
-        if len(base_string) == 0:
-            return 0
+    cdef hash_t intern(self, unicode py_string, Pool mem=None) except UINT64_MAX:
         if mem is None:
             mem = self.mem
-        if isinstance(base_string, unicode):
-            byte_string = base_string.encode('utf8')
-        else:
-            byte_string = base_string
         cdef hash_t map_key = id(mem)
-        cdef attr_t key = <attr_t>_hash_utf8(byte_string, len(byte_string))
+        cdef bytes byte_string = py_string.encode('utf8')
+        cdef hash_t key = _hash_utf8(byte_string, len(byte_string))
         cdef const Utf8Str* utf8str = <Utf8Str*>self._map.get(key)
         cdef hash_t map_id = id(mem)
         cdef MapStruct* oov_map
@@ -184,7 +179,7 @@ cdef class StringStore:
             map_set(mem, oov_map, key, new_utf8str)
             return key
 
-    def decode_int(self, attr_t int_, Pool mem=None):
+    def decode_int(self, hash_t int_, Pool mem=None):
         cdef hash_t map_key
         if int_ == 0:
             return u''
@@ -218,16 +213,15 @@ cdef class StringStore:
             int length) except NULL:
         if self.size == self._resize_at:
             self._realloc()
-        cdef attr_t key = <attr_t>_hash_utf8(utf8_string, length)
+        key = _hash_utf8(utf8_string, length)
         self.c[self.size] = _allocate(self.mem, utf8_string, length)
         self._map.set(key, <void*>&self.c[self.size])
         self.size += 1
         return &self.c[self.size-1]
 
     cpdef int remove_oov_map(self, Pool mem) except -1:
-        print("Remove mem", id(mem))
         cdef hash_t key = id(mem)
-        self.oov_maps.pop(key)
+        self._maps.pop(key)
 
     def dump(self, file_):
         # TODO: Is it problematic that we don't save the OOV strings? No, right?