Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert morphology serialisation

This commit is contained in:
Matthew Honnibal 2019-08-30 13:13:07 +02:00
parent efcb51ddc8
commit 67c3d03905
2 changed files with 0 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
spacy/morphology.pyx
View File

@ -326,30 +326,6 @@ cdef class Morphology:
for form_str, attrs in entries.items(): for form_str, attrs in entries.items():
self.add_special_case(tag_str, form_str, attrs) self.add_special_case(tag_str, form_str, attrs)
def to_bytes(self, exclude=tuple(), **kwargs):
exceptions = {}
for (tag_str, orth_int), attrs in sorted(self.exc.items()):
exceptions.setdefault(tag_str, {})
exceptions[tag_str][self.strings[orth_int]] = attrs
data = {"tag_map": self.tag_map, "exceptions": exceptions}
return srsly.msgpack_dumps(data)
def from_bytes(self, byte_string):
msg = srsly.msgpack_loads(byte_string)
self._load_from_tag_map(msg["tag_map"])
self.load_morph_exceptions(msg["exceptions"])
return self
def to_disk(self, path, exclude=tuple(), **kwargs):
path = ensure_path(path)
with path.open("wb") as file_:
file_.write(self.to_bytes())
def from_disk(self, path):
with path.open("rb") as file_:
byte_string = file_.read()
return self.from_bytes(byte_string)
@classmethod @classmethod
def create_class_map(cls): def create_class_map(cls):
return MorphologyClassMap(FEATURES) return MorphologyClassMap(FEATURES)

6
spacy/vocab.pyx
View File

@ -433,8 +433,6 @@ cdef class Vocab:
file_.write(self.lexemes_to_bytes()) file_.write(self.lexemes_to_bytes())
if "vectors" not in "exclude" and self.vectors is not None: if "vectors" not in "exclude" and self.vectors is not None:
self.vectors.to_disk(path) self.vectors.to_disk(path)
if "morphology" not in exclude:
self.morphology.to_disk(path / "morphology.bin")
def from_disk(self, path, exclude=tuple(), **kwargs): def from_disk(self, path, exclude=tuple(), **kwargs):
"""Loads state from a directory. Modifies the object in place and """Loads state from a directory. Modifies the object in place and
@ -459,8 +457,6 @@ cdef class Vocab:
self.vectors.from_disk(path, exclude=["strings"]) self.vectors.from_disk(path, exclude=["strings"])
if self.vectors.name is not None: if self.vectors.name is not None:
link_vectors_to_models(self) link_vectors_to_models(self)
if "morphology" not in exclude:
self.morphology.from_disk(path / "morphology.bin")
return self return self
def to_bytes(self, exclude=tuple(), **kwargs): def to_bytes(self, exclude=tuple(), **kwargs):
@ -481,7 +477,6 @@ cdef class Vocab:
("strings", lambda: self.strings.to_bytes()), ("strings", lambda: self.strings.to_bytes()),
("lexemes", lambda: self.lexemes_to_bytes()), ("lexemes", lambda: self.lexemes_to_bytes()),
("vectors", deserialize_vectors), ("vectors", deserialize_vectors),
("morphology", lambda: self.morphology.to_bytes())
)) ))
exclude = util.get_serialization_exclude(getters, exclude, kwargs) exclude = util.get_serialization_exclude(getters, exclude, kwargs)
return util.to_bytes(getters, exclude) return util.to_bytes(getters, exclude)
@ -505,7 +500,6 @@ cdef class Vocab:
("strings", lambda b: self.strings.from_bytes(b)), ("strings", lambda b: self.strings.from_bytes(b)),
("lexemes", lambda b: self.lexemes_from_bytes(b)), ("lexemes", lambda b: self.lexemes_from_bytes(b)),
("vectors", lambda b: serialize_vectors(b)), ("vectors", lambda b: serialize_vectors(b)),
("morphology", lambda b: self.morphology.from_bytes(b))
)) ))
exclude = util.get_serialization_exclude(setters, exclude, kwargs) exclude = util.get_serialization_exclude(setters, exclude, kwargs)
util.from_bytes(bytes_data, setters, exclude) util.from_bytes(bytes_data, setters, exclude)