From fdfbacb710489979baee7a79909536dffb48c7eb Mon Sep 17 00:00:00 2001 From: Sina Date: Sat, 7 Nov 2020 18:52:13 -0800 Subject: [PATCH] Fix bug --- genienlp/data_utils/numericalizer/transformer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/genienlp/data_utils/numericalizer/transformer.py b/genienlp/data_utils/numericalizer/transformer.py index 5af028ff..3e25ee69 100644 --- a/genienlp/data_utils/numericalizer/transformer.py +++ b/genienlp/data_utils/numericalizer/transformer.py @@ -430,7 +430,11 @@ class BartNumericalizer(TransformerNumericalizer): def load(self, save_dir): - raise NotImplementedError + self._tokenizer = BartTokenizer.from_pretrained(save_dir) + self.decoder_vocab = DecoderVocabulary(self._tokenizer.decoder.values(), None, pad_token=self._tokenizer.pad_token, eos_token=self._tokenizer.eos_token) + + def save(self, save_dir): + self._tokenizer.save_pretrained(save_dir) def build_vocab(self, vocab_fields, vocab_sets): raise NotImplementedError