diff --git a/fog/tokenizers/words.py b/fog/tokenizers/words.py index 83d6712..ca21436 100644 --- a/fog/tokenizers/words.py +++ b/fog/tokenizers/words.py @@ -645,6 +645,7 @@ class WordTokenizer(object): token_value = token_value[1:] token_changed = True + # NOTE: not an `elif` here because the token type can change is hashtags are splitted if token_type == 'word': if self.lower: token_value = token_value.lower()