From 94b0435e01e3c99e3f616dfd1ee10ec7221e4821 Mon Sep 17 00:00:00 2001
From: Yomguithereal <guillaumeplique@gmail.com>
Date: Thu, 27 May 2021 17:31:22 +0200
Subject: [PATCH] Note

---
 fog/tokenizers/words.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fog/tokenizers/words.py b/fog/tokenizers/words.py
index 83d6712..ca21436 100644
--- a/fog/tokenizers/words.py
+++ b/fog/tokenizers/words.py
@@ -645,6 +645,7 @@ class WordTokenizer(object):
                     token_value = token_value[1:]
                     token_changed = True
 
+            # NOTE: not an `elif` here because the token type can change is hashtags are splitted
             if token_type == 'word':
                 if self.lower:
                     token_value = token_value.lower()