From 7e8488d1a01bdb1faa5175f6fa40fd3b84b22fce Mon Sep 17 00:00:00 2001
From: Erez Shinan <erezshin+git@gmail.com>
Date: Thu, 15 Aug 2019 18:06:42 +0200
Subject: [PATCH] Fixed issue #425, keeping in mind unicode issue #411

---
 lark/lexer.py        | 2 +-
 lark/load_grammar.py | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lark/lexer.py b/lark/lexer.py
index 0966a81..48d8904 100644
--- a/lark/lexer.py
+++ b/lark/lexer.py
@@ -101,7 +101,7 @@ class Token(Str):
 
         self.type = type_
         self.pos_in_stream = pos_in_stream
-        self.value = Str(value)
+        self.value = value
         self.line = line
         self.column = column
         self.end_line = end_line
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index 8cd5742..12ae38f 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -12,7 +12,7 @@ from .parse_tree_builder import ParseTreeBuilder
 from .parser_frontends import LALR_TraditionalLexer
 from .common import LexerConf, ParserConf
 from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
-from .utils import classify, suppress, dedup_list
+from .utils import classify, suppress, dedup_list, Str
 from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken
 
 from .tree import Tree, SlottedTree as ST
@@ -451,9 +451,9 @@ class PrepareSymbols(Transformer_InPlace):
         if isinstance(v, Tree):
             return v
         elif v.type == 'RULE':
-            return NonTerminal(v.value)
+            return NonTerminal(Str(v.value))
         elif v.type == 'TERMINAL':
-            return Terminal(v.value, filter_out=v.startswith('_'))
+            return Terminal(Str(v.value), filter_out=v.startswith('_'))
         assert False
 
 def _choice_of_rules(rules):