diff --git a/spacy/orth.py b/spacy/orth.py
index 2400b38a6..7023f95c5 100644
--- a/spacy/orth.py
+++ b/spacy/orth.py
@@ -155,5 +155,5 @@ def word_shape(string):
 def asciied(string):
     ascii_string = unidecode(string)
     if not ascii_string:
-        return '???'
-    return ascii_string.decode('ascii')
+        return b'???'
+    return ascii_string