spaCy/spacy/en/language_data.py

# coding: utf8
from __future__ import unicode_literals

from .. import language_data as base
from ..language_data import update_exc, strings_to_exc, expand_exc
from ..symbols import ORTH, LEMMA

from .tag_map import TAG_MAP
from .word_sets import STOP_WORDS, NUM_WORDS
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS, ORTH_ONLY
from .morph_rules import MORPH_RULES


TAG_MAP = dict(TAG_MAP)
STOP_WORDS = set(STOP_WORDS)


TOKENIZER_EXCEPTIONS = dict(TOKENIZER_EXCEPTIONS)
update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(ORTH_ONLY))
update_exc(TOKENIZER_EXCEPTIONS, expand_exc(TOKENIZER_EXCEPTIONS, "'", "’"))
update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(base.EMOTICONS))
update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(base.ABBREVIATIONS))


__all__ = ["TOKENIZER_EXCEPTIONS", "TAG_MAP", "STOP_WORDS", "MORPH_RULES"]
-												Use consistent unicode declarations

											
										
										
											2017-03-12 12:07:28 +00:00
+								# coding: utf8
-												Add English language data, so that the tokenizer doesn't require the data download

											
										
										
											2016-09-25 12:49:00 +00:00
+								from __future__ import unicode_literals
-												Reorganise language data

											
										
										
											2016-12-18 15:54:19 +00:00
+								from .. import language_data as base
 								from ..language_data import update_exc, strings_to_exc, expand_exc
 								from ..symbols import ORTH, LEMMA
-												Update English language data

											
										
										
											2016-12-18 14:36:50 +00:00
 								from .tag_map import TAG_MAP
-												Rename stop_words.py to word_sets.py and include more sets

NUM_WORDS and ORDINAL_WORDS are currently not used, but the hard-coded
list should be removed from orth.pyx and replaced to use
language-specific functions. This will later allow other languages to
use their own functions to set those flags. (In English, this is easier
because it only needs to be checked against a set – in German for
example, this requires a more complex function, as most number words
are one word.)

											
										
										
											2017-03-12 12:53:46 +00:00
+								from .word_sets import STOP_WORDS, NUM_WORDS
-												Update English language data

											
										
										
											2016-12-18 14:36:50 +00:00
+								from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS, ORTH_ONLY
 								from .morph_rules import MORPH_RULES
-												Reorganize exceptions for English and German

											
										
										
											2016-12-08 12:58:32 +00:00
-												Reorganise language data

											
										
										
											2016-12-18 15:54:19 +00:00
+								TAG_MAP = dict(TAG_MAP)
 								STOP_WORDS = set(STOP_WORDS)
-												Add more custom rules for abbreviations

											
										
										
											2016-12-08 11:47:01 +00:00
-												Add English language data, so that the tokenizer doesn't require the data download

											
										
										
											2016-09-25 12:49:00 +00:00
-												Use global abbreviation data languages and remove duplicates

											
										
										
											2017-01-08 19:36:00 +00:00
+								TOKENIZER_EXCEPTIONS = dict(TOKENIZER_EXCEPTIONS)
-												Reorganise language data

											
										
										
											2016-12-18 15:54:19 +00:00
+								update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(ORTH_ONLY))
 								update_exc(TOKENIZER_EXCEPTIONS, expand_exc(TOKENIZER_EXCEPTIONS, "'", "’"))
 								update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(base.EMOTICONS))
-												Use global abbreviation data languages and remove duplicates

											
										
										
											2017-01-08 19:36:00 +00:00
+								update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(base.ABBREVIATIONS))
-												Update English language data

											
										
										
											2016-12-07 19:29:52 +00:00
-												Add Python-formatted lemmatizer data and rules

											
										
										
											2017-03-12 12:46:00 +00:00
+								__all__ = ["TOKENIZER_EXCEPTIONS", "TAG_MAP", "STOP_WORDS", "MORPH_RULES"]