diff --git a/website/docs/usage/_spacy-101/_architecture.jade b/website/docs/usage/_spacy-101/_architecture.jade new file mode 100644 index 000000000..4905171e7 --- /dev/null +++ b/website/docs/usage/_spacy-101/_architecture.jade @@ -0,0 +1,83 @@ +//- 💫 DOCS > USAGE > SPACY 101 > ARCHITECTURE + +p + | The central data structures in spaCy are the #[code Doc] and the + | #[code Vocab]. The #[code Doc] object owns the + | #[strong sequence of tokens] and all their annotations. The #[code Vocab] + | object owns a set of #[strong look-up tables] that make common + | information available across documents. By centralising strings, word + | vectors and lexical attributes, we avoid storing multiple copies of this + | data. This saves memory, and ensures there's a + | #[strong single source of truth]. + +p + | Text annotations are also designed to allow a single source of truth: the + | #[code Doc] object owns the data, and #[code Span] and #[code Token] are + | #[strong views that point into it]. The #[code Doc] object is constructed + | by the #[code Tokenizer], and then #[strong modified in place] by the + | components of the pipeline. The #[code Language] object coordinates these + | components. It takes raw text and sends it through the pipeline, + | returning an #[strong annotated document]. It also orchestrates training + | and serialization. + ++image + include ../../../assets/img/docs/architecture.svg + .u-text-right + +button("/assets/img/docs/architecture.svg", false, "secondary").u-text-tag View large graphic + ++table(["Name", "Description"]) + +row + +cell #[+api("language") #[code Language]] + +cell + | A text-processing pipeline. Usually you'll load this once per + | process as #[code nlp] and pass the instance around your application. + + +row + +cell #[+api("doc") #[code Doc]] + +cell A container for accessing linguistic annotations. + + +row + +cell #[+api("span") #[code Span]] + +cell A slice from a #[code Doc] object. + + +row + +cell #[+api("token") #[code Token]] + +cell + | An individual token — i.e. a word, punctuation symbol, whitespace, + | etc. + + +row + +cell #[+api("lexeme") #[code Lexeme]] + +cell + | An entry in the vocabulary. It's a word type with no context, as + | opposed to a word token. It therefore has no part-of-speech tag, + | dependency parse etc. + + +row + +cell #[+api("vocab") #[code Vocab]] + +cell + | A lookup table for the vocabulary that allows you to access + | #[code Lexeme] objects. + + +row + +cell #[code Morphology] + +cell + | Assign linguistic features like lemmas, noun case, verb tense etc. + | based on the word and its part-of-speech tag. + + +row + +cell #[+api("stringstore") #[code StringStore]] + +cell Map strings to and from hash values. + + +row + +row + +cell #[+api("tokenizer") #[code Tokenizer]] + +cell + | Segment text, and create #[code Doc] objects with the discovered + | segment boundaries. + + +row + +cell #[+api("matcher") #[code Matcher]] + +cell + | Match sequences of tokens, based on pattern rules, similar to + | regular expressions. diff --git a/website/docs/usage/index.jade b/website/docs/usage/index.jade index d3deaa17e..817b08ba9 100644 --- a/website/docs/usage/index.jade +++ b/website/docs/usage/index.jade @@ -21,13 +21,8 @@ p +qs({config: 'venv', os: 'linux'}) source .env/bin/activate +qs({config: 'venv', os: 'windows'}) .env\Scripts\activate - +qs({config: 'gpu', os: 'mac'}) export CUDA_HOME=/usr/local/cuda-8.0 - +qs({config: 'gpu', os: 'mac'}) export PATH=$PATH:$CUDA_HOME/bin - +qs({config: 'gpu', os: 'linux'}) export CUDA_HOME=/usr/local/cuda-8.0 - +qs({config: 'gpu', os: 'linux'}) export PATH=$PATH:$CUDA_HOME/bin - +qs({config: 'gpu', package: 'pip'}) pip install -U chainer - +qs({config: 'gpu', package: 'source'}) pip install -U chainer - +qs({config: 'gpu', package: 'conda'}) conda install -c anaconda chainer + +qs({config: 'gpu', os: 'mac'}) export PATH=$PATH:/usr/local/cuda-8.0/bin + +qs({config: 'gpu', os: 'linux'}) export PATH=$PATH:/usr/local/cuda-8.0/bin +qs({package: 'pip'}) pip install -U spacy +qs({package: 'conda'}) conda install -c conda-forge spacy @@ -96,27 +91,15 @@ p | #[+a("http://chainer.org") Chainer]'s CuPy module, which provides | a NumPy-compatible interface for GPU arrays. -+aside("Why is this so complicated?") - | Installing Chainer when no GPU is available currently causes an - | error. We therefore do not specify Chainer as a dependency. However, - | CuPy will be split out into - | #[+a("https://www.slideshare.net/beam2d/chainer-v2-alpha/7") its own package] - | in Chainer v2.0. We'll have a smoother installation process for this - | in an upcoming version. - p | First, install follows the normal CUDA installation procedure. Next, set | your environment variables so that the installation will be able to find - | CUDA. Next, install Chainer, and check that CuPy can be imported - | correctly. Finally, install spaCy. + | CUDA. Finally, install spaCy. +code(false, "bash"). export CUDA_HOME=/usr/local/cuda-8.0 # Or wherever your CUDA is export PATH=$PATH:$CUDA_HOME/bin - pip install chainer - python -c "import cupy; assert cupy" # Check it installed - pip install spacy python -c "import thinc.neural.gpu_ops" # Check the GPU ops were built diff --git a/website/docs/usage/spacy-101.jade b/website/docs/usage/spacy-101.jade index 4f2642af0..5b7908651 100644 --- a/website/docs/usage/spacy-101.jade +++ b/website/docs/usage/spacy-101.jade @@ -65,13 +65,15 @@ p | not designed specifically for chat bots, and only provides the | underlying text processing capabilities. +item #[strong spaCy is not research software]. - | It's is built on the latest research, but unlike - | #[+a("https://github./nltk/nltk") NLTK], which is intended for - | teaching and research, spaCy follows a more opinionated approach and - | focuses on production usage. Its aim is to provide you with the best - | possible general-purpose solution for text processing and machine learning - | with text input – but this also means that there's only one implementation - | of each component. + | It's is built on the latest research, but it's designed to get + | things done. This leads to fairly different design decisions than + | #[+a("https://github./nltk/nltk") NLTK] + | or #[+a("https://stanfordnlp.github.io/CoreNLP/") CoreNLP], which were + | created as platforms for teaching and research. The main difference + | is that spaCy is integrated and opinionated. spaCy tries to avoid asking + | the user to choose between multiple algorithms that deliver equivalent + | functionality. Keeping the menu small lets spaCy deliver generally better + | performance and developer experience. +item #[strong spaCy is not a company]. | It's an open-source library. Our company publishing spaCy and other | software is called #[+a(COMPANY_URL, true) Explosion AI]. @@ -79,7 +81,7 @@ p +h(2, "features") Features p - | Across the documentations, you'll come across mentions of spaCy's + | Across the documentation, you'll come across mentions of spaCy's | features and capabilities. Some of them refer to linguistic concepts, | while others are related to more general machine learning functionality. @@ -171,7 +173,9 @@ p p | Even though a #[code Doc] is processed – e.g. split into individual words | and annotated – it still holds #[strong all information of the original text], - | like whitespace characters. This way, you'll never lose any information + | like whitespace characters. You can always get the offset of a token into the + | original string, or reconstruct the original by joining the tokens and their + | trailing whitespace. This way, you'll never lose any information | when processing text with spaCy. +h(3, "annotations-token") Tokenization @@ -268,69 +272,7 @@ include _spacy-101/_language-data +h(2, "architecture") Architecture -+under-construction - -+image - include ../../assets/img/docs/architecture.svg - .u-text-right - +button("/assets/img/docs/architecture.svg", false, "secondary").u-text-tag View large graphic - -+table(["Name", "Description"]) - +row - +cell #[+api("language") #[code Language]] - +cell - | A text-processing pipeline. Usually you'll load this once per - | process as #[code nlp] and pass the instance around your application. - - +row - +cell #[+api("doc") #[code Doc]] - +cell A container for accessing linguistic annotations. - - +row - +cell #[+api("span") #[code Span]] - +cell A slice from a #[code Doc] object. - - +row - +cell #[+api("token") #[code Token]] - +cell - | An individual token — i.e. a word, punctuation symbol, whitespace, - | etc. - - +row - +cell #[+api("lexeme") #[code Lexeme]] - +cell - | An entry in the vocabulary. It's a word type with no context, as - | opposed to a word token. It therefore has no part-of-speech tag, - | dependency parse etc. - - +row - +cell #[+api("vocab") #[code Vocab]] - +cell - | A lookup table for the vocabulary that allows you to access - | #[code Lexeme] objects. - - +row - +cell #[code Morphology] - +cell - | Assign linguistic features like lemmas, noun case, verb tense etc. - | based on the word and its part-of-speech tag. - - +row - +cell #[+api("stringstore") #[code StringStore]] - +cell Map strings to and from hash values. - - +row - +row - +cell #[+api("tokenizer") #[code Tokenizer]] - +cell - | Segment text, and create #[code Doc] objects with the discovered - | segment boundaries. - - +row - +cell #[+api("matcher") #[code Matcher]] - +cell - | Match sequences of tokens, based on pattern rules, similar to - | regular expressions. +include _spacy-101/_architecture.jade +h(3, "architecture-pipeline") Pipeline components