spaCy/website/docs/usage/spacy-101.jade

//- 💫 DOCS > USAGE > SPACY 101

include ../../_includes/_mixins

+h(2, "features") Features

+under-construction

+aside
    |  If one of spaCy's functionalities #[strong needs a model], it means that
    |  you need to have one our the available
    |  #[+a("/docs/usage/models") statistical models] installed. Models are used
    |  to #[strong predict] linguistic annotations – for example, if a word is
    |  a verb or a noun.

+table(["Name", "Description", "Needs model"])
    +row
        +cell #[strong Tokenization]
        +cell Segmenting text into words, punctuations marks etc.
        +cell #[+procon("con")]

    +row
        +cell #[strong Part-of-speech] (POS) #[strong Tagging]
        +cell Assigning word types to tokens, like verb or noun.
        +cell #[+procon("pro")]

    +row
        +cell #[strong Dependency Parsing]
        +cell
            |  Assigning syntactic dependency labels, i.e. the relations between
            |  individual tokens.
        +cell #[+procon("pro")]

    +row
        +cell #[strong Sentence Boundary Detection] (SBD)
        +cell Finding and segmenting individual sentences.
        +cell #[+procon("pro")]

    +row
        +cell #[strong Named Entity Recongition] (NER)
        +cell
            |  Labelling named "real-world" objects, like persons, companies or
            |  locations.
        +cell #[+procon("pro")]

    +row
        +cell #[strong Rule-based Matching]
        +cell
            |  Finding sequences of tokens based on their texts and linguistic
            |  annotations, similar to regular expressions.
        +cell #[+procon("con")]

    +row
        +cell #[strong Similarity]
        +cell
            |  Comparing words, text spans and documents and how similar they
            |  are to each other.
        +cell #[+procon("pro")]

    +row
        +cell #[strong Training]
        +cell Updating and improving a statistical model's predictions.
        +cell #[+procon("neutral")]

    +row
        +cell #[strong Serialization]
        +cell Saving objects to files or byte strings.
        +cell #[+procon("neutral")]

+h(2, "annotations") Linguistic annotations

p
    |  spaCy provides a variety of linguistic annotations to give you
    |  #[strong insights into a text's grammatical structure]. This includes the
    |  word types, like the parts of speech, and how the words are related to
    |  each other. For example, if you're analysing text, it makes a huge
    |  difference whether a noun is the subject of a sentence, or the object –
    |  or whether "google" is used as a verb, or refers to the website or
    |  company in a specific context.

p
    |  Once you've downloaded and installed a #[+a("/docs/usage/models") model],
    |  you can load it via #[+api("spacy#load") #[code spacy.load()]]. This will
    |  return a #[code Language] object contaning all components and data needed
    |  to process text. We usually call it #[code nlp]. Calling the #[code nlp]
    |  object on a string of text will return a processed #[code Doc]:

+code.
    import spacy

    nlp = spacy.load('en')
    doc = nlp(u'Apple is looking at buying U.K. startup for $1 billion')

p
    |  Even though a #[code Doc] is processed – e.g. split into individual words
    |  and annotated – it still holds #[strong all information of the original text],
    |  like whitespace characters. This way, you'll never lose any information
    |  when processing text with spaCy.

+h(3, "annotations-token") Tokenization

include _spacy-101/_tokenization

+infobox
    |  To learn more about how spaCy's tokenization rules work in detail,
    |  how to #[strong customise and replace] the default tokenizer and how to
    |  #[strong add language-specific data], see the usage guides on
    |  #[+a("/docs/usage/adding-languages") adding languages] and
    |  #[+a("/docs/usage/customizing-tokenizer") customising the tokenizer].

+h(3, "annotations-pos-deps") Part-of-speech tags and dependencies
    +tag-model("dependency parse")

include _spacy-101/_pos-deps

+infobox
    |  To learn more about #[strong part-of-speech tagging] and rule-based
    |  morphology, and how to #[strong navigate and use the parse tree]
    |  effectively, see the usage guides on
    |  #[+a("/docs/usage/pos-tagging") part-of-speech tagging] and
    |  #[+a("/docs/usage/dependency-parse") using the dependency parse].

+h(3, "annotations-ner") Named Entities
    +tag-model("named entities")

include _spacy-101/_named-entities

+infobox
    |  To learn more about entity recognition in spaCy, how to
    |  #[strong add your own entities] to a document and how to
    |  #[strong train and update] the entity predictions of a model, see the
    |  usage guides on
    |  #[+a("/docs/usage/entity-recognition") named entity recognition] and
    |  #[+a("/docs/usage/training-ner") training the named entity recognizer].

+h(2, "vectors-similarity") Word vectors and similarity
    +tag-model("vectors")

include _spacy-101/_similarity

include _spacy-101/_word-vectors

+infobox
    |  To learn more about word vectors, how to #[strong customise them] and
    |  how to load #[strong your own vectors] into spaCy, see the usage
    |  guide on
    |  #[+a("/docs/usage/word-vectors-similarities") using word vectors and semantic similarities].

+h(2, "pipelines") Pipelines

include _spacy-101/_pipelines

+infobox
    |  To learn more about #[strong how processing pipelines work] in detail,
    |  how to enable and disable their components, and how to
    |  #[strong create your own], see the usage guide on
    |  #[+a("/docs/usage/language-processing-pipeline") language processing pipelines].

+h(2, "vocab") Vocab, hashes and lexemes

include _spacy-101/_vocab

+h(2, "serialization") Serialization

include _spacy-101/_serialization

+infobox
    |  To learn more about #[strong serialization] and how to
    |  #[strong save and load your own models], see the usage guide on
    |  #[+a("/docs/usage/saving-loading") saving, loading and data serialization].

+h(2, "training") Training

include _spacy-101/_training

+h(2, "architecture") Architecture

+under-construction

+image
    include ../../assets/img/docs/architecture.svg
    .u-text-right
        +button("/assets/img/docs/architecture.svg", false, "secondary").u-text-tag View large graphic

+table(["Name", "Description"])
    +row
        +cell #[+api("language") #[code Language]]
        +cell
            |  A text-processing pipeline. Usually you'll load this once per
            |  process as #[code nlp] and pass the instance around your application.

    +row
        +cell #[+api("doc") #[code Doc]]
        +cell A container for accessing linguistic annotations.

    +row
        +cell #[+api("span") #[code Span]]
        +cell A slice from a #[code Doc] object.

    +row
        +cell #[+api("token") #[code Token]]
        +cell
            |  An individual token — i.e. a word, punctuation symbol, whitespace,
            |  etc.

    +row
        +cell #[+api("lexeme") #[code Lexeme]]
        +cell
            |  An entry in the vocabulary. It's a word type with no context, as
            |  opposed to a word token. It therefore has no part-of-speech tag,
            |  dependency parse etc.

    +row
        +cell #[+api("vocab") #[code Vocab]]
        +cell
            |  A lookup table for the vocabulary that allows you to access
            |  #[code Lexeme] objects.

    +row
        +cell #[code Morphology]
        +cell

    +row
        +cell #[+api("stringstore") #[code StringStore]]
        +cell Map strings to and from integer IDs.

    +row
        +row
        +cell #[+api("tokenizer") #[code Tokenizer]]
        +cell
            |  Segment text, and create #[code Doc] objects with the discovered
            |  segment boundaries.

    +row
        +cell #[+api("matcher") #[code Matcher]]
        +cell
            |  Match sequences of tokens, based on pattern rules, similar to
            |  regular expressions.

+h(3, "architecture-pipeline") Pipeline components

+table(["Name", "Description"])
    +row
        +cell #[+api("tagger") #[code Tagger]]
        +cell Annotate part-of-speech tags on #[code Doc] objects.

    +row
        +cell #[+api("dependencyparser") #[code DependencyParser]]
        +cell Annotate syntactic dependencies on #[code Doc] objects.

    +row
        +cell #[+api("entityrecognizer") #[code EntityRecognizer]]
        +cell
            |  Annotate named entities, e.g. persons or products, on #[code Doc]
            |  objects.

+h(3, "architecture-other") Other classes

+table(["Name", "Description"])
    +row
        +cell #[+api("binder") #[code Binder]]
        +cell

    +row
        +cell #[+api("goldparse") #[code GoldParse]]
        +cell Collection for training annotations.

    +row
        +cell #[+api("goldcorpus") #[code GoldCorpus]]
        +cell
            |  An annotated corpus, using the JSON file format. Manages
            |  annotations for tagging, dependency parsing and NER.