Add support for 2-token state option

This commit is contained in:
Matthew Honnibal 2017-10-05 21:54:12 -05:00
parent 363aa47b40
commit b0618def8d
1 changed files with 3 additions and 0 deletions

View File

@ -93,6 +93,9 @@ cdef cppclass StateC:
free(this.shifted - PADDING) free(this.shifted - PADDING)
void set_context_tokens(int* ids, int n) nogil: void set_context_tokens(int* ids, int n) nogil:
if n == 2:
ids[0] = this.B(0)
ids[1] = this.S(0)
if n == 8: if n == 8:
ids[0] = this.B(0) ids[0] = this.B(0)
ids[1] = this.B(1) ids[1] = this.B(1)