mirror of https://github.com/explosion/spaCy.git
Reformat and rename Pragmatic Segmenter tests and mark xfails
This commit is contained in:
Ines Montani
parent
affcf1b19d
commit
094ff7396a
|
|
@ -1,247 +0,0 @@
|
||||||
# encoding: utf-8
|
|
||||||
# SBD tests from "Pragmatic Segmenter"
|
|
||||||
from __future__ import unicode_literals
|
|
||||||
|
|
||||||
from spacy.en import English
|
|
||||||
|
|
||||||
EN = English()
|
|
||||||
|
|
||||||
|
|
||||||
def get_sent_strings(text):
|
|
||||||
tokens = EN(text)
|
|
||||||
sents = []
|
|
||||||
for sent in tokens.sents:
|
|
||||||
sents.append(''.join(tokens[i].string
|
|
||||||
for i in range(sent.start, sent.end)).strip())
|
|
||||||
return sents
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr1():
|
|
||||||
sents = get_sent_strings("Hello World. My name is Jonas.")
|
|
||||||
assert sents == ["Hello World.", "My name is Jonas."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr2():
|
|
||||||
sents = get_sent_strings("What is your name? My name is Jonas.")
|
|
||||||
assert sents == ["What is your name?", "My name is Jonas."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr3():
|
|
||||||
sents = get_sent_strings("There it is! I found it.")
|
|
||||||
assert sents == ["There it is!", "I found it."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr4():
|
|
||||||
sents = get_sent_strings("My name is Jonas E. Smith.")
|
|
||||||
assert sents == ["My name is Jonas E. Smith."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr5():
|
|
||||||
sents = get_sent_strings("Please turn to p. 55.")
|
|
||||||
assert sents == ["Please turn to p. 55."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr6():
|
|
||||||
sents = get_sent_strings("Were Jane and co. at the party?")
|
|
||||||
assert sents == ["Were Jane and co. at the party?"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr7():
|
|
||||||
sents = get_sent_strings("They closed the deal with Pitt, Briggs & Co. at noon.")
|
|
||||||
assert sents == ["They closed the deal with Pitt, Briggs & Co. at noon."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr8():
|
|
||||||
sents = get_sent_strings("Let's ask Jane and co. They should know.")
|
|
||||||
assert sents == ["Let's ask Jane and co.", "They should know."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr9():
|
|
||||||
sents = get_sent_strings("They closed the deal with Pitt, Briggs & Co. It closed yesterday.")
|
|
||||||
assert sents == ["They closed the deal with Pitt, Briggs & Co.", "It closed yesterday."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr10():
|
|
||||||
sents = get_sent_strings("I can see Mt. Fuji from here.")
|
|
||||||
assert sents == ["I can see Mt. Fuji from here."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr11():
|
|
||||||
sents = get_sent_strings("St. Michael's Church is on 5th st. near the light.")
|
|
||||||
assert sents == ["St. Michael's Church is on 5th st. near the light."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr12():
|
|
||||||
sents = get_sent_strings("That is JFK Jr.'s book.")
|
|
||||||
assert sents == ["That is JFK Jr.'s book."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr13():
|
|
||||||
sents = get_sent_strings("I visited the U.S.A. last year.")
|
|
||||||
assert sents == ["I visited the U.S.A. last year."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr14():
|
|
||||||
sents = get_sent_strings("I live in the E.U. How about you?")
|
|
||||||
assert sents == ["I live in the E.U.", "How about you?"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr15():
|
|
||||||
sents = get_sent_strings("I live in the U.S. How about you?")
|
|
||||||
assert sents == ["I live in the U.S.", "How about you?"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr16():
|
|
||||||
sents = get_sent_strings("I work for the U.S. Government in Virginia.")
|
|
||||||
assert sents == ["I work for the U.S. Government in Virginia."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr17():
|
|
||||||
sents = get_sent_strings("I have lived in the U.S. for 20 years.")
|
|
||||||
assert sents == ["I have lived in the U.S. for 20 years."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr18():
|
|
||||||
sents = get_sent_strings("At 5 a.m. Mr. Smith went to the bank. He left the bank at 6 P.M. Mr. Smith then went to the store.")
|
|
||||||
assert sents == ["At 5 a.m. Mr. Smith went to the bank.", "He left the bank at 6 P.M.", "Mr. Smith then went to the store."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr19():
|
|
||||||
sents = get_sent_strings("She has $100.00 in her bag.")
|
|
||||||
assert sents == ["She has $100.00 in her bag."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr20():
|
|
||||||
sents = get_sent_strings("She has $100.00. It is in her bag.")
|
|
||||||
assert sents == ["She has $100.00.", "It is in her bag."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr21():
|
|
||||||
sents = get_sent_strings("He teaches science (He previously worked for 5 years as an engineer.) at the local University.")
|
|
||||||
assert sents == ["He teaches science (He previously worked for 5 years as an engineer.) at the local University."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr22():
|
|
||||||
sents = get_sent_strings("Her email is Jane.Doe@example.com. I sent her an email.")
|
|
||||||
assert sents == ["Her email is Jane.Doe@example.com.", "I sent her an email."]
|
|
||||||
|
|
||||||
|
|
||||||
def test_gr23():
|
|
||||||
sents = get_sent_strings("The site is: https://www.example.50.com/new-site/awesome_content.html. Please check it out.")
|
|
||||||
assert sents == ["The site is: https://www.example.50.com/new-site/awesome_content.html.", "Please check it out."]
|
|
||||||
|
|
||||||
def test_gr24():
|
|
||||||
sents = get_sent_strings("She turned to him, 'This is great.' she said.")
|
|
||||||
assert sents == ["She turned to him, 'This is great.' she said."]
|
|
||||||
|
|
||||||
def test_gr25():
|
|
||||||
sents = get_sent_strings('She turned to him, "This is great." she said.')
|
|
||||||
assert sents == ['She turned to him, "This is great." she said.']
|
|
||||||
|
|
||||||
def test_gr26():
|
|
||||||
sents = get_sent_strings('She turned to him, "This is great." She held the book out to show him.')
|
|
||||||
assert sents == ['She turned to him, "This is great."', "She held the book out to show him."]
|
|
||||||
|
|
||||||
def test_gr27():
|
|
||||||
sents = get_sent_strings("Hello!! Long time no see.")
|
|
||||||
assert sents == ["Hello!!", "Long time no see."]
|
|
||||||
|
|
||||||
def test_gr28():
|
|
||||||
sents = get_sent_strings("Hello?? Who is there?")
|
|
||||||
assert sents == ["Hello??", "Who is there?"]
|
|
||||||
|
|
||||||
def test_gr29():
|
|
||||||
sents = get_sent_strings("Hello!? Is that you?")
|
|
||||||
assert sents == ["Hello!?", "Is that you?"]
|
|
||||||
|
|
||||||
def test_gr30():
|
|
||||||
sents = get_sent_strings("Hello?! Is that you?")
|
|
||||||
assert sents == ["Hello?!", "Is that you?"]
|
|
||||||
|
|
||||||
def test_gr31():
|
|
||||||
sents = get_sent_strings("1.) The first item 2.) The second item")
|
|
||||||
assert sents == ["1.) The first item", "2.) The second item"]
|
|
||||||
|
|
||||||
def test_gr32():
|
|
||||||
sents = get_sent_strings("1.) The first item. 2.) The second item.")
|
|
||||||
assert sents == ["1.) The first item.", "2.) The second item."]
|
|
||||||
|
|
||||||
def test_gr33():
|
|
||||||
sents = get_sent_strings("1) The first item 2) The second item")
|
|
||||||
assert sents == ["1) The first item", "2) The second item"]
|
|
||||||
|
|
||||||
def test_gr34():
|
|
||||||
sents = get_sent_strings("1) The first item. 2) The second item.")
|
|
||||||
assert sents == ["1) The first item.", "2) The second item."]
|
|
||||||
|
|
||||||
def test_gr35():
|
|
||||||
sents = get_sent_strings("1. The first item 2. The second item")
|
|
||||||
assert sents == ["1. The first item", "2. The second item"]
|
|
||||||
|
|
||||||
def test_gr36():
|
|
||||||
sents = get_sent_strings("1. The first item. 2. The second item.")
|
|
||||||
assert sents == ["1. The first item.", "2. The second item."]
|
|
||||||
|
|
||||||
def test_gr37():
|
|
||||||
sents = get_sent_strings("• 9. The first item • 10. The second item")
|
|
||||||
assert sents == ["• 9. The first item", "• 10. The second item"]
|
|
||||||
|
|
||||||
def test_gr38():
|
|
||||||
sents = get_sent_strings("⁃9. The first item ⁃10. The second item")
|
|
||||||
assert sents == ["⁃9. The first item", "⁃10. The second item"]
|
|
||||||
|
|
||||||
def test_gr39():
|
|
||||||
sents = get_sent_strings("a. The first item b. The second item c. The third list item")
|
|
||||||
assert sents == ["a. The first item", "b. The second item", "c. The third list item"]
|
|
||||||
|
|
||||||
def test_gr40():
|
|
||||||
sents = get_sent_strings("This is a sentence\ncut off in the middle because pdf.")
|
|
||||||
assert sents == ["This is a sentence\ncut off in the middle because pdf."]
|
|
||||||
|
|
||||||
def test_gr41():
|
|
||||||
sents = get_sent_strings("It was a cold \nnight in the city.")
|
|
||||||
assert sents == ["It was a cold \nnight in the city."]
|
|
||||||
|
|
||||||
def test_gr42():
|
|
||||||
sents = get_sent_strings("features\ncontact manager\nevents, activities\n")
|
|
||||||
assert sents == ["features", "contact manager", "events, activities"]
|
|
||||||
|
|
||||||
def test_gr43():
|
|
||||||
sents = get_sent_strings("You can find it at N°. 1026.253.553. That is where the treasure is.")
|
|
||||||
assert sents == ["You can find it at N°. 1026.253.553.", "That is where the treasure is."]
|
|
||||||
|
|
||||||
def test_gr44():
|
|
||||||
sents = get_sent_strings("She works at Yahoo! in the accounting department.")
|
|
||||||
assert sents == ["She works at Yahoo! in the accounting department."]
|
|
||||||
|
|
||||||
def test_gr45():
|
|
||||||
sents = get_sent_strings("We make a good team, you and I. Did you see Albert I. Jones yesterday?")
|
|
||||||
assert sents == ["We make a good team, you and I.", "Did you see Albert I. Jones yesterday?"]
|
|
||||||
|
|
||||||
def test_gr46():
|
|
||||||
sents = get_sent_strings("Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .”")
|
|
||||||
assert sents == ["Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .”"]
|
|
||||||
|
|
||||||
def test_gr47():
|
|
||||||
sents = get_sent_strings(""""Bohr [...] used the analogy of parallel stairways [...]" (Smith 55).""")
|
|
||||||
assert sents == ['"Bohr [...] used the analogy of parallel stairways [...]" (Smith 55).']
|
|
||||||
|
|
||||||
def test_gr48():
|
|
||||||
sents = get_sent_strings("If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . . Next sentence.")
|
|
||||||
assert sents == ["If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . .", "Next sentence."]
|
|
||||||
|
|
||||||
def test_gr49():
|
|
||||||
sents = get_sent_strings("I never meant that.... She left the store.")
|
|
||||||
assert sents == ["I never meant that....", "She left the store."]
|
|
||||||
|
|
||||||
def test_gr50():
|
|
||||||
sents = get_sent_strings("I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it.")
|
|
||||||
assert sents == ["I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it."]
|
|
||||||
|
|
||||||
def test_gr51():
|
|
||||||
sents = get_sent_strings("One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . .")
|
|
||||||
assert sents == ["One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds.", ". . . The practice was not abandoned. . . ."]
|
|
||||||
|
|
||||||
def test_gr52():
|
|
||||||
sents = get_sent_strings("Hello world.Today is Tuesday.Mr. Smith went to the store and bought 1,000.That is a lot.",)
|
|
||||||
assert sents == ["Hello world.", "Today is Tuesday.", "Mr. Smith went to the store and bought 1,000.", "That is a lot."]
|
|
||||||
|
|
@ -0,0 +1,71 @@
|
||||||
|
# encoding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
TEST_CASES = [
|
||||||
|
("Hello World. My name is Jonas.", ["Hello World.", "My name is Jonas."]),
|
||||||
|
("What is your name? My name is Jonas.", ["What is your name?", "My name is Jonas."]),
|
||||||
|
("There it is! I found it.", ["There it is!", "I found it."]),
|
||||||
|
("My name is Jonas E. Smith.", ["My name is Jonas E. Smith."]),
|
||||||
|
("Please turn to p. 55.", ["Please turn to p. 55."]),
|
||||||
|
("Were Jane and co. at the party?", ["Were Jane and co. at the party?"]),
|
||||||
|
("They closed the deal with Pitt, Briggs & Co. at noon.", ["They closed the deal with Pitt, Briggs & Co. at noon."]),
|
||||||
|
pytest.mark.xfail(("Let's ask Jane and co. They should know.", ["Let's ask Jane and co.", "They should know."])),
|
||||||
|
("They closed the deal with Pitt, Briggs & Co. It closed yesterday.", ["They closed the deal with Pitt, Briggs & Co.", "It closed yesterday."]),
|
||||||
|
("I can see Mt. Fuji from here.", ["I can see Mt. Fuji from here."]),
|
||||||
|
pytest.mark.xfail(("St. Michael's Church is on 5th st. near the light.", ["St. Michael's Church is on 5th st. near the light."])),
|
||||||
|
("That is JFK Jr.'s book.", ["That is JFK Jr.'s book."]),
|
||||||
|
("I visited the U.S.A. last year.", ["I visited the U.S.A. last year."]),
|
||||||
|
pytest.mark.xfail(("I live in the E.U. How about you?", ["I live in the E.U.", "How about you?"])),
|
||||||
|
pytest.mark.xfail(("I live in the U.S. How about you?", ["I live in the U.S.", "How about you?"])),
|
||||||
|
("I work for the U.S. Government in Virginia.", ["I work for the U.S. Government in Virginia."]),
|
||||||
|
("I have lived in the U.S. for 20 years.", ["I have lived in the U.S. for 20 years."]),
|
||||||
|
pytest.mark.xfail(("At 5 a.m. Mr. Smith went to the bank. He left the bank at 6 P.M. Mr. Smith then went to the store.", ["At 5 a.m. Mr. Smith went to the bank.", "He left the bank at 6 P.M.", "Mr. Smith then went to the store."])),
|
||||||
|
("She has $100.00 in her bag.", ["She has $100.00 in her bag."]),
|
||||||
|
("She has $100.00. It is in her bag.", ["She has $100.00.", "It is in her bag."]),
|
||||||
|
("He teaches science (He previously worked for 5 years as an engineer.) at the local University.", ["He teaches science (He previously worked for 5 years as an engineer.) at the local University."]),
|
||||||
|
pytest.mark.xfail(("Her email is Jane.Doe@example.com. I sent her an email.", ["Her email is Jane.Doe@example.com.", "I sent her an email."])),
|
||||||
|
pytest.mark.xfail(("The site is: https://www.example.50.com/new-site/awesome_content.html. Please check it out.", ["The site is: https://www.example.50.com/new-site/awesome_content.html.", "Please check it out."])),
|
||||||
|
pytest.mark.xfail(("She turned to him, 'This is great.' she said.", ["She turned to him, 'This is great.' she said."])),
|
||||||
|
('She turned to him, "This is great." she said.', ['She turned to him, "This is great." she said.']),
|
||||||
|
('She turned to him, "This is great." She held the book out to show him.', ['She turned to him, "This is great."', "She held the book out to show him."]),
|
||||||
|
("Hello!! Long time no see.", ["Hello!!", "Long time no see."]),
|
||||||
|
("Hello?? Who is there?", ["Hello??", "Who is there?"]),
|
||||||
|
("Hello!? Is that you?", ["Hello!?", "Is that you?"]),
|
||||||
|
("Hello?! Is that you?", ["Hello?!", "Is that you?"]),
|
||||||
|
pytest.mark.xfail(("1.) The first item 2.) The second item", ["1.) The first item", "2.) The second item"])),
|
||||||
|
pytest.mark.xfail(("1.) The first item. 2.) The second item.", ["1.) The first item.", "2.) The second item."])),
|
||||||
|
pytest.mark.xfail(("1) The first item 2) The second item", ["1) The first item", "2) The second item"])),
|
||||||
|
pytest.mark.xfail(("1) The first item. 2) The second item.", ["1) The first item.", "2) The second item."])),
|
||||||
|
pytest.mark.xfail(("1. The first item 2. The second item", ["1. The first item", "2. The second item"])),
|
||||||
|
pytest.mark.xfail(("1. The first item. 2. The second item.", ["1. The first item.", "2. The second item."])),
|
||||||
|
pytest.mark.xfail(("• 9. The first item • 10. The second item", ["• 9. The first item", "• 10. The second item"])),
|
||||||
|
pytest.mark.xfail(("⁃9. The first item ⁃10. The second item", ["⁃9. The first item", "⁃10. The second item"])),
|
||||||
|
pytest.mark.xfail(("a. The first item b. The second item c. The third list item", ["a. The first item", "b. The second item", "c. The third list item"])),
|
||||||
|
("This is a sentence\ncut off in the middle because pdf.", ["This is a sentence\ncut off in the middle because pdf."]),
|
||||||
|
("It was a cold \nnight in the city.", ["It was a cold \nnight in the city."]),
|
||||||
|
pytest.mark.xfail(("features\ncontact manager\nevents, activities\n", ["features", "contact manager", "events, activities"])),
|
||||||
|
("You can find it at N°. 1026.253.553. That is where the treasure is.", ["You can find it at N°. 1026.253.553.", "That is where the treasure is."]),
|
||||||
|
("She works at Yahoo! in the accounting department.", ["She works at Yahoo! in the accounting department."]),
|
||||||
|
pytest.mark.xfail(("We make a good team, you and I. Did you see Albert I. Jones yesterday?", ["We make a good team, you and I.", "Did you see Albert I. Jones yesterday?"])),
|
||||||
|
pytest.mark.xfail(("Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .”", ["Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .”"])),
|
||||||
|
(""""Bohr [...] used the analogy of parallel stairways [...]" (Smith 55).""", ['"Bohr [...] used the analogy of parallel stairways [...]" (Smith 55).']),
|
||||||
|
pytest.mark.xfail(("If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . . Next sentence.", ["If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . .", "Next sentence."])),
|
||||||
|
("I never meant that.... She left the store.", ["I never meant that....", "She left the store."]),
|
||||||
|
pytest.mark.xfail(("I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it.", ["I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it."])),
|
||||||
|
pytest.mark.xfail(("One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . .", ["One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds.", ". . . The practice was not abandoned. . . ."])),
|
||||||
|
pytest.mark.xfail(("Hello world.Today is Tuesday.Mr. Smith went to the store and bought 1,000.That is a lot.", ["Hello world.", "Today is Tuesday.", "Mr. Smith went to the store and bought 1,000.", "That is a lot."]))
|
||||||
|
]
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
|
@pytest.mark.models
|
||||||
|
@pytest.mark.parametrize('text,expected_sents', TEST_CASES)
|
||||||
|
def test_parser_sbd_prag(EN, text, expected_sents):
|
||||||
|
"""SBD tests from Pragmatic Segmenter"""
|
||||||
|
doc = EN(text)
|
||||||
|
sents = []
|
||||||
|
for sent in doc.sents:
|
||||||
|
sents.append(''.join(doc[i].string for i in range(sent.start, sent.end)).strip())
|
||||||
|
assert sents == expected_sents
|
||||||
Loading…
Reference in New Issue