diff --git a/spacy/tests/parser/prag_sbd.py b/spacy/tests/parser/prag_sbd.py deleted file mode 100644 index 3a74bc9da..000000000 --- a/spacy/tests/parser/prag_sbd.py +++ /dev/null @@ -1,247 +0,0 @@ -# encoding: utf-8 -# SBD tests from "Pragmatic Segmenter" -from __future__ import unicode_literals - -from spacy.en import English - -EN = English() - - -def get_sent_strings(text): - tokens = EN(text) - sents = [] - for sent in tokens.sents: - sents.append(''.join(tokens[i].string - for i in range(sent.start, sent.end)).strip()) - return sents - - -def test_gr1(): - sents = get_sent_strings("Hello World. My name is Jonas.") - assert sents == ["Hello World.", "My name is Jonas."] - - -def test_gr2(): - sents = get_sent_strings("What is your name? My name is Jonas.") - assert sents == ["What is your name?", "My name is Jonas."] - - -def test_gr3(): - sents = get_sent_strings("There it is! I found it.") - assert sents == ["There it is!", "I found it."] - - -def test_gr4(): - sents = get_sent_strings("My name is Jonas E. Smith.") - assert sents == ["My name is Jonas E. Smith."] - - -def test_gr5(): - sents = get_sent_strings("Please turn to p. 55.") - assert sents == ["Please turn to p. 55."] - - -def test_gr6(): - sents = get_sent_strings("Were Jane and co. at the party?") - assert sents == ["Were Jane and co. at the party?"] - - -def test_gr7(): - sents = get_sent_strings("They closed the deal with Pitt, Briggs & Co. at noon.") - assert sents == ["They closed the deal with Pitt, Briggs & Co. at noon."] - - -def test_gr8(): - sents = get_sent_strings("Let's ask Jane and co. They should know.") - assert sents == ["Let's ask Jane and co.", "They should know."] - - -def test_gr9(): - sents = get_sent_strings("They closed the deal with Pitt, Briggs & Co. It closed yesterday.") - assert sents == ["They closed the deal with Pitt, Briggs & Co.", "It closed yesterday."] - - -def test_gr10(): - sents = get_sent_strings("I can see Mt. Fuji from here.") - assert sents == ["I can see Mt. Fuji from here."] - - -def test_gr11(): - sents = get_sent_strings("St. Michael's Church is on 5th st. near the light.") - assert sents == ["St. Michael's Church is on 5th st. near the light."] - - -def test_gr12(): - sents = get_sent_strings("That is JFK Jr.'s book.") - assert sents == ["That is JFK Jr.'s book."] - - -def test_gr13(): - sents = get_sent_strings("I visited the U.S.A. last year.") - assert sents == ["I visited the U.S.A. last year."] - - -def test_gr14(): - sents = get_sent_strings("I live in the E.U. How about you?") - assert sents == ["I live in the E.U.", "How about you?"] - - -def test_gr15(): - sents = get_sent_strings("I live in the U.S. How about you?") - assert sents == ["I live in the U.S.", "How about you?"] - - -def test_gr16(): - sents = get_sent_strings("I work for the U.S. Government in Virginia.") - assert sents == ["I work for the U.S. Government in Virginia."] - - -def test_gr17(): - sents = get_sent_strings("I have lived in the U.S. for 20 years.") - assert sents == ["I have lived in the U.S. for 20 years."] - - -def test_gr18(): - sents = get_sent_strings("At 5 a.m. Mr. Smith went to the bank. He left the bank at 6 P.M. Mr. Smith then went to the store.") - assert sents == ["At 5 a.m. Mr. Smith went to the bank.", "He left the bank at 6 P.M.", "Mr. Smith then went to the store."] - - -def test_gr19(): - sents = get_sent_strings("She has $100.00 in her bag.") - assert sents == ["She has $100.00 in her bag."] - - -def test_gr20(): - sents = get_sent_strings("She has $100.00. It is in her bag.") - assert sents == ["She has $100.00.", "It is in her bag."] - - -def test_gr21(): - sents = get_sent_strings("He teaches science (He previously worked for 5 years as an engineer.) at the local University.") - assert sents == ["He teaches science (He previously worked for 5 years as an engineer.) at the local University."] - - -def test_gr22(): - sents = get_sent_strings("Her email is Jane.Doe@example.com. I sent her an email.") - assert sents == ["Her email is Jane.Doe@example.com.", "I sent her an email."] - - -def test_gr23(): - sents = get_sent_strings("The site is: https://www.example.50.com/new-site/awesome_content.html. Please check it out.") - assert sents == ["The site is: https://www.example.50.com/new-site/awesome_content.html.", "Please check it out."] - -def test_gr24(): - sents = get_sent_strings("She turned to him, 'This is great.' she said.") - assert sents == ["She turned to him, 'This is great.' she said."] - -def test_gr25(): - sents = get_sent_strings('She turned to him, "This is great." she said.') - assert sents == ['She turned to him, "This is great." she said.'] - -def test_gr26(): - sents = get_sent_strings('She turned to him, "This is great." She held the book out to show him.') - assert sents == ['She turned to him, "This is great."', "She held the book out to show him."] - -def test_gr27(): - sents = get_sent_strings("Hello!! Long time no see.") - assert sents == ["Hello!!", "Long time no see."] - -def test_gr28(): - sents = get_sent_strings("Hello?? Who is there?") - assert sents == ["Hello??", "Who is there?"] - -def test_gr29(): - sents = get_sent_strings("Hello!? Is that you?") - assert sents == ["Hello!?", "Is that you?"] - -def test_gr30(): - sents = get_sent_strings("Hello?! Is that you?") - assert sents == ["Hello?!", "Is that you?"] - -def test_gr31(): - sents = get_sent_strings("1.) The first item 2.) The second item") - assert sents == ["1.) The first item", "2.) The second item"] - -def test_gr32(): - sents = get_sent_strings("1.) The first item. 2.) The second item.") - assert sents == ["1.) The first item.", "2.) The second item."] - -def test_gr33(): - sents = get_sent_strings("1) The first item 2) The second item") - assert sents == ["1) The first item", "2) The second item"] - -def test_gr34(): - sents = get_sent_strings("1) The first item. 2) The second item.") - assert sents == ["1) The first item.", "2) The second item."] - -def test_gr35(): - sents = get_sent_strings("1. The first item 2. The second item") - assert sents == ["1. The first item", "2. The second item"] - -def test_gr36(): - sents = get_sent_strings("1. The first item. 2. The second item.") - assert sents == ["1. The first item.", "2. The second item."] - -def test_gr37(): - sents = get_sent_strings("• 9. The first item • 10. The second item") - assert sents == ["• 9. The first item", "• 10. The second item"] - -def test_gr38(): - sents = get_sent_strings("⁃9. The first item ⁃10. The second item") - assert sents == ["⁃9. The first item", "⁃10. The second item"] - -def test_gr39(): - sents = get_sent_strings("a. The first item b. The second item c. The third list item") - assert sents == ["a. The first item", "b. The second item", "c. The third list item"] - -def test_gr40(): - sents = get_sent_strings("This is a sentence\ncut off in the middle because pdf.") - assert sents == ["This is a sentence\ncut off in the middle because pdf."] - -def test_gr41(): - sents = get_sent_strings("It was a cold \nnight in the city.") - assert sents == ["It was a cold \nnight in the city."] - -def test_gr42(): - sents = get_sent_strings("features\ncontact manager\nevents, activities\n") - assert sents == ["features", "contact manager", "events, activities"] - -def test_gr43(): - sents = get_sent_strings("You can find it at N°. 1026.253.553. That is where the treasure is.") - assert sents == ["You can find it at N°. 1026.253.553.", "That is where the treasure is."] - -def test_gr44(): - sents = get_sent_strings("She works at Yahoo! in the accounting department.") - assert sents == ["She works at Yahoo! in the accounting department."] - -def test_gr45(): - sents = get_sent_strings("We make a good team, you and I. Did you see Albert I. Jones yesterday?") - assert sents == ["We make a good team, you and I.", "Did you see Albert I. Jones yesterday?"] - -def test_gr46(): - sents = get_sent_strings("Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .”") - assert sents == ["Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .”"] - -def test_gr47(): - sents = get_sent_strings(""""Bohr [...] used the analogy of parallel stairways [...]" (Smith 55).""") - assert sents == ['"Bohr [...] used the analogy of parallel stairways [...]" (Smith 55).'] - -def test_gr48(): - sents = get_sent_strings("If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . . Next sentence.") - assert sents == ["If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . .", "Next sentence."] - -def test_gr49(): - sents = get_sent_strings("I never meant that.... She left the store.") - assert sents == ["I never meant that....", "She left the store."] - -def test_gr50(): - sents = get_sent_strings("I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it.") - assert sents == ["I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it."] - -def test_gr51(): - sents = get_sent_strings("One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . .") - assert sents == ["One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds.", ". . . The practice was not abandoned. . . ."] - -def test_gr52(): - sents = get_sent_strings("Hello world.Today is Tuesday.Mr. Smith went to the store and bought 1,000.That is a lot.",) - assert sents == ["Hello world.", "Today is Tuesday.", "Mr. Smith went to the store and bought 1,000.", "That is a lot."] diff --git a/spacy/tests/parser/test_sbd_prag.py b/spacy/tests/parser/test_sbd_prag.py new file mode 100644 index 000000000..0e537122c --- /dev/null +++ b/spacy/tests/parser/test_sbd_prag.py @@ -0,0 +1,71 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import pytest + + +TEST_CASES = [ + ("Hello World. My name is Jonas.", ["Hello World.", "My name is Jonas."]), + ("What is your name? My name is Jonas.", ["What is your name?", "My name is Jonas."]), + ("There it is! I found it.", ["There it is!", "I found it."]), + ("My name is Jonas E. Smith.", ["My name is Jonas E. Smith."]), + ("Please turn to p. 55.", ["Please turn to p. 55."]), + ("Were Jane and co. at the party?", ["Were Jane and co. at the party?"]), + ("They closed the deal with Pitt, Briggs & Co. at noon.", ["They closed the deal with Pitt, Briggs & Co. at noon."]), + pytest.mark.xfail(("Let's ask Jane and co. They should know.", ["Let's ask Jane and co.", "They should know."])), + ("They closed the deal with Pitt, Briggs & Co. It closed yesterday.", ["They closed the deal with Pitt, Briggs & Co.", "It closed yesterday."]), + ("I can see Mt. Fuji from here.", ["I can see Mt. Fuji from here."]), + pytest.mark.xfail(("St. Michael's Church is on 5th st. near the light.", ["St. Michael's Church is on 5th st. near the light."])), + ("That is JFK Jr.'s book.", ["That is JFK Jr.'s book."]), + ("I visited the U.S.A. last year.", ["I visited the U.S.A. last year."]), + pytest.mark.xfail(("I live in the E.U. How about you?", ["I live in the E.U.", "How about you?"])), + pytest.mark.xfail(("I live in the U.S. How about you?", ["I live in the U.S.", "How about you?"])), + ("I work for the U.S. Government in Virginia.", ["I work for the U.S. Government in Virginia."]), + ("I have lived in the U.S. for 20 years.", ["I have lived in the U.S. for 20 years."]), + pytest.mark.xfail(("At 5 a.m. Mr. Smith went to the bank. He left the bank at 6 P.M. Mr. Smith then went to the store.", ["At 5 a.m. Mr. Smith went to the bank.", "He left the bank at 6 P.M.", "Mr. Smith then went to the store."])), + ("She has $100.00 in her bag.", ["She has $100.00 in her bag."]), + ("She has $100.00. It is in her bag.", ["She has $100.00.", "It is in her bag."]), + ("He teaches science (He previously worked for 5 years as an engineer.) at the local University.", ["He teaches science (He previously worked for 5 years as an engineer.) at the local University."]), + pytest.mark.xfail(("Her email is Jane.Doe@example.com. I sent her an email.", ["Her email is Jane.Doe@example.com.", "I sent her an email."])), + pytest.mark.xfail(("The site is: https://www.example.50.com/new-site/awesome_content.html. Please check it out.", ["The site is: https://www.example.50.com/new-site/awesome_content.html.", "Please check it out."])), + pytest.mark.xfail(("She turned to him, 'This is great.' she said.", ["She turned to him, 'This is great.' she said."])), + ('She turned to him, "This is great." she said.', ['She turned to him, "This is great." she said.']), + ('She turned to him, "This is great." She held the book out to show him.', ['She turned to him, "This is great."', "She held the book out to show him."]), + ("Hello!! Long time no see.", ["Hello!!", "Long time no see."]), + ("Hello?? Who is there?", ["Hello??", "Who is there?"]), + ("Hello!? Is that you?", ["Hello!?", "Is that you?"]), + ("Hello?! Is that you?", ["Hello?!", "Is that you?"]), + pytest.mark.xfail(("1.) The first item 2.) The second item", ["1.) The first item", "2.) The second item"])), + pytest.mark.xfail(("1.) The first item. 2.) The second item.", ["1.) The first item.", "2.) The second item."])), + pytest.mark.xfail(("1) The first item 2) The second item", ["1) The first item", "2) The second item"])), + pytest.mark.xfail(("1) The first item. 2) The second item.", ["1) The first item.", "2) The second item."])), + pytest.mark.xfail(("1. The first item 2. The second item", ["1. The first item", "2. The second item"])), + pytest.mark.xfail(("1. The first item. 2. The second item.", ["1. The first item.", "2. The second item."])), + pytest.mark.xfail(("• 9. The first item • 10. The second item", ["• 9. The first item", "• 10. The second item"])), + pytest.mark.xfail(("⁃9. The first item ⁃10. The second item", ["⁃9. The first item", "⁃10. The second item"])), + pytest.mark.xfail(("a. The first item b. The second item c. The third list item", ["a. The first item", "b. The second item", "c. The third list item"])), + ("This is a sentence\ncut off in the middle because pdf.", ["This is a sentence\ncut off in the middle because pdf."]), + ("It was a cold \nnight in the city.", ["It was a cold \nnight in the city."]), + pytest.mark.xfail(("features\ncontact manager\nevents, activities\n", ["features", "contact manager", "events, activities"])), + ("You can find it at N°. 1026.253.553. That is where the treasure is.", ["You can find it at N°. 1026.253.553.", "That is where the treasure is."]), + ("She works at Yahoo! in the accounting department.", ["She works at Yahoo! in the accounting department."]), + pytest.mark.xfail(("We make a good team, you and I. Did you see Albert I. Jones yesterday?", ["We make a good team, you and I.", "Did you see Albert I. Jones yesterday?"])), + pytest.mark.xfail(("Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .”", ["Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .”"])), + (""""Bohr [...] used the analogy of parallel stairways [...]" (Smith 55).""", ['"Bohr [...] used the analogy of parallel stairways [...]" (Smith 55).']), + pytest.mark.xfail(("If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . . Next sentence.", ["If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . .", "Next sentence."])), + ("I never meant that.... She left the store.", ["I never meant that....", "She left the store."]), + pytest.mark.xfail(("I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it.", ["I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it."])), + pytest.mark.xfail(("One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . .", ["One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds.", ". . . The practice was not abandoned. . . ."])), + pytest.mark.xfail(("Hello world.Today is Tuesday.Mr. Smith went to the store and bought 1,000.That is a lot.", ["Hello world.", "Today is Tuesday.", "Mr. Smith went to the store and bought 1,000.", "That is a lot."])) +] + +@pytest.mark.slow +@pytest.mark.models +@pytest.mark.parametrize('text,expected_sents', TEST_CASES) +def test_parser_sbd_prag(EN, text, expected_sents): + """SBD tests from Pragmatic Segmenter""" + doc = EN(text) + sents = [] + for sent in doc.sents: + sents.append(''.join(doc[i].string for i in range(sent.start, sent.end)).strip()) + assert sents == expected_sents