From c467cf0274b3e2135393af4fc88683de9d930018 Mon Sep 17 00:00:00 2001 From: Paul Buchheit Date: Wed, 3 Nov 2010 17:12:31 -0700 Subject: [PATCH] fix linkify regex plus add unit test --- tornado/escape.py | 9 ++- tornado/test/escape_test.py | 127 ++++++++++++++++++++++++++++++++++++ tornado/test/runtests.py | 1 + 3 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 tornado/test/escape_test.py diff --git a/tornado/escape.py b/tornado/escape.py index 174c71ce..306a13df 100644 --- a/tornado/escape.py +++ b/tornado/escape.py @@ -95,9 +95,12 @@ def utf8(value): return value -# Regex from http://daringfireball.net/2010/07/improved_regex_for_matching_urls -# Modified to capture protocol and to avoid HTML character entities other than & -_URL_RE = re.compile(ur"""(?i)\b((?:([a-z][\w-]+):(?:(/{1,3})|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>&]+|&|\(([^\s()<>&]+|(\([^\s()<>&]+\)))*\))+(?:\(([^\s()<>&]+|(\([^\s()<>&]+\)))*\)|[^\s`!()\[\]{};:'".,<>?\xab\xbb\u201c\u201d\u2018\u2019&]))""") +# I originally used the regex from +# http://daringfireball.net/2010/07/improved_regex_for_matching_urls +# but it gets all exponential on certain patterns (such as too many trailing +# dots), causing the regex matcher to never return. +# This regex should avoid those problems. +_URL_RE = re.compile(ur"""\b((?:([\w-]+):(/{1,3})|www[.])(?:(?:(?:[^\s&()]|&|")*(?:[^!"#$%&'()*+,.:;<=>?@\[\]^`{|}~\s]))|(?:\((?:[^\s&()]|&|")*\)))+)""") def linkify(text, shorten=False, extra_params="", diff --git a/tornado/test/escape_test.py b/tornado/test/escape_test.py new file mode 100644 index 00000000..39f28e96 --- /dev/null +++ b/tornado/test/escape_test.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python + +import tornado.escape +import unittest + +linkify_tests = [ + # (input, linkify_kwargs, expected_output) + + ("hello http://world.com/!", {}, + u'hello http://world.com/!'), + + ("hello http://world.com/with?param=true&stuff=yes", {}, + u'hello http://world.com/with?param=true&stuff=yes'), + + # an opened paren followed by many chars killed Gruber's regex + ("http://url.com/w(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {}, + u'http://url.com/w(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'), + + # as did too many dots at the end + ("http://url.com/withmany.......................................", {}, + u'http://url.com/withmany.......................................'), + + ("http://url.com/withmany((((((((((((((((((((((((((((((((((a)", {}, + u'http://url.com/withmany((((((((((((((((((((((((((((((((((a)'), + + # some examples from http://daringfireball.net/2009/11/liberal_regex_for_matching_urls + # plus a fex extras (such as multiple parentheses). + ("http://foo.com/blah_blah", {}, + u'http://foo.com/blah_blah'), + + ("http://foo.com/blah_blah/", {}, + u'http://foo.com/blah_blah/'), + + ("(Something like http://foo.com/blah_blah)", {}, + u'(Something like http://foo.com/blah_blah)'), + + ("http://foo.com/blah_blah_(wikipedia)", {}, + u'http://foo.com/blah_blah_(wikipedia)'), + + ("http://foo.com/blah_(blah)_(wikipedia)_blah", {}, + u'http://foo.com/blah_(blah)_(wikipedia)_blah'), + + ("(Something like http://foo.com/blah_blah_(wikipedia))", {}, + u'(Something like http://foo.com/blah_blah_(wikipedia))'), + + ("http://foo.com/blah_blah.", {}, + u'http://foo.com/blah_blah.'), + + ("http://foo.com/blah_blah/.", {}, + u'http://foo.com/blah_blah/.'), + + ("", {}, + u'<http://foo.com/blah_blah>'), + + ("", {}, + u'<http://foo.com/blah_blah/>'), + + ("http://foo.com/blah_blah,", {}, + u'http://foo.com/blah_blah,'), + + ("http://www.example.com/wpstyle/?p=364.", {}, + u'http://www.example.com/wpstyle/?p=364.'), + + ("rdar://1234", + {"permitted_protocols": ["http", "rdar"]}, + u'rdar://1234'), + + ("rdar:/1234", + {"permitted_protocols": ["rdar"]}, + u'rdar:/1234'), + + ("http://userid:password@example.com:8080", {}, + u'http://userid:password@example.com:8080'), + + ("http://userid@example.com", {}, + u'http://userid@example.com'), + + ("http://userid@example.com:8080", {}, + u'http://userid@example.com:8080'), + + ("http://userid:password@example.com", {}, + u'http://userid:password@example.com'), + + ("message://%3c330e7f8409726r6a4ba78dkf1fd71420c1bf6ff@mail.gmail.com%3e", + {"permitted_protocols": ["http", "message"]}, + u'message://%3c330e7f8409726r6a4ba78dkf1fd71420c1bf6ff@mail.gmail.com%3e'), + + (u"http://\u27a1.ws/\u4a39", {}, + u'http://\u27a1.ws/\u4a39'), + + ("http://example.com", {}, + u'<tag>http://example.com</tag>'), + + ("Just a www.example.com link.", {}, + u'Just a www.example.com link.'), + + ("Just a www.example.com link.", + {"require_protocol": True}, + u'Just a www.example.com link.'), + + ("A http://reallylong.com/link/that/exceedsthelenglimit.html", + {"require_protocol": True, "shorten": True}, + u'A http://reallylong.com/link...'), + + ("A http://reallylongdomainnamethatwillbetoolong.com/hi!", + {"shorten": True}, + u'A http://reallylongdomainnametha...!'), + + ("A file:///passwords.txt and http://web.com link", {}, + u'A file:///passwords.txt and http://web.com link'), + + ("A file:///passwords.txt and http://web.com link", + {"permitted_protocols": ["file"]}, + u'A file:///passwords.txt and http://web.com link'), + + ("www.external-link.com", + {"extra_params": 'rel="nofollow" class="external"'}, + u'www.external-link.com'), +] + + +class EscapeTestCase(unittest.TestCase): + def test_linkify(self): + for text, kwargs, html in linkify_tests: + linked = tornado.escape.linkify(text, **kwargs) + self.assertEqual(linked, html) + diff --git a/tornado/test/runtests.py b/tornado/test/runtests.py index 1dedb6d1..b19152ba 100755 --- a/tornado/test/runtests.py +++ b/tornado/test/runtests.py @@ -3,6 +3,7 @@ import unittest TEST_MODULES = [ 'tornado.httputil.doctests', + 'tornado.test.escape_test', 'tornado.test.httpserver_test', 'tornado.test.ioloop_test', 'tornado.test.iostream_test',