Add a test that runs TwistedIOLoop on top of TornadoReactor.

This exposed a few more small bugs.

tornado/platform/twisted.py

@@ -186,8 +186,12 @@ class TornadoReactor(PosixReactorBase):
     def callFromThread(self, f, *args, **kw):
         """See `twisted.internet.interfaces.IReactorThreads.callFromThread`"""
         assert callable(f), "%s is not callable" % f
-        p = functools.partial(f, *args, **kw)
-        self._io_loop.add_callback(p)
+        with NullContext():
+            # This NullContext is mainly for an edge case when running
+            # TwistedIOLoop on top of a TornadoReactor.
+            # TwistedIOLoop.add_callback uses reactor.callFromThread and
+            # should not pick up additional StackContexts along the way.
+            self._io_loop.add_callback(f, *args, **kw)
 
     # We don't need the waker code from the super class, Tornado uses
     # its own waker.
@@ -392,16 +396,18 @@ class _FD(object):
 class TwistedIOLoop(tornado.ioloop.IOLoop):
     """IOLoop implementation that runs on Twisted.
 
-    Uses the global Twisted reactor.  It is possible to create multiple
-    TwistedIOLoops in the same process, but it doesn't really make sense
-    because they will all run in the same thread.
+    Uses the global Twisted reactor by default.  To create multiple
+    `TwistedIOLoops` in the same process, you must pass a unique reactor
+    when constructing each one.
 
     Not compatible with `tornado.process.Subprocess.set_exit_callback`
     because the ``SIGCHLD`` handlers used by Tornado and Twisted conflict
     with each other.
     """
-    def initialize(self):
-        from twisted.internet import reactor
+    def initialize(self, reactor=None):
+        if reactor is None:
+            import twisted.internet.reactor
+            reactor = twisted.internet.reactor
         self.reactor = reactor
         self.fds = {}
 
@@ -471,7 +477,8 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
         return self.reactor.callLater(delay, self._run_callback, wrap(callback))
 
     def remove_timeout(self, timeout):
-        timeout.cancel()
+        if timeout.active():
+            timeout.cancel()
 
     def add_callback(self, callback, *args, **kwargs):
         self.reactor.callFromThread(self._run_callback,

tornado/test/process_test.py

@@ -19,7 +19,7 @@ from tornado.web import RequestHandler, Application
 
 
 def skip_if_twisted():
-    if IOLoop.configured_class().__name__ == 'TwistedIOLoop':
+    if IOLoop.configured_class().__name__.endswith('TwistedIOLoop'):
         raise unittest.SkipTest("Process tests not compatible with TwistedIOLoop")
 
 # Not using AsyncHTTPTestCase because we need control over the IOLoop.

tornado/test/twisted_test.py

@@ -56,6 +56,7 @@ from tornado.httpclient import AsyncHTTPClient
 from tornado.httpserver import HTTPServer
 from tornado.ioloop import IOLoop
 from tornado.platform.auto import set_close_exec
+from tornado.platform.select import SelectIOLoop
 from tornado.testing import bind_unused_port
 from tornado.test.util import unittest
 from tornado.util import import_object
@@ -571,6 +572,41 @@ if have_twisted:
     log.defaultObserver.stop()
     # import sys; log.startLogging(sys.stderr, setStdout=0)
     # log.startLoggingWithObserver(log.PythonLoggingObserver().emit, setStdout=0)
+    # import logging; logging.getLogger('twisted').setLevel(logging.WARNING)
+
+if have_twisted:
+    class LayeredTwistedIOLoop(TwistedIOLoop):
+        """Layers a TwistedIOLoop on top of a TornadoReactor on a SelectIOLoop.
+
+        This is of course silly, but is useful for testing purposes to make
+        sure we're implementing both sides of the various interfaces
+        correctly.  In some tests another TornadoReactor is layered on top
+        of the whole stack.
+        """
+        def initialize(self):
+            # When configured to use LayeredTwistedIOLoop we can't easily
+            # get the next-best IOLoop implementation, so use the lowest common
+            # denominator.
+            self.real_io_loop = SelectIOLoop()
+            reactor = TornadoReactor(io_loop=self.real_io_loop)
+            super(LayeredTwistedIOLoop, self).initialize(reactor=reactor)
+
+        def close(self, all_fds=False):
+            super(LayeredTwistedIOLoop, self).close(all_fds=all_fds)
+            # HACK: This is the same thing that test_class.unbuildReactor does.
+            for reader in self.reactor._internalReaders:
+                self.reactor.removeReader(reader)
+                reader.connectionLost(None)
+            self.real_io_loop.close(all_fds=all_fds)
+
+        def stop(self):
+            # One of twisted's tests fails if I don't delay crash()
+            # until the reactor has started, but if I move this to
+            # TwistedIOLoop then the tests fail when I'm *not* running
+            # tornado-on-twisted-on-tornado.  I'm clearly missing something
+            # about the startup/crash semantics, but since stop and crash
+            # are really only used in tests it doesn't really matter.
+            self.reactor.callWhenRunning(self.reactor.crash)
 
 if __name__ == "__main__":
     unittest.main()

tox.ini

@@ -11,7 +11,7 @@
 [tox]
 # "-full" variants include optional dependencies, to ensure
 # that things work both in a bare install and with all the extras.
-envlist = py27-full, py27-curl, py32-full, pypy, py26, py26-full, py27, py32, py32-utf8, py33, py27-opt, py32-opt, pypy-full, py27-select, py27-monotonic, py33-monotonic, py27-twisted, py27-threadedresolver, py27-twistedresolver
+envlist = py27-full, py27-curl, py32-full, pypy, py26, py26-full, py27, py32, py32-utf8, py33, py27-opt, py32-opt, pypy-full, py27-select, py27-monotonic, py33-monotonic, py27-twisted, py27-threadedresolver, py27-twistedresolver, py27-twistedlayered
 [testenv]
 commands = python -m tornado.test.runtests {posargs:}
 
@@ -98,7 +98,7 @@ deps =
      mock
      pycurl
      twisted
-commands = python -m tornado.test.runtests --resolver=tornado.netutil.ThreadedResolver
+commands = python -m tornado.test.runtests --resolver=tornado.netutil.ThreadedResolver {posargs:}
 
 [testenv:py27-twistedresolver]
 basepython = python2.7
@@ -107,7 +107,16 @@ deps =
      mock
      pycurl
      twisted
-commands = python -m tornado.test.runtests --resolver=tornado.platform.twisted.TwistedResolver
+commands = python -m tornado.test.runtests --resolver=tornado.platform.twisted.TwistedResolver {posargs:}
+
+[testenv:py27-twistedlayered]
+basepython = python2.7
+deps =
+     futures
+     mock
+     pycurl
+     twisted
+commands = python -m tornado.test.runtests --ioloop=tornado.test.twisted_test.LayeredTwistedIOLoop --resolver=tornado.platform.twisted.TwistedResolver {posargs:}
 
 [testenv:pypy-full]
 # This configuration works with pypy 1.9.  pycurl installs ok but