import asyncio import contextvars import decimal import itertools import random import socket import ssl import sys import tempfile import unittest import weakref from uvloop import _testbase as tb class _BaseProtocol(asyncio.BaseProtocol): def __init__(self, cvar, *, loop=None): self.cvar = cvar self.transport = None self.connection_made_fut = asyncio.Future(loop=loop) self.buffered_ctx = None self.data_received_fut = asyncio.Future(loop=loop) self.eof_received_fut = asyncio.Future(loop=loop) self.pause_writing_fut = asyncio.Future(loop=loop) self.resume_writing_fut = asyncio.Future(loop=loop) self.pipe_ctx = {0, 1, 2} self.pipe_connection_lost_fut = asyncio.Future(loop=loop) self.process_exited_fut = asyncio.Future(loop=loop) self.error_received_fut = asyncio.Future(loop=loop) self.connection_lost_ctx = None self.done = asyncio.Future(loop=loop) def connection_made(self, transport): self.transport = transport self.connection_made_fut.set_result(self.cvar.get()) def connection_lost(self, exc): self.connection_lost_ctx = self.cvar.get() if exc is None: self.done.set_result(None) else: self.done.set_exception(exc) def eof_received(self): self.eof_received_fut.set_result(self.cvar.get()) def pause_writing(self): self.pause_writing_fut.set_result(self.cvar.get()) def resume_writing(self): self.resume_writing_fut.set_result(self.cvar.get()) class _Protocol(_BaseProtocol, asyncio.Protocol): def data_received(self, data): self.data_received_fut.set_result(self.cvar.get()) class _BufferedProtocol(_BaseProtocol, asyncio.BufferedProtocol): def get_buffer(self, sizehint): if self.buffered_ctx is None: self.buffered_ctx = self.cvar.get() elif self.cvar.get() != self.buffered_ctx: self.data_received_fut.set_exception(ValueError("{} != {}".format( self.buffered_ctx, self.cvar.get(), ))) return bytearray(65536) def buffer_updated(self, nbytes): if not self.data_received_fut.done(): if self.cvar.get() == self.buffered_ctx: self.data_received_fut.set_result(self.cvar.get()) else: self.data_received_fut.set_exception( ValueError("{} != {}".format( self.buffered_ctx, self.cvar.get(), )) ) class _DatagramProtocol(_BaseProtocol, asyncio.DatagramProtocol): def datagram_received(self, data, addr): self.data_received_fut.set_result(self.cvar.get()) def error_received(self, exc): self.error_received_fut.set_result(self.cvar.get()) class _SubprocessProtocol(_BaseProtocol, asyncio.SubprocessProtocol): def pipe_data_received(self, fd, data): self.data_received_fut.set_result(self.cvar.get()) def pipe_connection_lost(self, fd, exc): self.pipe_ctx.remove(fd) val = self.cvar.get() self.pipe_ctx.add(val) if not any(isinstance(x, int) for x in self.pipe_ctx): if len(self.pipe_ctx) == 1: self.pipe_connection_lost_fut.set_result(val) else: self.pipe_connection_lost_fut.set_exception( AssertionError(str(list(self.pipe_ctx)))) def process_exited(self): self.process_exited_fut.set_result(self.cvar.get()) class _SSLSocketOverSSL: # because wrap_socket() doesn't work correctly on # SSLSocket, we have to do the 2nd level SSL manually def __init__(self, ssl_sock, ctx, **kwargs): self.sock = ssl_sock self.incoming = ssl.MemoryBIO() self.outgoing = ssl.MemoryBIO() self.sslobj = ctx.wrap_bio( self.incoming, self.outgoing, **kwargs) self.do(self.sslobj.do_handshake) def do(self, func, *args): while True: try: rv = func(*args) break except ssl.SSLWantReadError: if self.outgoing.pending: self.sock.send(self.outgoing.read()) self.incoming.write(self.sock.recv(65536)) if self.outgoing.pending: self.sock.send(self.outgoing.read()) return rv def send(self, data): self.do(self.sslobj.write, data) def unwrap(self): self.do(self.sslobj.unwrap) def close(self): self.sock.unwrap() self.sock.close() class _ContextBaseTests(tb.SSLTestCase): ONLYCERT = tb._cert_fullname(__file__, 'ssl_cert.pem') ONLYKEY = tb._cert_fullname(__file__, 'ssl_key.pem') def test_task_decimal_context(self): async def fractions(t, precision, x, y): with decimal.localcontext() as ctx: ctx.prec = precision a = decimal.Decimal(x) / decimal.Decimal(y) await asyncio.sleep(t) b = decimal.Decimal(x) / decimal.Decimal(y ** 2) return a, b async def main(): r1, r2 = await asyncio.gather( fractions(0.1, 3, 1, 3), fractions(0.2, 6, 1, 3)) return r1, r2 r1, r2 = self.loop.run_until_complete(main()) self.assertEqual(str(r1[0]), '0.333') self.assertEqual(str(r1[1]), '0.111') self.assertEqual(str(r2[0]), '0.333333') self.assertEqual(str(r2[1]), '0.111111') def test_task_context_1(self): cvar = contextvars.ContextVar('cvar', default='nope') async def sub(): await asyncio.sleep(0.01) self.assertEqual(cvar.get(), 'nope') cvar.set('something else') async def main(): self.assertEqual(cvar.get(), 'nope') subtask = self.loop.create_task(sub()) cvar.set('yes') self.assertEqual(cvar.get(), 'yes') await subtask self.assertEqual(cvar.get(), 'yes') task = self.loop.create_task(main()) self.loop.run_until_complete(task) def test_task_context_2(self): cvar = contextvars.ContextVar('cvar', default='nope') async def main(): def fut_on_done(fut): # This change must not pollute the context # of the "main()" task. cvar.set('something else') self.assertEqual(cvar.get(), 'nope') for j in range(2): fut = self.loop.create_future() fut.add_done_callback(fut_on_done) cvar.set('yes{}'.format(j)) self.loop.call_soon(fut.set_result, None) await fut self.assertEqual(cvar.get(), 'yes{}'.format(j)) for i in range(3): # Test that task passed its context to add_done_callback: cvar.set('yes{}-{}'.format(i, j)) await asyncio.sleep(0.001) self.assertEqual(cvar.get(), 'yes{}-{}'.format(i, j)) task = self.loop.create_task(main()) self.loop.run_until_complete(task) self.assertEqual(cvar.get(), 'nope') def test_task_context_3(self): cvar = contextvars.ContextVar('cvar', default=-1) # Run 100 Tasks in parallel, each modifying cvar. async def sub(num): for i in range(10): cvar.set(num + i) await asyncio.sleep(random.uniform(0.001, 0.05)) self.assertEqual(cvar.get(), num + i) async def main(): tasks = [] for i in range(100): task = self.loop.create_task(sub(random.randint(0, 10))) tasks.append(task) await asyncio.gather(*tasks, return_exceptions=True) self.loop.run_until_complete(main()) self.assertEqual(cvar.get(), -1) def test_task_context_4(self): cvar = contextvars.ContextVar('cvar', default='nope') class TrackMe: pass tracked = TrackMe() ref = weakref.ref(tracked) async def sub(): cvar.set(tracked) # NoQA self.loop.call_soon(lambda: None) async def main(): await self.loop.create_task(sub()) await asyncio.sleep(0.01) task = self.loop.create_task(main()) self.loop.run_until_complete(task) del tracked self.assertIsNone(ref()) def _run_test(self, method, **switches): switches.setdefault('use_tcp', 'both') use_ssl = switches.setdefault('use_ssl', 'no') in {'yes', 'both'} names = ['factory'] options = [(_Protocol, _BufferedProtocol)] for k, v in switches.items(): if v == 'yes': options.append((True,)) elif v == 'no': options.append((False,)) elif v == 'both': options.append((True, False)) else: raise ValueError(f"Illegal {k}={v}, can only be yes/no/both") names.append(k) for combo in itertools.product(*options): values = dict(zip(names, combo)) with self.subTest(**values): cvar = contextvars.ContextVar('cvar', default='outer') values['proto'] = values.pop('factory')(cvar, loop=self.loop) async def test(): self.assertEqual(cvar.get(), 'outer') cvar.set('inner') tmp_dir = tempfile.TemporaryDirectory() if use_ssl: values['sslctx'] = self._create_server_ssl_context( self.ONLYCERT, self.ONLYKEY) values['client_sslctx'] = \ self._create_client_ssl_context() else: values['sslctx'] = values['client_sslctx'] = None if values['use_tcp']: values['addr'] = ('127.0.0.1', tb.find_free_port()) values['family'] = socket.AF_INET else: values['addr'] = tmp_dir.name + '/test.sock' values['family'] = socket.AF_UNIX try: await method(cvar=cvar, **values) finally: tmp_dir.cleanup() self.loop.run_until_complete(test()) def _run_server_test(self, method, async_sock=False, **switches): async def test(sslctx, client_sslctx, addr, family, **values): if values['use_tcp']: srv = await self.loop.create_server( lambda: values['proto'], *addr, ssl=sslctx) else: srv = await self.loop.create_unix_server( lambda: values['proto'], addr, ssl=sslctx) s = socket.socket(family) if async_sock: s.setblocking(False) await self.loop.sock_connect(s, addr) else: await self.loop.run_in_executor( None, s.connect, addr) if values['use_ssl']: values['ssl_sock'] = await self.loop.run_in_executor( None, client_sslctx.wrap_socket, s) try: await method(s=s, **values) finally: if values['use_ssl']: values['ssl_sock'].close() s.close() srv.close() await srv.wait_closed() return self._run_test(test, **switches) def test_create_server_protocol_factory_context(self): async def test(cvar, proto, use_tcp, family, addr, **_): factory_called_future = self.loop.create_future() def factory(): try: self.assertEqual(cvar.get(), 'inner') except Exception as e: factory_called_future.set_exception(e) else: factory_called_future.set_result(None) return proto if use_tcp: srv = await self.loop.create_server(factory, *addr) else: srv = await self.loop.create_unix_server(factory, addr) s = socket.socket(family) with s: s.setblocking(False) await self.loop.sock_connect(s, addr) try: await factory_called_future finally: srv.close() await proto.done await srv.wait_closed() self._run_test(test) def test_create_server_connection_protocol(self): async def test(proto, s, **_): inner = await proto.connection_made_fut self.assertEqual(inner, "inner") await self.loop.sock_sendall(s, b'data') inner = await proto.data_received_fut self.assertEqual(inner, "inner") s.shutdown(socket.SHUT_WR) inner = await proto.eof_received_fut self.assertEqual(inner, "inner") s.close() await proto.done self.assertEqual(proto.connection_lost_ctx, "inner") self._run_server_test(test, async_sock=True) def test_create_ssl_server_connection_protocol(self): async def test(cvar, proto, ssl_sock, **_): def resume_reading(transport): cvar.set("resume_reading") transport.resume_reading() try: inner = await proto.connection_made_fut self.assertEqual(inner, "inner") await self.loop.run_in_executor(None, ssl_sock.send, b'data') inner = await proto.data_received_fut self.assertEqual(inner, "inner") if self.implementation != 'asyncio': # this seems to be a bug in asyncio proto.data_received_fut = self.loop.create_future() proto.transport.pause_reading() await self.loop.run_in_executor(None, ssl_sock.send, b'data') self.loop.call_soon(resume_reading, proto.transport) inner = await proto.data_received_fut self.assertEqual(inner, "inner") await self.loop.run_in_executor(None, ssl_sock.unwrap) else: ssl_sock.shutdown(socket.SHUT_WR) inner = await proto.eof_received_fut self.assertEqual(inner, "inner") await self.loop.run_in_executor(None, ssl_sock.close) await proto.done self.assertEqual(proto.connection_lost_ctx, "inner") finally: if self.implementation == 'asyncio': # mute resource warning in asyncio proto.transport.close() self._run_server_test(test, use_ssl='yes') def test_create_server_manual_connection_lost(self): if self.implementation == 'asyncio': raise unittest.SkipTest('this seems to be a bug in asyncio') async def test(proto, cvar, **_): def close(): cvar.set('closing') proto.transport.close() inner = await proto.connection_made_fut self.assertEqual(inner, "inner") self.loop.call_soon(close) await proto.done self.assertEqual(proto.connection_lost_ctx, "inner") self._run_server_test(test, async_sock=True) def test_create_ssl_server_manual_connection_lost(self): async def test(proto, cvar, ssl_sock, **_): def close(): cvar.set('closing') proto.transport.close() inner = await proto.connection_made_fut self.assertEqual(inner, "inner") if self.implementation == 'asyncio': self.loop.call_soon(close) else: # asyncio doesn't have the flushing phase # put the incoming data on-hold proto.transport.pause_reading() # send data await self.loop.run_in_executor(None, ssl_sock.send, b'hello') # schedule a proactive transport close which will trigger # the flushing process to retrieve the remaining data self.loop.call_soon(close) # turn off the reading lock now (this also schedules a # resume operation after transport.close, therefore it # won't affect our test) proto.transport.resume_reading() await asyncio.sleep(0) await self.loop.run_in_executor(None, ssl_sock.unwrap) await proto.done self.assertEqual(proto.connection_lost_ctx, "inner") self.assertFalse(proto.data_received_fut.done()) self._run_server_test(test, use_ssl='yes') def test_create_connection_protocol(self): async def test(cvar, proto, addr, sslctx, client_sslctx, family, use_sock, use_ssl, use_tcp): ss = socket.socket(family) ss.bind(addr) ss.listen(1) def accept(): sock, _ = ss.accept() if use_ssl: sock = sslctx.wrap_socket(sock, server_side=True) return sock async def write_over(): cvar.set("write_over") count = 0 if use_ssl: proto.transport.set_write_buffer_limits(high=256, low=128) while not proto.transport.get_write_buffer_size(): proto.transport.write(b'q' * 16384) count += 1 else: proto.transport.write(b'q' * 16384) proto.transport.set_write_buffer_limits(high=256, low=128) count += 1 return count s = self.loop.run_in_executor(None, accept) try: method = ('create_connection' if use_tcp else 'create_unix_connection') params = {} if use_sock: cs = socket.socket(family) cs.connect(addr) params['sock'] = cs if use_ssl: params['server_hostname'] = '127.0.0.1' elif use_tcp: params['host'] = addr[0] params['port'] = addr[1] else: params['path'] = addr if use_ssl: params['server_hostname'] = '127.0.0.1' if use_ssl: params['ssl'] = client_sslctx await getattr(self.loop, method)(lambda: proto, **params) s = await s inner = await proto.connection_made_fut self.assertEqual(inner, "inner") await self.loop.run_in_executor(None, s.send, b'data') inner = await proto.data_received_fut self.assertEqual(inner, "inner") if self.implementation != 'asyncio': # asyncio bug count = await self.loop.create_task(write_over()) inner = await proto.pause_writing_fut self.assertEqual(inner, "inner") for i in range(count): await self.loop.run_in_executor(None, s.recv, 16384) inner = await proto.resume_writing_fut self.assertEqual(inner, "inner") if use_ssl and self.implementation != 'asyncio': await self.loop.run_in_executor(None, s.unwrap) else: s.shutdown(socket.SHUT_WR) inner = await proto.eof_received_fut self.assertEqual(inner, "inner") s.close() await proto.done self.assertEqual(proto.connection_lost_ctx, "inner") finally: ss.close() proto.transport.close() self._run_test(test, use_sock='both', use_ssl='both') def test_start_tls(self): if self.implementation == 'asyncio': raise unittest.SkipTest('this seems to be a bug in asyncio') async def test(cvar, proto, addr, sslctx, client_sslctx, family, ssl_over_ssl, use_tcp, **_): ss = socket.socket(family) ss.bind(addr) ss.listen(1) def accept(): sock, _ = ss.accept() sock = sslctx.wrap_socket(sock, server_side=True) if ssl_over_ssl: sock = _SSLSocketOverSSL(sock, sslctx, server_side=True) return sock s = self.loop.run_in_executor(None, accept) transport = None try: if use_tcp: await self.loop.create_connection(lambda: proto, *addr) else: await self.loop.create_unix_connection(lambda: proto, addr) inner = await proto.connection_made_fut self.assertEqual(inner, "inner") cvar.set('start_tls') transport = await self.loop.start_tls( proto.transport, proto, client_sslctx, server_hostname='127.0.0.1', ) if ssl_over_ssl: cvar.set('start_tls_over_tls') transport = await self.loop.start_tls( transport, proto, client_sslctx, server_hostname='127.0.0.1', ) s = await s await self.loop.run_in_executor(None, s.send, b'data') inner = await proto.data_received_fut self.assertEqual(inner, "inner") await self.loop.run_in_executor(None, s.unwrap) inner = await proto.eof_received_fut self.assertEqual(inner, "inner") s.close() await proto.done self.assertEqual(proto.connection_lost_ctx, "inner") finally: ss.close() if transport: transport.close() self._run_test(test, use_ssl='yes', ssl_over_ssl='both') def test_connect_accepted_socket(self): async def test(proto, addr, family, sslctx, client_sslctx, use_ssl, **_): ss = socket.socket(family) ss.bind(addr) ss.listen(1) s = self.loop.run_in_executor(None, ss.accept) cs = socket.socket(family) cs.connect(addr) s, _ = await s try: if use_ssl: cs = self.loop.run_in_executor( None, client_sslctx.wrap_socket, cs) await self.loop.connect_accepted_socket(lambda: proto, s, ssl=sslctx) cs = await cs else: await self.loop.connect_accepted_socket(lambda: proto, s) inner = await proto.connection_made_fut self.assertEqual(inner, "inner") await self.loop.run_in_executor(None, cs.send, b'data') inner = await proto.data_received_fut self.assertEqual(inner, "inner") if use_ssl and self.implementation != 'asyncio': await self.loop.run_in_executor(None, cs.unwrap) else: cs.shutdown(socket.SHUT_WR) inner = await proto.eof_received_fut self.assertEqual(inner, "inner") cs.close() await proto.done self.assertEqual(proto.connection_lost_ctx, "inner") finally: proto.transport.close() ss.close() self._run_test(test, use_ssl='both') def test_subprocess_protocol(self): cvar = contextvars.ContextVar('cvar', default='outer') proto = _SubprocessProtocol(cvar, loop=self.loop) async def test(): self.assertEqual(cvar.get(), 'outer') cvar.set('inner') await self.loop.subprocess_exec( lambda: proto, sys.executable, b'-c', b';'.join((b'import sys', b'data = sys.stdin.buffer.read()', b'sys.stdout.buffer.write(data)'))) try: inner = await proto.connection_made_fut self.assertEqual(inner, "inner") proto.transport.get_pipe_transport(0).write(b'data') proto.transport.get_pipe_transport(0).write_eof() inner = await proto.data_received_fut self.assertEqual(inner, "inner") inner = await proto.pipe_connection_lost_fut self.assertEqual(inner, "inner") inner = await proto.process_exited_fut if self.implementation != 'asyncio': # bug in asyncio self.assertEqual(inner, "inner") await proto.done if self.implementation != 'asyncio': # bug in asyncio self.assertEqual(proto.connection_lost_ctx, "inner") finally: proto.transport.close() self.loop.run_until_complete(test()) def test_datagram_protocol(self): cvar = contextvars.ContextVar('cvar', default='outer') proto = _DatagramProtocol(cvar, loop=self.loop) server_addr = ('127.0.0.1', 8888) client_addr = ('127.0.0.1', 0) async def run(): self.assertEqual(cvar.get(), 'outer') cvar.set('inner') def close(): cvar.set('closing') proto.transport.close() try: await self.loop.create_datagram_endpoint( lambda: proto, local_addr=server_addr) inner = await proto.connection_made_fut self.assertEqual(inner, "inner") s = socket.socket(socket.AF_INET, type=socket.SOCK_DGRAM) s.bind(client_addr) s.sendto(b'data', server_addr) inner = await proto.data_received_fut self.assertEqual(inner, "inner") self.loop.call_soon(close) await proto.done if self.implementation != 'asyncio': # bug in asyncio self.assertEqual(proto.connection_lost_ctx, "inner") finally: proto.transport.close() s.close() # let transports close await asyncio.sleep(0.1) self.loop.run_until_complete(run()) class Test_UV_Context(_ContextBaseTests, tb.UVTestCase): pass class Test_AIO_Context(_ContextBaseTests, tb.AIOTestCase): pass