diff --git a/Makefile b/Makefile index 01c76ab10..6d6eff5ec 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,8 @@ LDFLAGS=\ -s MODULARIZE=1 \ $(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \ -s "BINARYEN_METHOD='native-wasm'" \ - -s TOTAL_MEMORY=536870912 \ + -s TOTAL_MEMORY=1073741824 \ + -s ALLOW_MEMORY_GROWTH=1 \ -s MAIN_MODULE=1 \ -s EMULATED_FUNCTION_POINTERS=1 \ -s EMULATE_FUNCTION_POINTER_CASTS=1 \ @@ -50,7 +51,6 @@ all: build/pyodide.asm.js \ build/renderedhtml.css \ build/test.data \ build/packages.json \ - build/test_data.txt \ build/test.html @@ -107,10 +107,6 @@ test: all pytest test/ -v -build/test_data.txt: test/data.txt - cp test/data.txt build/test_data.txt - - lint: flake8 src flake8 test diff --git a/test/conftest.py b/test/conftest.py index 9362c7e0b..6fba04039 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -214,16 +214,29 @@ def run_web_server(q): print("Running webserver...") os.chdir(BUILD_PATH) - Handler = http.server.SimpleHTTPRequestHandler - Handler.extensions_map['.wasm'] = 'application/wasm' - def dummy_log(*args, **kwargs): - pass - Handler.log_message = dummy_log + class Handler(http.server.CGIHTTPRequestHandler): + def translate_path(self, path): + if path.startswith('/test/'): + return TEST_PATH / path[6:] + return super(Handler, self).translate_path(path) + + def is_cgi(self): + if self.path.startswith('/test/') and self.path.endswith('.cgi'): + self.cgi_info = '/test', self.path[6:] + return True + return False + + def log_message(self, *args, **kwargs): + pass + + Handler.extensions_map['.wasm'] = 'application/wasm' with socketserver.TCPServer(("", 0), Handler) as httpd: host, port = httpd.server_address print("serving at port", port) + httpd.server_name = 'test-server' + httpd.server_port = port q.put(port) def service_actions(): diff --git a/test/data.cgi b/test/data.cgi new file mode 100755 index 000000000..cb8d07314 --- /dev/null +++ b/test/data.cgi @@ -0,0 +1,4 @@ +#!/usr/bin/env python +print("Content-Type: text/text") +print() +print("HELLO") diff --git a/test/largish.json.cgi b/test/largish.json.cgi new file mode 100755 index 000000000..8c5455425 --- /dev/null +++ b/test/largish.json.cgi @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +import json +import random +import sys + +random.seed(0) + +columns = [ + ('column0', lambda: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX'), + ('column1', lambda: random.choice([ + 'notification-interval-longer', 'notification-interval-short', 'control'])), + ('column2', lambda: random.choice([True, False])), + ('column3', lambda: random.randint(0, 4)), + ('column4', lambda: random.randint(0, 4)), + ('column5', lambda: random.randint(0, 4)), + ('column6', lambda: random.randint(0, 4)), + ('column7', lambda: random.randint(0, 4)) +] + +N_ROWS = 91746 # the output JSON size will be ~15 MB/10k rows + + +class StreamDict(dict): + """ + To serialize to JSON, we create an iterable object that inherits from a + known supported object type: dict. + """ + def __init__(self, generator): + self.generator = generator + + def items(self): + for i in range(N_ROWS): + yield i, self.generator() + + def __len__(self): + return 1 + + +data = {} +for name, generator in columns: + data[name] = StreamDict(generator) + + +print("Content-Type: application/json") +print() +json.dump(data, sys.stdout) diff --git a/test/test_pandas.py b/test/test_pandas.py index 3726a56f0..57f59a167 100644 --- a/test/test_pandas.py +++ b/test/test_pandas.py @@ -16,3 +16,22 @@ def test_extra_import(selenium, request): selenium.load_package("pandas") selenium.run("from pandas import Series, DataFrame, Panel") + + +def test_load_largish_file(selenium_standalone, request): + selenium = selenium_standalone + + if selenium.browser == 'chrome': + request.applymarker(pytest.mark.xfail( + run=False, reason='chrome not supported')) + + selenium.load_package("pandas") + selenium.load_package("matplotlib") + + selenium.run(""" + import pyodide + import matplotlib.pyplot as plt + import pandas as pd + + pd.read_json(pyodide.open_url('test/largish.json.cgi')) + """) diff --git a/test/test_python.py b/test/test_python.py index c4000ad3e..aa87a0623 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -317,7 +317,15 @@ def test_open_url(selenium): assert selenium.run( """ import pyodide - pyodide.open_url('test_data.txt').read() + pyodide.open_url('test/data.txt').read() + """) == 'HELLO\n' + + +def test_open_url_cgi(selenium): + assert selenium.run( + """ + import pyodide + pyodide.open_url('test/data.cgi').read() """) == 'HELLO\n'