From bf68705eb79145ef5e9ad5075ab1aaf3b20f1607 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 11 Sep 2018 17:07:31 -0400 Subject: [PATCH 1/3] Set ALLOW_MEMORY_GROWTH=1 --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index fd6b23612..0257c3931 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ LDFLAGS=\ $(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \ -s "BINARYEN_METHOD='native-wasm'" \ -s TOTAL_MEMORY=536870912 \ + -s ALLOW_MEMORY_GROWTH=1 \ -s MAIN_MODULE=1 \ -s EMULATED_FUNCTION_POINTERS=1 \ -s EMULATE_FUNCTION_POINTER_CASTS=1 \ From 65817abc7d4099dbbb1ac8c8d0a44a800d35ca41 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 12 Sep 2018 10:14:51 -0400 Subject: [PATCH 2/3] Add test --- Makefile | 7 +------ test/conftest.py | 23 ++++++++++++++++++----- test/data.cgi | 4 ++++ test/largish.json.cgi | 33 +++++++++++++++++++++++++++++++++ test/test_pandas.py | 19 +++++++++++++++++++ test/test_python.py | 10 +++++++++- 6 files changed, 84 insertions(+), 12 deletions(-) create mode 100755 test/data.cgi create mode 100755 test/largish.json.cgi diff --git a/Makefile b/Makefile index 0257c3931..e06269919 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ LDFLAGS=\ -s MODULARIZE=1 \ $(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \ -s "BINARYEN_METHOD='native-wasm'" \ - -s TOTAL_MEMORY=536870912 \ + -s TOTAL_MEMORY=1073741824 \ -s ALLOW_MEMORY_GROWTH=1 \ -s MAIN_MODULE=1 \ -s EMULATED_FUNCTION_POINTERS=1 \ @@ -51,7 +51,6 @@ all: build/pyodide.asm.js \ build/renderedhtml.css \ build/test.data \ build/packages.json \ - build/test_data.txt \ build/test.html @@ -108,10 +107,6 @@ test: all pytest test/ -v -build/test_data.txt: test/data.txt - cp test/data.txt build/test_data.txt - - lint: flake8 src flake8 test diff --git a/test/conftest.py b/test/conftest.py index 68893da3f..06f544c6c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -214,16 +214,29 @@ def run_web_server(q): print("Running webserver...") os.chdir(BUILD_PATH) - Handler = http.server.SimpleHTTPRequestHandler - Handler.extensions_map['.wasm'] = 'application/wasm' - def dummy_log(*args, **kwargs): - pass - Handler.log_message = dummy_log + class Handler(http.server.CGIHTTPRequestHandler): + def translate_path(self, path): + if path.startswith('/test/'): + return TEST_PATH / path[6:] + return super(Handler, self).translate_path(path) + + def is_cgi(self): + if self.path.startswith('/test/') and self.path.endswith('.cgi'): + self.cgi_info = '/test', self.path[6:] + return True + return False + + def log_message(self, *args, **kwargs): + pass + + Handler.extensions_map['.wasm'] = 'application/wasm' with socketserver.TCPServer(("", 0), Handler) as httpd: host, port = httpd.server_address print("serving at port", port) + httpd.server_name = 'test-server' + httpd.server_port = port q.put(port) def service_actions(): diff --git a/test/data.cgi b/test/data.cgi new file mode 100755 index 000000000..cb8d07314 --- /dev/null +++ b/test/data.cgi @@ -0,0 +1,4 @@ +#!/usr/bin/env python +print("Content-Type: text/text") +print() +print("HELLO") diff --git a/test/largish.json.cgi b/test/largish.json.cgi new file mode 100755 index 000000000..bd462c958 --- /dev/null +++ b/test/largish.json.cgi @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import json +import random +import sys + +random.seed(0) + +print("Content-Type: application/json") +print() + +columns = [ + ('column0', lambda: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX'), + ('column1', lambda: random.choice([ + 'notification-interval-longer', 'notification-interval-short', 'control'])), + ('column2', lambda: random.choice([True, False])), + ('column3', lambda: random.randint(0, 4)), + ('column4', lambda: random.randint(0, 4)), + ('column5', lambda: random.randint(0, 4)), + ('column6', lambda: random.randint(0, 4)), + ('column7', lambda: random.randint(0, 4)) +] +N_ROWS = 91746 + +data = {} + +for name, generator in columns: + column = {} + for i in range(N_ROWS): + column[str(i)] = generator() + data[name] = column + +json.dump(data, sys.stdout) diff --git a/test/test_pandas.py b/test/test_pandas.py index 3726a56f0..57f59a167 100644 --- a/test/test_pandas.py +++ b/test/test_pandas.py @@ -16,3 +16,22 @@ def test_extra_import(selenium, request): selenium.load_package("pandas") selenium.run("from pandas import Series, DataFrame, Panel") + + +def test_load_largish_file(selenium_standalone, request): + selenium = selenium_standalone + + if selenium.browser == 'chrome': + request.applymarker(pytest.mark.xfail( + run=False, reason='chrome not supported')) + + selenium.load_package("pandas") + selenium.load_package("matplotlib") + + selenium.run(""" + import pyodide + import matplotlib.pyplot as plt + import pandas as pd + + pd.read_json(pyodide.open_url('test/largish.json.cgi')) + """) diff --git a/test/test_python.py b/test/test_python.py index c4000ad3e..aa87a0623 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -317,7 +317,15 @@ def test_open_url(selenium): assert selenium.run( """ import pyodide - pyodide.open_url('test_data.txt').read() + pyodide.open_url('test/data.txt').read() + """) == 'HELLO\n' + + +def test_open_url_cgi(selenium): + assert selenium.run( + """ + import pyodide + pyodide.open_url('test/data.cgi').read() """) == 'HELLO\n' From 20361551a864fdbd7d25b0342b6ba96a2f2cd691 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 12 Sep 2018 16:26:51 -0400 Subject: [PATCH 3/3] Stream the data from the CGI script --- test/largish.json.cgi | 50 +++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/test/largish.json.cgi b/test/largish.json.cgi index bd462c958..8c5455425 100755 --- a/test/largish.json.cgi +++ b/test/largish.json.cgi @@ -6,28 +6,42 @@ import sys random.seed(0) -print("Content-Type: application/json") -print() - columns = [ - ('column0', lambda: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX'), - ('column1', lambda: random.choice([ - 'notification-interval-longer', 'notification-interval-short', 'control'])), - ('column2', lambda: random.choice([True, False])), - ('column3', lambda: random.randint(0, 4)), - ('column4', lambda: random.randint(0, 4)), - ('column5', lambda: random.randint(0, 4)), - ('column6', lambda: random.randint(0, 4)), - ('column7', lambda: random.randint(0, 4)) + ('column0', lambda: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX'), + ('column1', lambda: random.choice([ + 'notification-interval-longer', 'notification-interval-short', 'control'])), + ('column2', lambda: random.choice([True, False])), + ('column3', lambda: random.randint(0, 4)), + ('column4', lambda: random.randint(0, 4)), + ('column5', lambda: random.randint(0, 4)), + ('column6', lambda: random.randint(0, 4)), + ('column7', lambda: random.randint(0, 4)) ] -N_ROWS = 91746 + +N_ROWS = 91746 # the output JSON size will be ~15 MB/10k rows + + +class StreamDict(dict): + """ + To serialize to JSON, we create an iterable object that inherits from a + known supported object type: dict. + """ + def __init__(self, generator): + self.generator = generator + + def items(self): + for i in range(N_ROWS): + yield i, self.generator() + + def __len__(self): + return 1 + data = {} - for name, generator in columns: - column = {} - for i in range(N_ROWS): - column[str(i)] = generator() - data[name] = column + data[name] = StreamDict(generator) + +print("Content-Type: application/json") +print() json.dump(data, sys.stdout)