From 409fe3b3f4b080fe614359c409b5c78bb94dde2f Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 23 Jul 2018 17:19:47 -0400 Subject: [PATCH 01/10] Use ccache --- .circleci/config.yml | 5 +++-- .gitignore | 1 + Makefile | 8 +++++++- Makefile.envs | 2 +- cpython/Setup.local | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c0e79f4fe..e3b1d529f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,7 @@ jobs: # Set up the Debian testing repo, and then install g++ from there... sudo bash -c "echo \"deb http://ftp.us.debian.org/debian testing main contrib non-free\" >> /etc/apt/sources.list" sudo apt-get update - sudo apt-get install node-less cmake build-essential clang-format-6.0 flake8 uglifyjs python3-yaml chromium + sudo apt-get install node-less cmake build-essential clang-format-6.0 flake8 uglifyjs python3-yaml chromium ccache sudo apt-get install -t testing g++-8 sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-6 60 --slave /usr/bin/g++ g++ /usr/bin/g++-6 sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8 @@ -47,7 +47,7 @@ jobs: - restore_cache: keys: - - v1-emsdk-{{ checksum "emsdk/Makefile" }}-v4 + - v1-emsdk-{{ checksum "emsdk/Makefile" }}-v5 - run: name: build @@ -58,6 +58,7 @@ jobs: - save_cache: paths: - ./emsdk/emsdk + - $HOME/.ccache key: v1-emsdk-{{ checksum "emsdk/Makefile" }}-v4 - run: diff --git a/.gitignore b/.gitignore index 31de30260..92dbc2d7c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ build downloads host installs +ccache /root/ /build/ diff --git a/Makefile b/Makefile index 1a5006f48..ce7e1cef3 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ LDFLAGS=\ -s WASM=1 \ -s SWAPPABLE_ASM_MODULE=1 \ -s USE_FREETYPE=1 \ + -s USE_LIBPNG=1 \ -std=c++14 \ -lstdc++ \ --memory-init-file 0 @@ -48,6 +49,11 @@ all: build/pyodide.asm.js \ build/renderedhtml.css \ build/test.data \ build/packages.json + if hash ccache &>/dev/null; then \ + mkdir -p $(PYODIDE_ROOT)/ccache ; \ + ln -s `which ccache` $(PYODIDE_ROOT)/ccache/emcc ; \ + ln -s `which ccache` $(PYODIDE_ROOT)/ccache/em++ ; \ + fi build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \ @@ -126,7 +132,7 @@ clean: %.bc: %.c $(CPYTHONLIB) - $(CC) -o $@ $< $(CFLAGS) + $(CC) -o $@ -c $< $(CFLAGS) build/test.data: $(CPYTHONLIB) diff --git a/Makefile.envs b/Makefile.envs index 2f1bf651e..b94e90988 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -1,4 +1,4 @@ -export PATH := $(PYODIDE_ROOT)/emsdk/emsdk:$(PYODIDE_ROOT)/emsdk/emsdk/clang/tag-e-1.38.4/build_tag-e1.38.4_64/bin:$(PYODIDE_ROOT)/emsdk/emsdk/node/8.9.1_64bit/bin:$(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-1.38.4:$(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-1.38.4_64bit_binaryen/bin:$(PATH) +export PATH := $(PYODIDE_ROOT)/ccache:$(PYODIDE_ROOT)/emsdk/emsdk:$(PYODIDE_ROOT)/emsdk/emsdk/clang/tag-e-1.38.4/build_tag-e1.38.4_64/bin:$(PYODIDE_ROOT)/emsdk/emsdk/node/8.9.1_64bit/bin:$(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-1.38.4:$(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-1.38.4_64bit_binaryen/bin:$(PATH) export EMSDK = $(PYODIDE_ROOT)/emsdk/emsdk export EM_CONFIG = $(PYODIDE_ROOT)/emsdk/emsdk/.emscripten diff --git a/cpython/Setup.local b/cpython/Setup.local index 24278bb01..80517c913 100644 --- a/cpython/Setup.local +++ b/cpython/Setup.local @@ -24,7 +24,7 @@ _posixsubprocess _posixsubprocess.c binascii binascii.c -zlib zlibmodule.c -IModules/zlib zlib/adler32.c zlib/crc32.c zlib/deflate.c zlib/infback.c zlib/inffast.c zlib/inflate.c zlib/inftrees.c zlib/trees.c zlib/zutil.c zlib/compress.c zlib/uncompr.c zlib/gzclose.c zlib/gzlib.c zlib/gzread.c zlib/gzwrite.c +zlib zlibmodule.c -IModules/zlib pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI -DXML_POOR_ENTROPY From ad3a7599fc2e261a8f5193b619a583a41f20599e Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 23 Jul 2018 17:21:11 -0400 Subject: [PATCH 02/10] Recreate symlinks --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ce7e1cef3..7a6beecd9 100644 --- a/Makefile +++ b/Makefile @@ -51,8 +51,8 @@ all: build/pyodide.asm.js \ build/packages.json if hash ccache &>/dev/null; then \ mkdir -p $(PYODIDE_ROOT)/ccache ; \ - ln -s `which ccache` $(PYODIDE_ROOT)/ccache/emcc ; \ - ln -s `which ccache` $(PYODIDE_ROOT)/ccache/em++ ; \ + ln -f -s `which ccache` $(PYODIDE_ROOT)/ccache/emcc ; \ + ln -f -s `which ccache` $(PYODIDE_ROOT)/ccache/em++ ; \ fi From 876c9a5bc0acbe0d3f805b189a4a82cd78adccde Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 24 Jul 2018 08:56:50 -0400 Subject: [PATCH 03/10] Turn off full Python tests --- test/test_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_python.py b/test/test_python.py index 4647cfda8..ea8efc6e2 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -286,7 +286,7 @@ def test_run_core_python_test(python_test, selenium): def pytest_generate_tests(metafunc): if 'python_test' in metafunc.fixturenames: test_modules = [] - if True: + if 'CIRCLECI' not in os.environ: with open( str(pathlib.Path(__file__).parents[0] / "python_tests.txt")) as fp: From f2426e57192e88b0aadd897d7b8cf594b86f1d1a Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 24 Jul 2018 09:09:42 -0400 Subject: [PATCH 04/10] Fix how the ccache symlinks are installed --- Makefile | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 7a6beecd9..12d067596 100644 --- a/Makefile +++ b/Makefile @@ -49,11 +49,6 @@ all: build/pyodide.asm.js \ build/renderedhtml.css \ build/test.data \ build/packages.json - if hash ccache &>/dev/null; then \ - mkdir -p $(PYODIDE_ROOT)/ccache ; \ - ln -f -s `which ccache` $(PYODIDE_ROOT)/ccache/emcc ; \ - ln -f -s `which ccache` $(PYODIDE_ROOT)/ccache/em++ ; \ - fi build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \ @@ -167,7 +162,25 @@ root/.built: \ touch root/.built -$(CPYTHONLIB): emsdk/emsdk/.complete +ccache/emcc: + if hash ccache &>/dev/null; then \ + mkdir -p $(PYODIDE_ROOT)/ccache ; \ + ln -s `which ccache` $(PYODIDE_ROOT)/ccache/emcc ; \ + else \ + ln -s emsdk/emsdk/emscripten/tag-1.38.4/emcc $(PYODIDE_ROOT)/ccache/emcc; \ + fi + + +ccache/em++: + if hash ccache &>/dev/null; then \ + mkdir -p $(PYODIDE_ROOT)/ccache ; \ + ln -s `which ccache` $(PYODIDE_ROOT)/ccache/em++ ; \ + else \ + ln -s emsdk/emsdk/emscripten/tag-1.38.4/em++ $(PYODIDE_ROOT)/ccache/em++; \ + fi + + +$(CPYTHONLIB): emsdk/emsdk/.complete ccache/emcc ccache/em++ make -C $(CPYTHONROOT) From 2261af6dae612e1b262a7f2cd8b95532fe71b0c0 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 24 Jul 2018 09:10:05 -0400 Subject: [PATCH 05/10] Mention ccache in the README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f50e64b37..bcbcd61cb 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Additional build prerequisites are: - PyYAML - [lessc](https://lesscss.org/) to compile less to css. - [uglifyjs](https://github.com/mishoo/UglifyJS) to minify Javascript builds. +- [ccache](https://ccache.samba.org) (optional) recommended for much faster rebuilds. `make` From 4bd13f2689b79eca62d248d79201f01b0c4fbb50 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 24 Jul 2018 09:19:40 -0400 Subject: [PATCH 06/10] Update key in both places --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e3b1d529f..f976c505c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,7 +59,7 @@ jobs: paths: - ./emsdk/emsdk - $HOME/.ccache - key: v1-emsdk-{{ checksum "emsdk/Makefile" }}-v4 + key: v1-emsdk-{{ checksum "emsdk/Makefile" }}-v5 - run: name: test From 5ea44b2d1870aa7a6a034de4bd5c07e67f22de7f Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 24 Jul 2018 11:18:01 -0400 Subject: [PATCH 07/10] Trivial change to trigger rebuild --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bcbcd61cb..268685f3c 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Additional build prerequisites are: - [uglifyjs](https://github.com/mishoo/UglifyJS) to minify Javascript builds. - [ccache](https://ccache.samba.org) (optional) recommended for much faster rebuilds. + `make` # Testing From d82396a09dafeb4ca896444b0bcb5b3ab4086af2 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 24 Jul 2018 11:27:58 -0400 Subject: [PATCH 08/10] Fix how ~/.ccache is specified --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f976c505c..0e6dfebd4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ jobs: - restore_cache: keys: - - v1-emsdk-{{ checksum "emsdk/Makefile" }}-v5 + - v1-emsdk-{{ checksum "emsdk/Makefile" }}-v6 - run: name: build @@ -58,8 +58,8 @@ jobs: - save_cache: paths: - ./emsdk/emsdk - - $HOME/.ccache - key: v1-emsdk-{{ checksum "emsdk/Makefile" }}-v5 + - ~/.ccache + key: v1-emsdk-{{ checksum "emsdk/Makefile" }}-v6 - run: name: test From 144e6f04bc025f11d921f008e65191d0cb8261e2 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 24 Jul 2018 13:02:49 -0400 Subject: [PATCH 09/10] LINT --- test/test_python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_python.py b/test/test_python.py index ea8efc6e2..78fde14c2 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -1,3 +1,4 @@ +import os import pathlib import time From 4fe9f5cda4cb8e6a4683708b81b496a89a5d2ae0 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 24 Jul 2018 15:40:25 -0400 Subject: [PATCH 10/10] Display ccache stats after build --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e6dfebd4..2868edc64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,6 +54,7 @@ jobs: no_output_timeout: 1200 command: | make + ccache -s - save_cache: paths: