BLD Use outer ccache wrapper (#1805)

.circleci/config.yml

@@ -8,6 +8,9 @@ defaults: &defaults
     - EMSDK_NUM_CORES: 3
       EMCC_CORES: 3
       PYODIDE_JOBS: 3
+      # Make sure the ccache dir is consistent between core and package builds
+      # (it's not the case otherwise)
+      CCACHE_DIR: /root/.ccache/
 
 jobs:
   lint:
@@ -41,20 +44,29 @@ jobs:
 
       - restore_cache:
           keys:
-            - -{{ checksum "Makefile.envs" }}-v20201205-
+            - -core-{{ checksum "Makefile.envs" }}-{{ checksum "python/Makefile" }}-v20210910-
 
       - run:
           name: build emsdk
           no_output_timeout: 1200
           command: |
+            # This is necessary to use the ccache from emsdk
+            source pyodide_env.sh
+
             ccache -z
             make -C emsdk
             ccache -s
 
+            # Set mtime for EM_CONFIG to avoid ccache cache misses
+            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten
+
       - run:
           name: build cpython
           no_output_timeout: 1200
           command: |
+            # This is necessary to use the ccache from emsdk
+            source pyodide_env.sh
+
             ccache -z
             make -C cpython
             ccache -s
@@ -63,6 +75,9 @@ jobs:
           name: build pyodide core
           no_output_timeout: 1200
           command: |
+            # This is necessary to use the ccache from emsdk
+            source pyodide_env.sh
+
             ccache -z
             PYODIDE_PACKAGES="core" make
             ccache -s
@@ -73,8 +88,8 @@ jobs:
 
       - save_cache:
           paths:
-            - ~/.ccache
-          key: -{{ checksum "Makefile.envs" }}-v20201205-{{ .BuildNum }}
+            - /root/.ccache
+          key: -core-{{ checksum "Makefile.envs" }}-{{ checksum "python/Makefile" }}-v20210910-
 
       - run:
           name: Clean up workspace
@@ -100,12 +115,17 @@ jobs:
 
       - restore_cache:
           keys:
-            - -{{ checksum "Makefile.envs" }}-v20201205-
+            - -pkg-{{ checksum "Makefile.envs" }}-v20210911-
 
       - run:
           name: build packages
           no_output_timeout: 1800
           command: |
+            source pyodide_env.sh
+
+            # Set mtime for EM_CONFIG to avoid ccache cache misses
+            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten
+
             ccache -z
             PYODIDE_PACKAGES='*' make -C packages
             ccache -s
@@ -116,8 +136,8 @@ jobs:
 
       - save_cache:
           paths:
-            - ~/.ccache
-          key: -{{ checksum "Makefile.envs" }}-v20201205-{{ .BuildNum }}
+            - /root/.ccache
+          key: -pkg-{{ checksum "Makefile.envs" }}-v20210911-
 
       - persist_to_workspace:
           root: .

Makefile.envs

@@ -28,7 +28,6 @@ export PYODIDE_BASE_URL?=./
 export PYODIDE=1
 # This is the legacy environment variable used for the aforementioned purpose
 export PYODIDE_PACKAGE_ABI=1
-export EM_COMPILER_WRAPPER=ccache
 
 export OPTFLAGS=-O2
 export CFLAGS_BASE=\

docs/project/changelog.md

@@ -23,6 +23,9 @@ substitutions:
   compile and post build scripts.
   {pr}`1706`
 
+- {{ Enhancement }} Better support for ccache when building Pyodide
+  {pr}`1805`
+
 ### Uncategorized
 
 ## Version 0.18.1 (unreleased)

emsdk/Makefile

@@ -6,7 +6,7 @@ all: emsdk/.complete
 emsdk/.complete: ../Makefile.envs $(wildcard patches/*.patch)
 	if [ -d emsdk ]; then rm -rf emsdk; fi
 	git clone --depth 1 https://github.com/emscripten-core/emsdk.git
-	cd emsdk && ./emsdk install --build=Release $(PYODIDE_EMSCRIPTEN_VERSION)
+	cd emsdk && ./emsdk install --build=Release $(PYODIDE_EMSCRIPTEN_VERSION) ccache-git-emscripten-64bit
 	git clone https://github.com/WebAssembly/binaryen.git emsdk/binaryen
 	cd emsdk/binaryen && git checkout $(PYODIDE_BINARYEN_VERSION)
 	cat patches/*.patch | patch -p1

pyodide_env.sh

@@ -1,9 +1,15 @@
 #!/bin/bash
 
-ROOT=`dirname ${BASH_SOURCE[0]}`
+# get the absolute path of the root folder
+ROOT=`cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 ; pwd -P`
 
 # emsdk_env.sh is fairly noisy, and suppress error message if the file doesn't
 # exist yet (i.e. before building emsdk)
 source "$ROOT/emsdk/emsdk/emsdk_env.sh" 2> /dev/null || true
-export PATH="$ROOT/node_modules/.bin/:$PATH:$ROOT/packages/.artifacts/bin/"
+export PATH="$ROOT/node_modules/.bin/:$ROOT/emsdk/emsdk/ccache/git-emscripten_64bit/bin:$PATH:$ROOT/packages/.artifacts/bin/"
 export EM_DIR=$(dirname $(which emcc.py || echo "."))
+
+# Following two variables are set by emsdk activated otherwise
+export _EMCC_CCACHE=1
+# mtime of this file is checked by ccache, we set it to avoid cache misses.
+export EM_CONFIG="$ROOT/emsdk/emsdk/.emscripten"