CI Fix ccache in package build (#3104)

.circleci/config.yml

@@ -42,9 +42,15 @@ jobs:
 
       - restore_cache:
           keys:
-            - -core-v20220303-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
-            - -core-v20220303-{{ checksum "cpython/Makefile" }}
-            - -core-v20220303
+            - -core-v20220915-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
+            - -core-v20220915-{{ checksum "cpython/Makefile" }}
+            - -core-v20220915
+
+      - run:
+          name: calculate build hash for ccache
+          command: |
+            pip install pathspec
+            ./tools/calculate_build_cache_key.py > build_hash.txt
 
       - run:
           name: build emsdk
@@ -89,7 +95,7 @@ jobs:
       - save_cache:
           paths:
             - /root/.ccache
-          key: -core-v20220303-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
+          key: -core-v20220915-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
 
       - run:
           name: Clean up workspace
@@ -135,8 +141,7 @@ jobs:
 
       - restore_cache:
           keys:
-            - -pkg2-v20220303-{{ checksum "Makefile.envs" }}
-            - -pkg2-v20220303
+            - -pkg3-v20220915-{{ checksum "build_hash.txt" }}-<< parameters.packages >>
 
       - run:
           name: install rust
@@ -162,11 +167,6 @@ jobs:
           name: check-size
           command: du dist/ -abh --max-depth 1  | sort -k 2
 
-      - save_cache:
-          paths:
-            - /root/.ccache
-          key: -pkg2-v20220303-{{ checksum "Makefile.envs" }}
-
       - run:
           name: Zip build directory
           command: |
@@ -188,6 +188,11 @@ jobs:
       - store_artifacts:
           path: /root/repo/build-logs.tar.gz
 
+      - save_cache:
+          paths:
+            - /root/.ccache
+          key: -pkg3-v20220915-{{ checksum "build_hash.txt" }}-<< parameters.packages >>
+
   build-pyodide-debug:
     <<: *defaults
     resource_class: large

pyodide-build/pyodide_build/buildpkg.py

@@ -160,9 +160,7 @@ def get_bash_runner() -> Iterator[BashRunnerWithSharedEnvironment]:
         env["PKG_CONFIG_PATH"] += f":{os.environ['PKG_CONFIG_PATH']}"
 
     with BashRunnerWithSharedEnvironment(env=env) as b:
-        b.run(
-            f"source {PYODIDE_ROOT}/emsdk/emsdk/emsdk_env.sh", stderr=subprocess.DEVNULL
-        )
+        b.run(f"source {PYODIDE_ROOT}/pyodide_env.sh", stderr=subprocess.DEVNULL)
         yield b
 
 

tools/calculate_build_cache_key.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+# This file is a helper script to calculate the hash for cache key
+# in the CI. It is not used in the build process.
+
+from hashlib import sha256
+from pathlib import Path
+
+import pathspec  # pip install pathspec
+
+
+def hash_file(filename):
+    with open(filename, "rb") as f:
+        return sha256(f.read()).hexdigest()
+
+
+def get_ignore_pattern(root: Path) -> list[str]:
+    ignorefile = root / ".gitignore"
+    if not ignorefile.exists():
+        raise RuntimeError("No .gitignore file found in root directory")
+
+    return ignorefile.read_text().splitlines()
+
+
+def main():
+    root: Path = Path(__file__).parent.parent
+    targets: list[Path] = [
+        root / "Makefile",
+        root / "Makefile.envs",
+        root / "packages" / "Makefile",
+        root / "pyodide-build" / "setup.cfg",
+        root / "pyodide-build" / "pyodide_build" / "__init__.py",
+        root / "pyodide-build" / "pyodide_build" / "pywasmcross.py",
+        root / "tools",
+    ]
+
+    ignore_pattern = get_ignore_pattern(root)
+    ignore_spec = pathspec.PathSpec.from_lines("gitwildmatch", ignore_pattern)
+
+    hash_candidates: list[Path] = []
+    for target in targets:
+        if target.is_file():
+            hash_candidates.append(target)
+        else:
+            hash_candidates.extend(list(target.glob("**/*")))
+
+    hash_candidates_filtered = sorted(
+        list(
+            filter(
+                lambda file: file.is_file() and not ignore_spec.match_file(str(file)),
+                hash_candidates,
+            )
+        )
+    )
+
+    hashes = []
+    for file in hash_candidates_filtered:
+        hashes.append(hash_file(file))
+
+    print("".join(hashes))
+
+
+if __name__ == "__main__":
+    main()