diff --git a/.circleci/config.yml b/.circleci/config.yml index 41aecc9f9..4667f8dc6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,6 +129,11 @@ jobs: - -pkg2-v20220303-{{ checksum "Makefile.envs" }} - -pkg2-v20220303 + - run: + name: install rust + command: | + make rust + - run: name: build packages no_output_timeout: 60m diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 996bb1a17..5441dad90 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -101,5 +101,5 @@ repos: rev: "v2.1.0" hooks: - id: codespell - args: ["-L", "te,slowy,aray,ba,nd,classs,feld"] + args: ["-L", "te,slowy,aray,ba,nd,classs,crate,feld"] exclude: ^benchmark/benchmarks/pystone_benchmarks/pystone\.py$ diff --git a/Makefile b/Makefile index 846ee1a2c..66ec8db4c 100644 --- a/Makefile +++ b/Makefile @@ -239,6 +239,12 @@ emsdk/emsdk/.complete: date +"[%F %T] done building emsdk." +rust: + wget https://sh.rustup.rs -O /rustup.sh + sh /rustup.sh -y + source $(HOME)/.cargo/env && rustup target add wasm32-unknown-emscripten --toolchain stable + + FORCE: diff --git a/Makefile.envs b/Makefile.envs index c6e38c332..00c5fe6e0 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -146,5 +146,18 @@ export MAIN_MODULE_CFLAGS= $(CFLAGS_BASE) \ export STDLIB_MODULE_CFLAGS= $(SIDE_MODULE_CFLAGS) -I Include/ -I . +# For RUST +export CARGO_HOME ?= $(HOME)/.cargo +export CARGO_BUILD_TARGET=wasm32-unknown-emscripten +export CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER=emcc +export PYO3_CONFIG_FILE=$(PYODIDE_ROOT)/tools/pyo3_config.ini + +# idealy we could automatically include all SIDE_MODULE_LDFLAGS here +export RUSTFLAGS= \ + -C relocation-model=pic \ + -C target-feature=+mutable-globals \ + -C link-arg=-sSIDE_MODULE=1 \ + -C link-arg=-sWASM_BIGINT + .output_vars: set diff --git a/cpython/adjust_sysconfig.py b/cpython/adjust_sysconfig.py index 511fffa32..6801b0faa 100644 --- a/cpython/adjust_sysconfig.py +++ b/cpython/adjust_sysconfig.py @@ -23,7 +23,7 @@ def adjust_sysconfig(config_vars: dict[str, str]): MAINCC="cc", LDSHARED="cc", LINKCC="cc", - BLDSHARED="cc", + BLDSHARED="emcc -sSIDE_MODULE=1", # setuptools-rust looks at this CXX="c++", LDCXXSHARED="c++", ) diff --git a/docs/development/building-from-sources.md b/docs/development/building-from-sources.md index d5211e342..cad60693c 100644 --- a/docs/development/building-from-sources.md +++ b/docs/development/building-from-sources.md @@ -151,9 +151,18 @@ meta-package. Other supported meta-packages are, "pytest". This option is non exhaustive and is mainly intended to make build faster while testing a diverse set of scientific packages. - "\*" builds all packages +- You can exclude a package by prefixing it with "!". micropip and distutils are always automatically included. +The cryptography package is a Rust extension. If you want to build it, you will +need Rust >= 1.41, you need the +[CARGO_HOME](https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-reads) +environment variable set appropriately, and you need the +`wasm32-unknown-emscripten` toolchain installed. If you run `make rust`, Pyodide +will install this stuff automatically. If you want to build every package except +for cryptography, you can set `PYODIDE_PACKAGES="*,!cryptography"`. + ## Environment variables The following environment variables additionally impact the build: diff --git a/docs/development/new-packages.md b/docs/development/new-packages.md index 598af4404..440411d20 100644 --- a/docs/development/new-packages.md +++ b/docs/development/new-packages.md @@ -328,3 +328,10 @@ imports are synchronous so it is impossible to load `.so` files lazily. meta-yaml.md ``` + +### Rust/PyO3 Packages + +We currently build Cryptography which is a Rust extension built with PyO3 and +setuptools-rust. It should be reasonably easy to build other Rust extensions. +Currently it is necessary to run `source $CARGO_HOME/env` in the build script, +but other than that there may be no other issues if you are lucky. diff --git a/emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch b/emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch index 171237788..3f3267744 100644 --- a/emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch +++ b/emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch @@ -1,7 +1,7 @@ From e83a295f8e1b8c48d4748d1811a4b22840f25e14 Mon Sep 17 00:00:00 2001 From: Hood Date: Thu, 24 Jun 2021 04:08:02 -0700 -Subject: [PATCH 1/6] Throw away errors in minify_wasm_js +Subject: [PATCH 1/8] Throw away errors in minify_wasm_js --- emcc.py | 13 ++++++++----- diff --git a/emsdk/patches/0002-Fix-dup.patch b/emsdk/patches/0002-Fix-dup.patch index 9c3287cb3..e1bc67ed5 100644 --- a/emsdk/patches/0002-Fix-dup.patch +++ b/emsdk/patches/0002-Fix-dup.patch @@ -1,7 +1,7 @@ From 40956dee436737d9dd40e0b57c6e2ebd26569920 Mon Sep 17 00:00:00 2001 From: Hood Date: Wed, 8 Sep 2021 17:49:15 -0700 -Subject: [PATCH 2/6] Fix dup +Subject: [PATCH 2/8] Fix dup This fixes two problems with the `dup` system calls: 1. `dup` expects that every file descriptor has a corresponding file (so pipes and (https://github.com/emscripten-core/emscripten/issues/14640) diff --git a/emsdk/patches/0003-Fix-side-module-exception-handling.patch b/emsdk/patches/0003-Fix-side-module-exception-handling.patch index 771011710..a57ae4374 100644 --- a/emsdk/patches/0003-Fix-side-module-exception-handling.patch +++ b/emsdk/patches/0003-Fix-side-module-exception-handling.patch @@ -1,7 +1,7 @@ From 73b89ee1b5a57c65824baf91b547be32b69decbd Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 15 Feb 2022 23:27:03 -0500 -Subject: [PATCH 3/6] Fix side module exception handling +Subject: [PATCH 3/8] Fix side module exception handling See https://github.com/emscripten-core/emscripten/pull/16309 diff --git a/emsdk/patches/0004-Fix-lookupPath-when-applied-to-a-symlink-loop.patch b/emsdk/patches/0004-Fix-lookupPath-when-applied-to-a-symlink-loop.patch index 39d6e0c0d..51c85a9d0 100644 --- a/emsdk/patches/0004-Fix-lookupPath-when-applied-to-a-symlink-loop.patch +++ b/emsdk/patches/0004-Fix-lookupPath-when-applied-to-a-symlink-loop.patch @@ -1,7 +1,7 @@ From 8de43377d0e72a4c5794f8494a06d81a9609090f Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 2 Mar 2022 13:44:14 -0800 -Subject: [PATCH 4/6] Fix lookupPath when applied to a symlink loop +Subject: [PATCH 4/8] Fix lookupPath when applied to a symlink loop The following code leads to an infinite loop in lookupPath: diff --git a/emsdk/patches/0005-Indicate-Emscripten-version-in-uname.patch b/emsdk/patches/0005-Indicate-Emscripten-version-in-uname.patch index 8ba939594..8b5e1b5db 100644 --- a/emsdk/patches/0005-Indicate-Emscripten-version-in-uname.patch +++ b/emsdk/patches/0005-Indicate-Emscripten-version-in-uname.patch @@ -1,7 +1,7 @@ From 8eed4062c1a0bd6aa7859938240f8761e6e72114 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 18 May 2022 21:49:17 -0700 -Subject: [PATCH 5/6] Indicate Emscripten version in uname +Subject: [PATCH 5/8] Indicate Emscripten version in uname This patch has been upstreamed: https://github.com/emscripten-core/emscripten/pull/17026 diff --git a/emsdk/patches/0006-Add-BigInt64Array-shim-for-Safari-14.patch b/emsdk/patches/0006-Add-BigInt64Array-shim-for-Safari-14.patch index b4013f144..482e0b6ef 100644 --- a/emsdk/patches/0006-Add-BigInt64Array-shim-for-Safari-14.patch +++ b/emsdk/patches/0006-Add-BigInt64Array-shim-for-Safari-14.patch @@ -1,7 +1,7 @@ From 4462c2871933db0adc4b46a24f77b942caff969c Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sun, 29 May 2022 18:17:29 -0700 -Subject: [PATCH 6/6] Add BigInt64Array shim for Safari 14 +Subject: [PATCH 6/8] Add BigInt64Array shim for Safari 14 --- src/parseTools.js | 2 +- diff --git a/emsdk/patches/0007-Disable-whole-archive-when-linking-rust.patch b/emsdk/patches/0007-Disable-whole-archive-when-linking-rust.patch new file mode 100644 index 000000000..b9ddfa717 --- /dev/null +++ b/emsdk/patches/0007-Disable-whole-archive-when-linking-rust.patch @@ -0,0 +1,33 @@ +From 9fb8391c7e6b9f36f45a1448e49597f899cce29d Mon Sep 17 00:00:00 2001 +From: Hood Chatham +Date: Thu, 6 Jan 2022 09:40:39 -0800 +Subject: [PATCH 7/8] Disable whole-archive when linking rust + +Rust .rlib archives contain an extra metadata file called lib.rmeta. +Emscripten sets `--whole-archive` by default if LINKABLE is set. +But with `--whole-archive` the linker crashes with an error due to +the extra files. This disables the problematic `--whole-archive` +setting when we are linking rust. + +See emscripten issue: +https://github.com/emscripten-core/emscripten/issues/17109 +--- + tools/building.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tools/building.py b/tools/building.py +index 512561c69..d180993a1 100644 +--- a/tools/building.py ++++ b/tools/building.py +@@ -443,7 +443,7 @@ def link_lld(args, target, external_symbols=None): + + # Emscripten currently expects linkable output (SIDE_MODULE/MAIN_MODULE) to + # include all archive contents. +- if settings.LINKABLE: ++ if settings.LINKABLE and not any(arg.endswith(".rlib") for arg in args): + args.insert(0, '--whole-archive') + args.append('--no-whole-archive') + +-- +2.25.1 + diff --git a/emsdk/patches/0008-Add-signature-to-emscripten_get_now.patch b/emsdk/patches/0008-Add-signature-to-emscripten_get_now.patch new file mode 100644 index 000000000..ac4763cce --- /dev/null +++ b/emsdk/patches/0008-Add-signature-to-emscripten_get_now.patch @@ -0,0 +1,26 @@ +From b6839bba05c1a5eab390f47309a7bef1b7294eca Mon Sep 17 00:00:00 2001 +From: Hood Chatham +Date: Wed, 1 Jun 2022 10:54:08 -0700 +Subject: [PATCH 8/8] Add signature to emscripten_get_now + +Upstream PR: +https://github.com/emscripten-core/emscripten/pull/17123/files +--- + src/library.js | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/library.js b/src/library.js +index ef8a618bb..5b5a9d838 100644 +--- a/src/library.js ++++ b/src/library.js +@@ -2657,6 +2657,7 @@ LibraryManager.library = { + }, + + emscripten_get_now__import: true, ++ emscripten_get_now__sig: 'd', + emscripten_get_now: ';' + + #if ENVIRONMENT_MAY_BE_NODE + "if (ENVIRONMENT_IS_NODE) {\n" + +-- +2.25.1 + diff --git a/packages/cryptography/meta.yaml b/packages/cryptography/meta.yaml index cc71441fa..511f5a086 100644 --- a/packages/cryptography/meta.yaml +++ b/packages/cryptography/meta.yaml @@ -1,20 +1,29 @@ package: name: cryptography - version: 3.4.8 + version: 36.0.2 source: - url: https://files.pythonhosted.org/packages/cc/98/8a258ab4787e6f835d350639792527d2eb7946ff9fc0caca9c3f4cf5dcfe/cryptography-3.4.8.tar.gz - sha256: 94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c + url: https://files.pythonhosted.org/packages/10/a7/51953e73828deef2b58ba1604de9167843ee9cd4185d8aaffcb45dd1932d/cryptography-36.0.2.tar.gz + sha256: 70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9 + patches: + # TODO: remove this when chrono makes a release + - patches/0001-Use-patched-chrono.patch + - patches/0002-Add-instant-patch.patch build: + script: | + export OPENSSL_INCLUDE_PATH=$(pkg-config --cflags-only-I --dont-define-prefix openssl) + export OPENSSL_LIBRARY_PATH=$(pkg-config --libs-only-L --dont-define-prefix openssl) + source $CARGO_HOME/env + + # TODO: remove this when chrono makes a release + git clone --depth 1 https://github.com/hoodmane/chrono.git --branch pyodide + # TODO: remove this when instant makes a release + git clone --depth 1 https://github.com/hoodmane/instant.git --branch emscripten-no-leading-underscore cflags: | -Wno-implicit-function-declaration $(OPENSSL_INCLUDE_PATH) ldflags: | $(OPENSSL_LIBRARY_PATH) - script: | - export CRYPTOGRAPHY_DONT_BUILD_RUST=1 - export OPENSSL_INCLUDE_PATH=$(pkg-config --cflags-only-I --dont-define-prefix openssl) - export OPENSSL_LIBRARY_PATH=$(pkg-config --libs-only-L --dont-define-prefix openssl) requirements: run: - openssl diff --git a/packages/cryptography/patches/0001-Use-patched-chrono.patch b/packages/cryptography/patches/0001-Use-patched-chrono.patch new file mode 100644 index 000000000..72117638f --- /dev/null +++ b/packages/cryptography/patches/0001-Use-patched-chrono.patch @@ -0,0 +1,26 @@ +From 53ee3cee6aae5333d61ac687677cf2ca39cb831a Mon Sep 17 00:00:00 2001 +From: Hood Chatham +Date: Fri, 8 Apr 2022 18:17:51 -0700 +Subject: [PATCH 1/2] Use patched chrono + +--- + src/rust/Cargo.toml | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml +index 617167d0..35bf38ae 100644 +--- a/src/rust/Cargo.toml ++++ b/src/rust/Cargo.toml +@@ -5,6 +5,9 @@ authors = ["The cryptography developers "] + edition = "2018" + publish = false + ++[patch.crates-io] ++chrono = { path = "../../chrono" } ++ + [dependencies] + lazy_static = "1" + pyo3 = { version = "0.15.1" } +-- +2.25.1 + diff --git a/packages/cryptography/patches/0002-Add-instant-patch.patch b/packages/cryptography/patches/0002-Add-instant-patch.patch new file mode 100644 index 000000000..5278e9760 --- /dev/null +++ b/packages/cryptography/patches/0002-Add-instant-patch.patch @@ -0,0 +1,24 @@ +From 196ace04edb63a93000418f91d56a3cec11b8345 Mon Sep 17 00:00:00 2001 +From: Hood Chatham +Date: Wed, 1 Jun 2022 10:11:44 -0700 +Subject: [PATCH 2/2] Add instant patch + +--- + src/rust/Cargo.toml | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml +index 35bf38ae..23a64977 100644 +--- a/src/rust/Cargo.toml ++++ b/src/rust/Cargo.toml +@@ -7,6 +7,7 @@ publish = false + + [patch.crates-io] + chrono = { path = "../../chrono" } ++instant = { path = "../../instant" } + + [dependencies] + lazy_static = "1" +-- +2.25.1 + diff --git a/packages/cryptography/test_cryptography.py b/packages/cryptography/test_cryptography.py index adb9aad15..2caa35272 100644 --- a/packages/cryptography/test_cryptography.py +++ b/packages/cryptography/test_cryptography.py @@ -17,149 +17,6 @@ def test_cryptography(selenium): assert f1.decrypt(f.encrypt(b"abc")) == b"abc" -@run_in_pyodide(packages=["cryptography", "pytest"]) -def test_der_reader_basic(selenium): - import pytest - from cryptography.hazmat._der import DERReader - - reader = DERReader(b"123456789") - assert reader.read_byte() == ord(b"1") - assert reader.read_bytes(1).tobytes() == b"2" - assert reader.read_bytes(4).tobytes() == b"3456" - - with pytest.raises(ValueError): - reader.read_bytes(4) - - assert reader.read_bytes(3).tobytes() == b"789" - - # The input is now empty. - with pytest.raises(ValueError): - reader.read_bytes(1) - with pytest.raises(ValueError): - reader.read_byte() - - -@run_in_pyodide(packages=["cryptography", "pytest"]) -def test_der(selenium): - import pytest - from cryptography.hazmat._der import ( - INTEGER, - NULL, - OCTET_STRING, - SEQUENCE, - DERReader, - encode_der, - encode_der_integer, - ) - - # This input is the following structure, using - # https://github.com/google/der-ascii - # - # SEQUENCE { - # SEQUENCE { - # NULL {} - # INTEGER { 42 } - # OCTET_STRING { "hello" } - # } - # } - der = b"\x30\x0e\x30\x0c\x05\x00\x02\x01\x2a\x04\x05\x68\x65\x6c\x6c\x6f" - reader = DERReader(der) - with pytest.raises(ValueError): - reader.check_empty() - - with pytest.raises(ValueError): - with reader: - pass - - with pytest.raises(ZeroDivisionError): - with DERReader(der): - raise ZeroDivisionError - - # Parse the outer element. - outer = reader.read_element(SEQUENCE) - reader.check_empty() - assert outer.data.tobytes() == der[2:] - - # Parse the outer element with read_any_element. - reader = DERReader(der) - tag, outer2 = reader.read_any_element() - reader.check_empty() - assert tag == SEQUENCE - assert outer2.data.tobytes() == der[2:] - - # Parse the outer element with read_single_element. - outer3 = DERReader(der).read_single_element(SEQUENCE) - assert outer3.data.tobytes() == der[2:] - - # read_single_element rejects trailing data. - with pytest.raises(ValueError): - DERReader(der + der).read_single_element(SEQUENCE) - - # Continue parsing the structure. - inner = outer.read_element(SEQUENCE) - outer.check_empty() - - # Parsing a missing optional element should work. - assert inner.read_optional_element(INTEGER) is None - - null = inner.read_element(NULL) - null.check_empty() - - # Parsing a present optional element should work. - integer = inner.read_optional_element(INTEGER) - assert integer.as_integer() == 42 - - octet_string = inner.read_element(OCTET_STRING) - assert octet_string.data.tobytes() == b"hello" - - # Parsing a missing optional element should work when the input is empty. - inner.check_empty() - assert inner.read_optional_element(INTEGER) is None - - # Re-encode the same structure. - der2 = encode_der( - SEQUENCE, - encode_der( - SEQUENCE, - encode_der(NULL), - encode_der(INTEGER, encode_der_integer(42)), - encode_der(OCTET_STRING, b"hello"), - ), - ) - assert der2 == der - - -@run_in_pyodide(packages=["cryptography"]) -def test_der_lengths(selenium): - - from cryptography.hazmat._der import OCTET_STRING, DERReader, encode_der - - for [length, header] in [ - # Single-byte lengths. - (0, b"\x04\x00"), - (1, b"\x04\x01"), - (2, b"\x04\x02"), - (127, b"\x04\x7f"), - # Long-form lengths. - (128, b"\x04\x81\x80"), - (129, b"\x04\x81\x81"), - (255, b"\x04\x81\xff"), - (0x100, b"\x04\x82\x01\x00"), - (0x101, b"\x04\x82\x01\x01"), - (0xFFFF, b"\x04\x82\xff\xff"), - (0x10000, b"\x04\x83\x01\x00\x00"), - ]: - body = length * b"a" - der = header + body - - reader = DERReader(der) - element = reader.read_element(OCTET_STRING) - reader.check_empty() - assert element.data.tobytes() == body - - assert encode_der(OCTET_STRING, body) == der - - @settings(suppress_health_check=[HealthCheck.too_slow], deadline=None) @given(data=binary()) def test_fernet(selenium_module_scope, data): diff --git a/pyodide-build/pyodide_build/buildpkg.py b/pyodide-build/pyodide_build/buildpkg.py index f87de8f03..20f496db0 100755 --- a/pyodide-build/pyodide_build/buildpkg.py +++ b/pyodide-build/pyodide_build/buildpkg.py @@ -135,6 +135,7 @@ def get_bash_runner(): "PYTHONINCLUDE", "NUMPY_LIB", "PYODIDE_PACKAGE_ABI", + "HOME", "HOSTINSTALLDIR", "TARGETINSTALLDIR", "SYSCONFIG_NAME", @@ -149,6 +150,11 @@ def get_bash_runner(): "UNISOLATED_PACKAGES", "WASM_LIBRARY_DIR", "WASM_PKG_CONFIG_PATH", + "CARGO_BUILD_TARGET", + "CARGO_HOME", + "CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER", + "RUSTFLAGS", + "PYO3_CONFIG_FILE", ] } | {"PYODIDE": "1"} if "PYODIDE_JOBS" in os.environ: diff --git a/pyodide-build/pyodide_build/pywasmcross.py b/pyodide-build/pyodide_build/pywasmcross.py index cd8c5aa4a..e1e94a005 100755 --- a/pyodide-build/pyodide_build/pywasmcross.py +++ b/pyodide-build/pyodide_build/pywasmcross.py @@ -10,6 +10,7 @@ cross-compiling and then pass the command long to emscripten. """ import json import os +import shutil import sys IS_MAIN = __name__ == "__main__" @@ -30,7 +31,7 @@ from typing import Any, MutableMapping, NoReturn from pyodide_build import common from pyodide_build._f2c_fixes import fix_f2c_input, fix_f2c_output, scipy_fixes -symlinks = {"cc", "c++", "ld", "ar", "gcc", "gfortran"} +symlinks = {"cc", "c++", "ld", "ar", "gcc", "gfortran", "cargo"} def symlink_dir(): @@ -537,6 +538,13 @@ def handle_command( if returncode != 0: sys.exit(returncode) + # Rust gives output files a `.wasm` suffix, but we need them to have a `.so` + # suffix. + if line[0:2] == ["cargo", "rustc"]: + p = Path(args.builddir) + for x in p.glob("**/*.wasm"): + shutil.move(x, x.with_suffix(".so")) + sys.exit(returncode) diff --git a/tools/pyo3_config.ini b/tools/pyo3_config.ini new file mode 100644 index 000000000..a91b4f9c6 --- /dev/null +++ b/tools/pyo3_config.ini @@ -0,0 +1,7 @@ +implementation=CPython +version=3.10 +shared=true +abi3=false +lib_name=python3.10 +pointer_width=32 +suppress_build_script_link_lines=false