Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cryptography v36.0.2 w/ Rust (#2378)

This commit is contained in:
Hood Chatham 2022-06-09 10:57:34 -07:00 committed by GitHub
parent b880ec2395
commit dbbbbb67b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 195 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.circleci/config.yml
View File

@ -129,6 +129,11 @@ jobs:
- -pkg2-v20220303-{{ checksum "Makefile.envs" }}
- -pkg2-v20220303
- run:
name: install rust
command: |
make rust
- run:
name: build packages
no_output_timeout: 60m

2
.pre-commit-config.yaml
View File

@ -101,5 +101,5 @@ repos:
rev: "v2.1.0"
hooks:
- id: codespell
args: ["-L", "te,slowy,aray,ba,nd,classs,feld"]
args: ["-L", "te,slowy,aray,ba,nd,classs,crate,feld"]
exclude: ^benchmark/benchmarks/pystone_benchmarks/pystone\.py$

6
Makefile
View File

@ -239,6 +239,12 @@ emsdk/emsdk/.complete:
date +"[%F %T] done building emsdk."
rust:
wget https://sh.rustup.rs -O /rustup.sh
sh /rustup.sh -y
source $(HOME)/.cargo/env && rustup target add wasm32-unknown-emscripten --toolchain stable
FORCE:

13
Makefile.envs
View File

@ -146,5 +146,18 @@ export MAIN_MODULE_CFLAGS= $(CFLAGS_BASE) \
export STDLIB_MODULE_CFLAGS= $(SIDE_MODULE_CFLAGS) -I Include/ -I .
# For RUST
export CARGO_HOME ?= $(HOME)/.cargo
export CARGO_BUILD_TARGET=wasm32-unknown-emscripten
export CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER=emcc
export PYO3_CONFIG_FILE=$(PYODIDE_ROOT)/tools/pyo3_config.ini
# idealy we could automatically include all SIDE_MODULE_LDFLAGS here
export RUSTFLAGS= \
-C relocation-model=pic \
-C target-feature=+mutable-globals \
-C link-arg=-sSIDE_MODULE=1 \
-C link-arg=-sWASM_BIGINT
.output_vars:
set

2
cpython/adjust_sysconfig.py
View File

@ -23,7 +23,7 @@ def adjust_sysconfig(config_vars: dict[str, str]):
MAINCC="cc",
LDSHARED="cc",
LINKCC="cc",
BLDSHARED="cc",
BLDSHARED="emcc -sSIDE_MODULE=1", # setuptools-rust looks at this
CXX="c++",
LDCXXSHARED="c++",
)

9
docs/development/building-from-sources.md
View File

@ -151,9 +151,18 @@ meta-package. Other supported meta-packages are,
"pytest". This option is non exhaustive and is mainly intended to make build
faster while testing a diverse set of scientific packages.
- "\*" builds all packages
- You can exclude a package by prefixing it with "!".
micropip and distutils are always automatically included.
The cryptography package is a Rust extension. If you want to build it, you will
need Rust >= 1.41, you need the
[CARGO_HOME](https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-reads)
environment variable set appropriately, and you need the
`wasm32-unknown-emscripten` toolchain installed. If you run `make rust`, Pyodide
will install this stuff automatically. If you want to build every package except
for cryptography, you can set `PYODIDE_PACKAGES="*,!cryptography"`.
## Environment variables
The following environment variables additionally impact the build:

7
docs/development/new-packages.md
View File

@ -328,3 +328,10 @@ imports are synchronous so it is impossible to load `.so` files lazily.
meta-yaml.md
```
### Rust/PyO3 Packages
We currently build Cryptography which is a Rust extension built with PyO3 and
setuptools-rust. It should be reasonably easy to build other Rust extensions.
Currently it is necessary to run `source $CARGO_HOME/env` in the build script,
but other than that there may be no other issues if you are lucky.

2
emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch
View File

@ -1,7 +1,7 @@
From e83a295f8e1b8c48d4748d1811a4b22840f25e14 Mon Sep 17 00:00:00 2001
From: Hood <hood@mit.edu>
Date: Thu, 24 Jun 2021 04:08:02 -0700
Subject: [PATCH 1/6] Throw away errors in minify_wasm_js
Subject: [PATCH 1/8] Throw away errors in minify_wasm_js
---
emcc.py | 13 ++++++++-----

2
emsdk/patches/0002-Fix-dup.patch
View File

@ -1,7 +1,7 @@
From 40956dee436737d9dd40e0b57c6e2ebd26569920 Mon Sep 17 00:00:00 2001
From: Hood <hood@mit.edu>
Date: Wed, 8 Sep 2021 17:49:15 -0700
Subject: [PATCH 2/6] Fix dup
Subject: [PATCH 2/8] Fix dup
This fixes two problems with the `dup` system calls:
1. `dup` expects that every file descriptor has a corresponding file (so pipes and (https://github.com/emscripten-core/emscripten/issues/14640)

2
emsdk/patches/0003-Fix-side-module-exception-handling.patch
View File

@ -1,7 +1,7 @@
From 73b89ee1b5a57c65824baf91b547be32b69decbd Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Tue, 15 Feb 2022 23:27:03 -0500
Subject: [PATCH 3/6] Fix side module exception handling
Subject: [PATCH 3/8] Fix side module exception handling
See https://github.com/emscripten-core/emscripten/pull/16309

2
emsdk/patches/0004-Fix-lookupPath-when-applied-to-a-symlink-loop.patch
View File

@ -1,7 +1,7 @@
From 8de43377d0e72a4c5794f8494a06d81a9609090f Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Wed, 2 Mar 2022 13:44:14 -0800
Subject: [PATCH 4/6] Fix lookupPath when applied to a symlink loop
Subject: [PATCH 4/8] Fix lookupPath when applied to a symlink loop
The following code leads to an infinite loop in lookupPath:

2
emsdk/patches/0005-Indicate-Emscripten-version-in-uname.patch
View File

@ -1,7 +1,7 @@
From 8eed4062c1a0bd6aa7859938240f8761e6e72114 Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Wed, 18 May 2022 21:49:17 -0700
Subject: [PATCH 5/6] Indicate Emscripten version in uname
Subject: [PATCH 5/8] Indicate Emscripten version in uname
This patch has been upstreamed:
https://github.com/emscripten-core/emscripten/pull/17026

2
emsdk/patches/0006-Add-BigInt64Array-shim-for-Safari-14.patch
View File

@ -1,7 +1,7 @@
From 4462c2871933db0adc4b46a24f77b942caff969c Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Sun, 29 May 2022 18:17:29 -0700
Subject: [PATCH 6/6] Add BigInt64Array shim for Safari 14
Subject: [PATCH 6/8] Add BigInt64Array shim for Safari 14
---
src/parseTools.js | 2 +-

33
emsdk/patches/0007-Disable-whole-archive-when-linking-rust.patch Normal file
View File

@ -0,0 +1,33 @@
From 9fb8391c7e6b9f36f45a1448e49597f899cce29d Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Thu, 6 Jan 2022 09:40:39 -0800
Subject: [PATCH 7/8] Disable whole-archive when linking rust
Rust .rlib archives contain an extra metadata file called lib.rmeta.
Emscripten sets `--whole-archive` by default if LINKABLE is set.
But with `--whole-archive` the linker crashes with an error due to
the extra files. This disables the problematic `--whole-archive`
setting when we are linking rust.
See emscripten issue:
https://github.com/emscripten-core/emscripten/issues/17109
---
tools/building.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/building.py b/tools/building.py
index 512561c69..d180993a1 100644
--- a/tools/building.py
+++ b/tools/building.py
@@ -443,7 +443,7 @@ def link_lld(args, target, external_symbols=None):
# Emscripten currently expects linkable output (SIDE_MODULE/MAIN_MODULE) to
# include all archive contents.
- if settings.LINKABLE:
+ if settings.LINKABLE and not any(arg.endswith(".rlib") for arg in args):
args.insert(0, '--whole-archive')
args.append('--no-whole-archive')
--
2.25.1

26
emsdk/patches/0008-Add-signature-to-emscripten_get_now.patch Normal file
View File

@ -0,0 +1,26 @@
From b6839bba05c1a5eab390f47309a7bef1b7294eca Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Wed, 1 Jun 2022 10:54:08 -0700
Subject: [PATCH 8/8] Add signature to emscripten_get_now
Upstream PR:
https://github.com/emscripten-core/emscripten/pull/17123/files
---
src/library.js | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/library.js b/src/library.js
index ef8a618bb..5b5a9d838 100644
--- a/src/library.js
+++ b/src/library.js
@@ -2657,6 +2657,7 @@ LibraryManager.library = {
},
emscripten_get_now__import: true,
+ emscripten_get_now__sig: 'd',
emscripten_get_now: ';' +
#if ENVIRONMENT_MAY_BE_NODE
"if (ENVIRONMENT_IS_NODE) {\n" +
--
2.25.1

23
packages/cryptography/meta.yaml
View File

@ -1,20 +1,29 @@
package:
name: cryptography
version: 3.4.8
version: 36.0.2
source:
url: https://files.pythonhosted.org/packages/cc/98/8a258ab4787e6f835d350639792527d2eb7946ff9fc0caca9c3f4cf5dcfe/cryptography-3.4.8.tar.gz
sha256: 94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c
url: https://files.pythonhosted.org/packages/10/a7/51953e73828deef2b58ba1604de9167843ee9cd4185d8aaffcb45dd1932d/cryptography-36.0.2.tar.gz
sha256: 70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9
patches:
# TODO: remove this when chrono makes a release
- patches/0001-Use-patched-chrono.patch
- patches/0002-Add-instant-patch.patch
build:
script: |
export OPENSSL_INCLUDE_PATH=$(pkg-config --cflags-only-I --dont-define-prefix openssl)
export OPENSSL_LIBRARY_PATH=$(pkg-config --libs-only-L --dont-define-prefix openssl)
source $CARGO_HOME/env
# TODO: remove this when chrono makes a release
git clone --depth 1 https://github.com/hoodmane/chrono.git --branch pyodide
# TODO: remove this when instant makes a release
git clone --depth 1 https://github.com/hoodmane/instant.git --branch emscripten-no-leading-underscore
cflags: |
-Wno-implicit-function-declaration
$(OPENSSL_INCLUDE_PATH)
ldflags: |
$(OPENSSL_LIBRARY_PATH)
script: |
export CRYPTOGRAPHY_DONT_BUILD_RUST=1
export OPENSSL_INCLUDE_PATH=$(pkg-config --cflags-only-I --dont-define-prefix openssl)
export OPENSSL_LIBRARY_PATH=$(pkg-config --libs-only-L --dont-define-prefix openssl)
requirements:
run:
- openssl

26
packages/cryptography/patches/0001-Use-patched-chrono.patch Normal file
View File

@ -0,0 +1,26 @@
From 53ee3cee6aae5333d61ac687677cf2ca39cb831a Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Fri, 8 Apr 2022 18:17:51 -0700
Subject: [PATCH 1/2] Use patched chrono
---
src/rust/Cargo.toml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml
index 617167d0..35bf38ae 100644
--- a/src/rust/Cargo.toml
+++ b/src/rust/Cargo.toml
@@ -5,6 +5,9 @@ authors = ["The cryptography developers <cryptography-dev@python.org>"]
edition = "2018"
publish = false
+[patch.crates-io]
+chrono = { path = "../../chrono" }
+
[dependencies]
lazy_static = "1"
pyo3 = { version = "0.15.1" }
--
2.25.1

24
packages/cryptography/patches/0002-Add-instant-patch.patch Normal file
View File

@ -0,0 +1,24 @@
From 196ace04edb63a93000418f91d56a3cec11b8345 Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Wed, 1 Jun 2022 10:11:44 -0700
Subject: [PATCH 2/2] Add instant patch
---
src/rust/Cargo.toml | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml
index 35bf38ae..23a64977 100644
--- a/src/rust/Cargo.toml
+++ b/src/rust/Cargo.toml
@@ -7,6 +7,7 @@ publish = false
[patch.crates-io]
chrono = { path = "../../chrono" }
+instant = { path = "../../instant" }
[dependencies]
lazy_static = "1"
--
2.25.1

143
packages/cryptography/test_cryptography.py
View File

@ -17,149 +17,6 @@ def test_cryptography(selenium):
assert f1.decrypt(f.encrypt(b"abc")) == b"abc"
@run_in_pyodide(packages=["cryptography", "pytest"])
def test_der_reader_basic(selenium):
import pytest
from cryptography.hazmat._der import DERReader
reader = DERReader(b"123456789")
assert reader.read_byte() == ord(b"1")
assert reader.read_bytes(1).tobytes() == b"2"
assert reader.read_bytes(4).tobytes() == b"3456"
with pytest.raises(ValueError):
reader.read_bytes(4)
assert reader.read_bytes(3).tobytes() == b"789"
# The input is now empty.
with pytest.raises(ValueError):
reader.read_bytes(1)
with pytest.raises(ValueError):
reader.read_byte()
@run_in_pyodide(packages=["cryptography", "pytest"])
def test_der(selenium):
import pytest
from cryptography.hazmat._der import (
INTEGER,
NULL,
OCTET_STRING,
SEQUENCE,
DERReader,
encode_der,
encode_der_integer,
)
# This input is the following structure, using
# https://github.com/google/der-ascii
#
# SEQUENCE {
# SEQUENCE {
# NULL {}
# INTEGER { 42 }
# OCTET_STRING { "hello" }
# }
# }
der = b"\x30\x0e\x30\x0c\x05\x00\x02\x01\x2a\x04\x05\x68\x65\x6c\x6c\x6f"
reader = DERReader(der)
with pytest.raises(ValueError):
reader.check_empty()
with pytest.raises(ValueError):
with reader:
pass
with pytest.raises(ZeroDivisionError):
with DERReader(der):
raise ZeroDivisionError
# Parse the outer element.
outer = reader.read_element(SEQUENCE)
reader.check_empty()
assert outer.data.tobytes() == der[2:]
# Parse the outer element with read_any_element.
reader = DERReader(der)
tag, outer2 = reader.read_any_element()
reader.check_empty()
assert tag == SEQUENCE
assert outer2.data.tobytes() == der[2:]
# Parse the outer element with read_single_element.
outer3 = DERReader(der).read_single_element(SEQUENCE)
assert outer3.data.tobytes() == der[2:]
# read_single_element rejects trailing data.
with pytest.raises(ValueError):
DERReader(der + der).read_single_element(SEQUENCE)
# Continue parsing the structure.
inner = outer.read_element(SEQUENCE)
outer.check_empty()
# Parsing a missing optional element should work.
assert inner.read_optional_element(INTEGER) is None
null = inner.read_element(NULL)
null.check_empty()
# Parsing a present optional element should work.
integer = inner.read_optional_element(INTEGER)
assert integer.as_integer() == 42
octet_string = inner.read_element(OCTET_STRING)
assert octet_string.data.tobytes() == b"hello"
# Parsing a missing optional element should work when the input is empty.
inner.check_empty()
assert inner.read_optional_element(INTEGER) is None
# Re-encode the same structure.
der2 = encode_der(
SEQUENCE,
encode_der(
SEQUENCE,
encode_der(NULL),
encode_der(INTEGER, encode_der_integer(42)),
encode_der(OCTET_STRING, b"hello"),
),
)
assert der2 == der
@run_in_pyodide(packages=["cryptography"])
def test_der_lengths(selenium):
from cryptography.hazmat._der import OCTET_STRING, DERReader, encode_der
for [length, header] in [
# Single-byte lengths.
(0, b"\x04\x00"),
(1, b"\x04\x01"),
(2, b"\x04\x02"),
(127, b"\x04\x7f"),
# Long-form lengths.
(128, b"\x04\x81\x80"),
(129, b"\x04\x81\x81"),
(255, b"\x04\x81\xff"),
(0x100, b"\x04\x82\x01\x00"),
(0x101, b"\x04\x82\x01\x01"),
(0xFFFF, b"\x04\x82\xff\xff"),
(0x10000, b"\x04\x83\x01\x00\x00"),
]:
body = length * b"a"
der = header + body
reader = DERReader(der)
element = reader.read_element(OCTET_STRING)
reader.check_empty()
assert element.data.tobytes() == body
assert encode_der(OCTET_STRING, body) == der
@settings(suppress_health_check=[HealthCheck.too_slow], deadline=None)
@given(data=binary())
def test_fernet(selenium_module_scope, data):

6
pyodide-build/pyodide_build/buildpkg.py
View File

@ -135,6 +135,7 @@ def get_bash_runner():
"PYTHONINCLUDE",
"NUMPY_LIB",
"PYODIDE_PACKAGE_ABI",
"HOME",
"HOSTINSTALLDIR",
"TARGETINSTALLDIR",
"SYSCONFIG_NAME",
@ -149,6 +150,11 @@ def get_bash_runner():
"UNISOLATED_PACKAGES",
"WASM_LIBRARY_DIR",
"WASM_PKG_CONFIG_PATH",
"CARGO_BUILD_TARGET",
"CARGO_HOME",
"CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER",
"RUSTFLAGS",
"PYO3_CONFIG_FILE",
]
} | {"PYODIDE": "1"}
if "PYODIDE_JOBS" in os.environ:

10
pyodide-build/pyodide_build/pywasmcross.py
View File

@ -10,6 +10,7 @@ cross-compiling and then pass the command long to emscripten.
"""
import json
import os
import shutil
import sys
IS_MAIN = __name__ == "__main__"
@ -30,7 +31,7 @@ from typing import Any, MutableMapping, NoReturn
from pyodide_build import common
from pyodide_build._f2c_fixes import fix_f2c_input, fix_f2c_output, scipy_fixes
symlinks = {"cc", "c++", "ld", "ar", "gcc", "gfortran"}
symlinks = {"cc", "c++", "ld", "ar", "gcc", "gfortran", "cargo"}
def symlink_dir():
@ -537,6 +538,13 @@ def handle_command(
if returncode != 0:
sys.exit(returncode)
# Rust gives output files a `.wasm` suffix, but we need them to have a `.so`
# suffix.
if line[0:2] == ["cargo", "rustc"]:
p = Path(args.builddir)
for x in p.glob("**/*.wasm"):
shutil.move(x, x.with_suffix(".so"))
sys.exit(returncode)

7
tools/pyo3_config.ini Normal file
View File

@ -0,0 +1,7 @@
implementation=CPython
version=3.10
shared=true
abi3=false
lib_name=python3.10
pointer_width=32
suppress_build_script_link_lines=false