From d385b7c9f11202e2f9154767e491f2c321a0ca20 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 4 May 2018 13:13:56 -0400 Subject: [PATCH] Use Proxy to make calling Python objects from JS more natural --- src/js2python.cpp | 9 ++- src/main.cpp | 17 +++-- src/pyodide.js | 26 ++++++-- src/pyproxy.cpp | 158 ++++++++++++++++++++++++-------------------- src/pyproxy.hpp | 49 +++++++++++--- src/python.html | 12 ++-- src/python2js.cpp | 100 ++++++++++++++++------------ src/test.html | 1 + test/test_python.py | 22 ++++++ 9 files changed, 249 insertions(+), 145 deletions(-) diff --git a/src/js2python.cpp b/src/js2python.cpp index 45b0c51e5..56ab090d6 100644 --- a/src/js2python.cpp +++ b/src/js2python.cpp @@ -27,8 +27,13 @@ PyObject *jsToPython(val x) { Py_INCREF(Py_False); return Py_False; } else if (!x["$$"].isUndefined() && - x["$$"]["ptrType"]["name"].equals(val("Py*"))) { - Py py_x = x.as(); + x["$$"]["ptrType"]["name"].equals(val("PyObject*"))) { + PyObject *py_x = x.as(emscripten::allow_raw_pointers()); + Py_INCREF(py_x); + return py_x; + } else if (!x["$$"].isUndefined() && + x["$$"]["ptrType"]["name"].equals(val("PyCallable*"))) { + PyCallable py_x = x.as(); PyObject *pypy_x = py_x.x; Py_INCREF(pypy_x); return pypy_x; diff --git a/src/main.cpp b/src/main.cpp index 3d7deca3d..44c0af294 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -38,14 +38,17 @@ EMSCRIPTEN_BINDINGS(python) { emscripten::function("runPython", &runPython); emscripten::function("pyimport", &pyimport); emscripten::function("repr", &repr); + emscripten::class_("PyObject"); emscripten::class_("Py") - .function("call", &Py::call) - .function("getattr", &Py::getattr) - .function("setattr", &Py::setattr) - .function("hasattr", &Py::hasattr) - .function("getitem", &Py::getitem) - .function("setitem", &Py::setitem) - .function("hasitem", &Py::hasitem); + .class_function("isExtensible", &Py::isExtensible) + .class_function("has", &Py::has) + .class_function("get", &Py::get) + .class_function("set", &Py::set) + .class_function("deleteProperty", &Py::deleteProperty) + .class_function("ownKeys", &Py::ownKeys) + .class_function("enumerate", &Py::enumerate); + emscripten::class_("PyCallable") + .function("call", &PyCallable::call); } extern "C" { diff --git a/src/pyodide.js b/src/pyodide.js index c71d6e28f..912ac3638 100644 --- a/src/pyodide.js +++ b/src/pyodide.js @@ -67,6 +67,19 @@ var languagePluginLoader = new Promise((resolve, reject) => { return promise; }; + let makeCallableProxy = (obj) => { + var clone = obj.clone(); + return (args, kwargs) => { + if (args === undefined) { + args = []; + } + if (kwargs === undefined) { + kwargs = {}; + } + return clone.call(args, kwargs); + }; + }; + let wasmURL = `${baseURL}pyodide.asm.wasm`; let Module = {}; @@ -86,11 +99,10 @@ var languagePluginLoader = new Promise((resolve, reject) => { script.src = `${baseURL}pyodide.asm.js`; script.onload = () => { window.pyodide = pyodide(Module); - if (window.iodide !== undefined) { - window.pyodide.loadPackage = loadPackage; - } + window.pyodide.loadPackage = loadPackage; + window.pyodide.makeCallableProxy = makeCallableProxy; }; - document.body.appendChild(script); + document.head.appendChild(script); if (window.iodide !== undefined) { // Load the custom CSS for Pyodide @@ -105,15 +117,15 @@ var languagePluginLoader = new Promise((resolve, reject) => { shouldHandle: (val) => { return (typeof val === 'object' && val['$$'] !== undefined && - val['$$']['ptrType']['name'] === 'Py*'); + val['$$']['ptrType']['name'] === 'PyObject*'); }, render: (val) => { let div = document.createElement('div'); div.className = 'rendered_html'; - if (val.hasattr('_repr_html_')) { + if ('_repr_html_' in val) { div.appendChild(new DOMParser().parseFromString( - val.getattr('_repr_html_').call([], {}), 'text/html').body.firstChild); + val._repr_html_(), 'text/html').body.firstChild); } else { let pre = document.createElement('pre'); pre.textContent = window.pyodide.repr(val); diff --git a/src/pyproxy.cpp b/src/pyproxy.cpp index e57358ae8..fa4727da0 100644 --- a/src/pyproxy.cpp +++ b/src/pyproxy.cpp @@ -2,19 +2,99 @@ using emscripten::val; -Py::Py(PyObject *obj) : x(obj) { - Py_INCREF(x); +val Py::makeProxy(PyObject *obj) { + Py_INCREF(obj); + return val::global("Proxy").new_(val(obj), val::global("pyodide")["Py"]); } -Py::Py(const Py& o) : x(o.x) { - Py_INCREF(x); +bool Py::isExtensible(val obj, val proxy) { + return true; } -Py::~Py() { - Py_DECREF(x); +bool Py::has(val obj, val idx) { + PyObject *x = obj.as(emscripten::allow_raw_pointers()); + PyObject *pyidx = jsToPython(idx); + bool result = PyObject_HasAttr(x, pyidx) ? true: false; + Py_DECREF(pyidx); + return result; } -val Py::call(val args, val kwargs) { +val Py::get(val obj, val idx, val proxy) { + if (idx.equals(val("$$"))) { + return obj["$$"]; + } + + PyObject *x = obj.as(emscripten::allow_raw_pointers()); + PyObject *pyidx = jsToPython(idx); + PyObject *attr = PyObject_GetAttr(x, pyidx); + Py_DECREF(pyidx); + if (attr == NULL) { + return pythonExcToJs(); + } + + val ret = pythonToJs(attr); + Py_DECREF(attr); + return ret; +}; + +val Py::set(val obj, val idx, val value, val proxy) { + PyObject *x = obj.as(emscripten::allow_raw_pointers()); + PyObject *pyidx = jsToPython(idx); + PyObject *pyvalue = jsToPython(value); + int ret = PyObject_SetAttr(x, pyidx, pyvalue); + Py_DECREF(pyidx); + Py_DECREF(pyvalue); + + if (ret) { + return pythonExcToJs(); + } + return value; +} + +val Py::deleteProperty(val obj, val idx) { + PyObject *x = obj.as(emscripten::allow_raw_pointers()); + PyObject *pyidx = jsToPython(idx); + + int ret = PyObject_DelAttr(x, pyidx); + Py_DECREF(pyidx); + + printf("return %d\n", ret); + + if (ret) { + return pythonExcToJs(); + } + + return val::global("undefined"); +} + +val Py::ownKeys(val obj) { + PyObject *x = obj.as(emscripten::allow_raw_pointers()); + PyObject *dir = PyObject_Dir(x); + if (dir == NULL) { + return pythonExcToJs(); + } + + val result = val::global("Array").new_(); + result.call("push", val("$$")); + Py_ssize_t n = PyList_Size(dir); + for (Py_ssize_t i = 0; i < n; ++i) { + PyObject *entry = PyList_GetItem(dir, i); + result.call("push", pythonToJs(entry)); + } + Py_DECREF(dir); + + return result; +} + +val Py::enumerate(val obj) { + return Py::ownKeys(obj); +} + +val PyCallable::makeCallableProxy(PyObject *obj) { + return val::global("pyodide").call("makeCallableProxy", PyCallable(obj)); +} + +val PyCallable::call(val args, val kwargs) { PyObject *pyargs = jsToPythonArgs(args); if (pyargs == NULL) { return pythonExcToJs(); @@ -37,67 +117,3 @@ val Py::call(val args, val kwargs) { Py_DECREF(pyret); return ret; } - -val Py::getattr(val idx) { - PyObject *pyidx = jsToPython(idx); - PyObject *attr = PyObject_GetAttr(x, pyidx); - Py_DECREF(pyidx); - if (attr == NULL) { - return pythonExcToJs(); - } - - val ret = pythonToJs(attr); - Py_DECREF(attr); - return ret; -} - -void Py::setattr(val idx, val v) { - PyObject *pyidx = jsToPython(idx); - PyObject *pyv = jsToPython(v); - - int ret = PyObject_SetAttr(x, pyidx, pyv); - Py_DECREF(pyidx); - Py_DECREF(pyv); - if (ret) { - pythonExcToJs(); - } -} - -val Py::hasattr(val idx) { - PyObject *pyidx = jsToPython(idx); - val result(PyObject_HasAttr(x, pyidx) ? true : false); - Py_DECREF(pyidx); - return result; -} - -val Py::getitem(val idx) { - PyObject *pyidx = jsToPython(idx); - PyObject *item = PyObject_GetItem(x, pyidx); - Py_DECREF(pyidx); - if (item == NULL) { - return pythonExcToJs(); - } - - val ret = pythonToJs(item); - Py_DECREF(item); - return ret; -} - -void Py::setitem(val idx, val v) { - PyObject *pyidx = jsToPython(idx); - PyObject *pyv = jsToPython(v); - - int ret = PyObject_SetItem(x, pyidx, pyv); - Py_DECREF(pyidx); - Py_DECREF(pyv); - if (ret) { - pythonExcToJs(); - } -} - -val Py::hasitem(val idx) { - PyObject *pyidx = jsToPython(idx); - val result(PySequence_Contains(x, pyidx) ? true : false); - Py_DECREF(pyidx); - return result; -} diff --git a/src/pyproxy.hpp b/src/pyproxy.hpp index 802c05f42..df725934d 100644 --- a/src/pyproxy.hpp +++ b/src/pyproxy.hpp @@ -9,21 +9,50 @@ #include "js2python.hpp" #include "python2js.hpp" +// This implements the Javascript Proxy handler interface as defined here: +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy + class Py { +public: + static emscripten::val makeProxy(PyObject *obj); + + static bool isExtensible( + emscripten::val obj, emscripten::val proxy); + static bool has( + emscripten::val obj, emscripten::val idx); + static emscripten::val get( + emscripten::val obj, emscripten::val idx, emscripten::val proxy); + static emscripten::val set( + emscripten::val obj, emscripten::val idx, emscripten::val value, + emscripten::val proxy); + static emscripten::val deleteProperty( + emscripten::val obj, emscripten::val idx); + static emscripten::val ownKeys( + emscripten::val obj); + static emscripten::val enumerate( + emscripten::val obj); +}; + +class PyCallable { public: PyObject *x; - Py(PyObject *obj); - Py(const Py& o); - ~Py(); + PyCallable(PyObject *x_) : x(x_) { + Py_INCREF(x); + } - emscripten::val call(emscripten::val args, emscripten::val kwargs); - emscripten::val getattr(emscripten::val idx); - void setattr(emscripten::val idx, emscripten::val v); - emscripten::val hasattr(emscripten::val idx); - emscripten::val getitem(emscripten::val idx); - void setitem(emscripten::val idx, emscripten::val v); - emscripten::val hasitem(emscripten::val idx); + PyCallable(const PyCallable& o) : x(o.x) { + Py_INCREF(x); + } + + ~PyCallable() { + Py_DECREF(x); + } + + emscripten::val call( + emscripten::val args, emscripten::val kwargs); + + static emscripten::val makeCallableProxy(PyObject *obj); }; #endif /* PYPROXY_H */ diff --git a/src/python.html b/src/python.html index 0c53f6731..a7c7919d0 100644 --- a/src/python.html +++ b/src/python.html @@ -32,7 +32,7 @@ "pluginType": "language" } }, - "lastExport": "2018-04-06T14:10:51.879Z" + "lastExport": "2018-05-04T17:13:00.489Z" } %% md @@ -124,12 +124,12 @@ def square(x, integer=False): return x * x %% md -To call a Python callable from Javascript, we use its `call` method, which takes two arguments: the positional arguments as an array, and the keyword arguments as an object. +Since calling conventions are a bit different in Python than in Javascript, all Python callables take two arguments when called from Javascript: the positional arguments as an array, and the keyword arguments as an object. %% js // javascript var square = pyodide.pyimport("square") -square.call([2.5], {integer: true}) +square([2.5], {integer: true}) %% md This is equivalent to the following Python syntax: @@ -155,12 +155,12 @@ We can get the value of its `val` property as so: %% js // javascript var foo = pyodide.pyimport("foo") -foo.getattr("val") +foo.val %% md ### Using Javascript objects from Python -Likewise, you can use Javascript objects from Python, and since the overloading features of Python are a little more dynamic, the syntax for doing so is also a little bit easier. +Likewise, you can use Javascript objects from Python. %% js // javascript @@ -266,4 +266,4 @@ A couple things that already work that will be coming to this example notebook s
- + \ No newline at end of file diff --git a/src/python2js.cpp b/src/python2js.cpp index 7e8a0528a..d78155f0b 100644 --- a/src/python2js.cpp +++ b/src/python2js.cpp @@ -17,49 +17,62 @@ val pythonExcToJs() { val excval(""); - PyObject *tbmod = PyImport_ImportModule("traceback"); - if (tbmod == NULL) { - excval = pythonToJs(PyObject_Repr(value)); - } else { - PyObject *format_exception; - if (traceback == NULL || traceback == Py_None) { - no_traceback = true; - format_exception = PyObject_GetAttrString(tbmod, "format_exception_only"); - } else { - format_exception = PyObject_GetAttrString(tbmod, "format_exception"); - } - if (format_exception == NULL) { - excval = val("Couldn't get format_exception function"); - } else { - PyObject *pylines; - if (no_traceback) { - pylines = PyObject_CallFunctionObjArgs - (format_exception, type, value, NULL); - } else { - pylines = PyObject_CallFunctionObjArgs - (format_exception, type, value, traceback, NULL); - } - if (pylines == NULL) { - excval = val("Error calling traceback.format_exception"); - PyErr_Print(); - } else { - PyObject *newline = PyUnicode_FromString("\n"); - PyObject *pystr = PyUnicode_Join(newline, pylines); - PyObject_Print(pystr, stderr, 0); - excval = pythonToJs(pystr); - Py_DECREF(pystr); - Py_DECREF(newline); - Py_DECREF(pylines); - } - Py_DECREF(format_exception); - } - Py_DECREF(tbmod); + if (type == NULL || type == Py_None || + value == NULL || value == Py_None) { + excval = val("No exception type or value"); + PyErr_Print(); + PyErr_Clear(); + goto exit; } + { + PyObject *tbmod = PyImport_ImportModule("traceback"); + if (tbmod == NULL) { + excval = pythonToJs(PyObject_Repr(value)); + } else { + PyObject *format_exception; + if (traceback == NULL || traceback == Py_None) { + no_traceback = true; + format_exception = PyObject_GetAttrString(tbmod, "format_exception_only"); + } else { + format_exception = PyObject_GetAttrString(tbmod, "format_exception"); + } + if (format_exception == NULL) { + excval = val("Couldn't get format_exception function"); + } else { + PyObject *pylines; + if (no_traceback) { + pylines = PyObject_CallFunctionObjArgs + (format_exception, type, value, NULL); + } else { + pylines = PyObject_CallFunctionObjArgs + (format_exception, type, value, traceback, NULL); + } + if (pylines == NULL) { + excval = val("Error calling traceback.format_exception"); + PyErr_Print(); + PyErr_Clear(); + goto exit; + } else { + PyObject *newline = PyUnicode_FromString("\n"); + PyObject *pystr = PyUnicode_Join(newline, pylines); + PyObject_Print(pystr, stderr, 0); + excval = pythonToJs(pystr); + Py_DECREF(pystr); + Py_DECREF(newline); + Py_DECREF(pylines); + } + Py_DECREF(format_exception); + } + Py_DECREF(tbmod); + } + } + + exit: val exc = val::global("Error").new_(excval); - Py_DECREF(type); - Py_DECREF(value); + Py_XDECREF(type); + Py_XDECREF(value); Py_XDECREF(traceback); PyErr_Clear(); @@ -114,9 +127,10 @@ val pythonToJs(PyObject *x) { for (size_t i = 0; i < length; ++i) { PyObject *item = PySequence_GetItem(x, i); if (item == NULL) { - // If something goes wrong converting the sequence, fallback to the Python proxy + // If something goes wrong converting the sequence (as is the case with + // Pandas data frames), fallback to the Python object proxy PyErr_Clear(); - return val(Py(x)); + return Py::makeProxy(x); } x_array.call("push", pythonToJs(item)); Py_DECREF(item); @@ -131,8 +145,10 @@ val pythonToJs(PyObject *x) { x_object.set(pythonToJs(k), pythonToJs(v)); } return x_object; + } else if (PyCallable_Check(x)) { + return PyCallable::makeCallableProxy(x); } else { - return val(Py(x)); + return Py::makeProxy(x); } } diff --git a/src/test.html b/src/test.html index 881a21305..d29e53a70 100644 --- a/src/test.html +++ b/src/test.html @@ -15,4 +15,5 @@ + diff --git a/test/test_python.py b/test/test_python.py index 01872b7c0..afb52f426 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -26,6 +26,28 @@ def test_import_js(selenium): assert result == 'Foo' +def test_py_proxy(selenium): + selenium.run( + "class Foo:\n bar = 42\n def get_value(self):\n return 64\nf = Foo()\n") + assert selenium.run_js("return pyodide.pyimport('f').get_value()") == 64 + assert selenium.run_js("return pyodide.pyimport('f').bar") == 42 + assert selenium.run_js("return ('bar' in pyodide.pyimport('f'))") == True + selenium.run_js("f = pyodide.pyimport('f'); f.baz = 32") + assert selenium.run("f.baz") == 32 + assert set(selenium.run_js( + "return Object.getOwnPropertyNames(pyodide.pyimport('f'))")) == set( + ['$$', '__class__', '__delattr__', '__dict__', '__dir__', + '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', + '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', + '__lt__', '__module__', '__ne__', '__new__', '__reduce__', + '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', + '__str__', '__subclasshook__', '__weakref__', 'bar', 'baz', + 'get_value']) + assert selenium.run("hasattr(f, 'baz')") == True + selenium.run_js("delete pyodide.pyimport('f').baz") + assert selenium.run("hasattr(f, 'baz')") == False + + def test_run_core_python_test(python_test, selenium): selenium.run( "import sys\n"