diff --git a/.gitignore b/.gitignore index 0ac425a1f..f82bacbbd 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,9 @@ geckodriver.log /numpy/build /numpy/host /numpy/downloads -/numpy/.patched \ No newline at end of file +/numpy/.patched + +/pandas/build +/pandas/host +/pandas/downloads +/pandas/.patched \ No newline at end of file diff --git a/Makefile b/Makefile index a86eb6f9b..729792b87 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,10 @@ NUMPY_LIBS=\ $(NUMPY_ROOT)/random/mtrand.so \ $(NUMPY_ROOT)/fft/fftpack_lite.so +PANDAS_ROOT=pandas/build/pandas +PANDAS_LIBS=\ + $(PANDAS_ROOT)/_libs/lib.so + SITEPACKAGES=root/lib/python$(PYMINOR)/site-packages all: build/pyodide.asm.html build/pyodide.js build/pyodide_dev.js @@ -85,6 +89,7 @@ clean: root/.built: \ $(CPYTHONLIB) \ $(NUMPY_LIBS) \ + $(PANDAS_LIBS) \ src/lazy_import.py \ src/sitecustomize.py \ src/webbrowser.py \ @@ -93,6 +98,7 @@ root/.built: \ mkdir -p root/lib cp -a $(CPYTHONLIB)/ root/lib cp -a numpy/build/numpy $(SITEPACKAGES) + cp -a pandas/build/pandas $(SITEPACKAGES) rm -fr $(SITEPACKAGES)/numpy/distutils cp src/lazy_import.py $(SITEPACKAGES) cp src/sitecustomize.py $(SITEPACKAGES) @@ -117,6 +123,8 @@ $(CPYTHONLIB): emsdk/emsdk/emsdk $(NUMPY_LIBS): $(CPYTHONLIB) make -C numpy +$(PANDAS_LIBS): $(NUMPY_LIBS) + make -C pandas emsdk/emsdk/emsdk: make -C emsdk diff --git a/pandas/Makefile b/pandas/Makefile new file mode 100644 index 000000000..6796a54ec --- /dev/null +++ b/pandas/Makefile @@ -0,0 +1,269 @@ +PYODIDE_ROOT=$(abspath ..) +include ../Makefile.envs + +PYVERSION=3.6.4 +PYMINOR=$(basename $(PYVERSION)) + +PDVERSION=0.22.0 +NPYVERSION=1.14.1 + +ROOT=$(abspath .) + +HOSTPYTHONINSTALL=$(ROOT)/../cpython/build/$(PYVERSION)/host +HOSTPYTHON=$(HOSTPYTHONINSTALL)/bin/python3$(EXE) +PYTHONINCLUDE=$(ROOT)/../cpython/installs/python-$(PYVERSION)/include/python$(PYMINOR) + +HOSTROOT=$(ROOT)/host +HOSTDIR=$(HOSTROOT)/pandas-$(PDVERSION) +HOSTBUILD=$(HOSTDIR)/build + +BUILD=$(ROOT)/build/pandas + +TARBALL=$(ROOT)/downloads/pandas-$(PDVERSION).tar.gz +URL=https://pypi.python.org/packages/08/01/803834bc8a4e708aedebb133095a88a4dad9f45bbaf5ad777d2bea543c7e/pandas-0.22.0.tar.gz + +PLATFORMSLUG=$(shell $(HOSTPYTHON) -c "import sysconfig; print(sysconfig.get_platform())")-$(PYMINOR) + +CC=emcc +CXX=em++ +AR=emar +CFLAGS=-Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I$(HOSTDIR)/pandas/_libs/src/klib -I$(HOSTDIR)/pandas/_libs/src -I../numpy/host/numpy-$(NPYVERSION)/numpy/core/include -I../numpy/config -I../numpy/host/numpy-$(NPYVERSION)/build/src.$(PLATFORMSLUG)/numpy/core/include/numpy -I$(PYTHONINCLUDE) -I$(HOSTDIR)/pandas/_libs/src/ujson/lib -I$(HOSTDIR)/pandas/_libs/src/datetime -Wno-unused-function +LDFLAGS=\ + -O3 \ + -Werror \ + -s EMULATED_FUNCTION_POINTERS=1 \ + -s EMULATE_FUNCTION_POINTER_CASTS=1 \ + -s SIDE_MODULE=1 \ + -s WASM=1 \ + --memory-init-file 0 + + +all: \ + $(BUILD)/__init__.py \ + $(BUILD)/_libs/algos.so \ + $(BUILD)/_libs/groupby.so \ + $(BUILD)/_libs/hashtable.so \ + $(BUILD)/_libs/index.so \ + $(BUILD)/_libs/interval.so \ + $(BUILD)/_libs/join.so \ + $(BUILD)/_libs/json.so \ + $(BUILD)/_libs/lib.so \ + $(BUILD)/_libs/parsers.so \ + $(BUILD)/_libs/period.so \ + $(BUILD)/_libs/properties.so \ + $(BUILD)/_libs/sparse.so \ + $(BUILD)/_libs/tslib.so \ + $(BUILD)/_libs/tslibs/fields.so \ + $(BUILD)/_libs/tslibs/frequencies.so \ + $(BUILD)/_libs/tslibs/strptime.so \ + $(BUILD)/_libs/tslibs/timedeltas.so \ + $(BUILD)/_libs/tslibs/timezones.so \ + $(BUILD)/_libs/window.so \ + $(BUILD)/io/msgpack/_packer.so \ + $(BUILD)/io/msgpack/_unpacker.so \ + $(BUILD)/io/sas/_sas.so \ + $(BUILD)/util/_move.so + + +clean: + rm -fr $(HOSTROOT) + rm -fr $(BUILD) + + +$(TARBALL): + [ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads + wget -q -O $@ $(URL) + md5sum --quiet --check checksums || (rm $@; false) + + +$(HOSTDIR)/setup.py: $(TARBALL) + [ -d $(HOSTROOT) ] || mkdir $(HOSTROOT) + tar -xf $(TARBALL) -C $(HOSTROOT) + touch $(HOSTDIR)/setup.py + + +$(HOSTBUILD)/lib.$(PLATFORMSLUG)/pandas/__init__.py: $(ROOT)/.patched + ( \ + cd $(HOSTDIR); \ + $(HOSTPYTHON) setup.py build \ + ) + + +$(BUILD)/__init__.py: $(HOSTBUILD)/lib.$(PLATFORMSLUG)/pandas/__init__.py + [ -d $(ROOT)/build ] || mkdir $(ROOT)/build + cp -r $(HOSTBUILD)/lib.$(PLATFORMSLUG)/pandas $(ROOT)/build && \ + cd $(BUILD); find . -name "*.so" -type f -delete + + +$(ROOT)/.patched: $(HOSTDIR)/setup.py + cat patches/*.patch | (cd $(HOSTDIR) ; patch -p1) + touch $@ + + +$(BUILD)/_libs/lib.so: $(HOSTDIR)/pandas/_libs/lib.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/properties.so: $(HOSTDIR)/pandas/_libs/properties.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/hashtable.so: $(HOSTDIR)/pandas/_libs/hashtable.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/tslibs/strptime.so: \ + $(HOSTDIR)/pandas/_libs/tslibs/strptime.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime_strings.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/tslib.so: \ + $(HOSTDIR)/pandas/_libs/tslib.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime_strings.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/tslibs/timedeltas.so: \ + $(HOSTDIR)/pandas/_libs/tslibs/timedeltas.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/tslibs/timezones.so: \ + $(HOSTDIR)/pandas/_libs/tslibs/timezones.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/tslibs/fields.so: \ + $(HOSTDIR)/pandas/_libs/tslibs/fields.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime_strings.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/period.so: \ + $(HOSTDIR)/pandas/_libs/period.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime_strings.bc \ + $(HOSTDIR)/pandas/_libs/src/period_helper.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/tslibs/frequencies.so: \ + $(HOSTDIR)/pandas/_libs/tslibs/frequencies.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/index.so: \ + $(HOSTDIR)/pandas/_libs/index.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime_strings.bc \ + $(HOSTDIR)/pandas/_libs/src/period_helper.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/algos.so: \ + $(HOSTDIR)/pandas/_libs/algos.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/groupby.so: \ + $(HOSTDIR)/pandas/_libs/groupby.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/join.so: \ + $(HOSTDIR)/pandas/_libs/join.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/interval.so: \ + $(HOSTDIR)/pandas/_libs/interval.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/window.so: \ + $(HOSTDIR)/pandas/_libs/window.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/parsers.so: \ + $(HOSTDIR)/pandas/_libs/parsers.bc \ + $(HOSTDIR)/pandas/_libs/src/parser/tokenizer.bc \ + $(HOSTDIR)/pandas/_libs/src/parser/io.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/sparse.so: \ + $(HOSTDIR)/pandas/_libs/sparse.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/hashing.so: \ + $(HOSTDIR)/pandas/_libs/hashing.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/io/sas/_sas.so: \ + $(HOSTDIR)/pandas/io/sas/sas.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/io/msgpack/_packer.so: \ + $(HOSTDIR)/pandas/io/msgpack/_packer.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/io/msgpack/_unpacker.so: \ + $(HOSTDIR)/pandas/io/msgpack/_unpacker.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/_libs/json.so: \ + $(HOSTDIR)/pandas/_libs/src/ujson/python/ujson.bc \ + $(HOSTDIR)/pandas/_libs/src/ujson/python/objToJSON.bc \ + $(HOSTDIR)/pandas/_libs/src/ujson/python/JSONtoObj.bc \ + $(HOSTDIR)/pandas/_libs/src/ujson/lib/ultrajsonenc.bc \ + $(HOSTDIR)/pandas/_libs/src/ujson/lib/ultrajsondec.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime.bc \ + $(HOSTDIR)/pandas/_libs/src/datetime/np_datetime_strings.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +$(BUILD)/util/_move.so: \ + $(HOSTDIR)/pandas/util/move.bc + $(CC) $(LDFLAGS) $^ -o tmp.wasm + mv tmp.wasm $@ + + +%.bc: %.c $(BUILD)/__init__.py + $(CC) $(CFLAGS) -include src/state.h -c $< -o $@ + + +%.bc: %.cpp $(BUILD)/__init__.py + $(CXX) $(CFLAGS) -include src/state.h -c $< -o $@ diff --git a/pandas/checksums b/pandas/checksums new file mode 100644 index 000000000..617a14ec3 --- /dev/null +++ b/pandas/checksums @@ -0,0 +1 @@ +c7a2757b607748255f3270221ac61d59 ./downloads/pandas-0.22.0.tar.gz \ No newline at end of file diff --git a/pandas/src/state.h b/pandas/src/state.h new file mode 100644 index 000000000..6f11baeb2 --- /dev/null +++ b/pandas/src/state.h @@ -0,0 +1,5 @@ +#include "Python.h" + +inline PyGILState_STATE PyGILState_Ensure(void) { return PyGILState_UNLOCKED; } + +inline void PyGILState_Release(PyGILState_STATE state) { } diff --git a/test/test_pandas.py b/test/test_pandas.py new file mode 100644 index 000000000..fca51898a --- /dev/null +++ b/test/test_pandas.py @@ -0,0 +1,2 @@ +def test_pandas(selenium): + selenium.run("import pandas")