diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 5ed4814a0..a92ac31da 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -1,11 +1,12 @@ import json -import os +from pathlib import Path import re import subprocess import sys -sys.path.insert(0, os.path.abspath( - os.path.join(os.path.dirname(__file__), '..', 'test'))) +sys.path.insert( + 0, str((Path(__file__).resolve().parents[1] / 'test'))) + import conftest @@ -14,11 +15,12 @@ SKIP = set(['fft', 'hyantes']) def run_native(hostpython, code): output = subprocess.check_output( - [os.path.abspath(hostpython), '-c', code], - cwd=os.path.dirname(__file__), + [hostpython.resolve(), '-c', code], + cwd=Path(__file__).resolve().parent, env={ 'PYTHONPATH': - os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src'))} + str(Path(__file__).resolve().parents[1] / 'src') + } ) return float(output.strip().split()[-1]) @@ -72,12 +74,12 @@ def parse_numpy_benchmark(filename): def get_numpy_benchmarks(): - root = '../numpy-benchmarks/benchmarks' - for filename in os.listdir(root): - name = os.path.splitext(filename)[0] + root = Path('../numpy-benchmarks/benchmarks') + for filename in root.iterdir(): + name = filename.name if name in SKIP: continue - content = parse_numpy_benchmark(os.path.join(root, filename)) + content = parse_numpy_benchmark(filename) content += ( "import numpy as np\n" "_ = np.empty(())\n" @@ -103,6 +105,6 @@ def main(hostpython): if __name__ == '__main__': - results = main(sys.argv[-2]) + results = main(Path(sys.argv[-2]).resolve()) with open(sys.argv[-1], 'w') as fp: json.dump(results, fp) diff --git a/test/make_test_list.py b/test/make_test_list.py index d436664a3..d77d59032 100644 --- a/test/make_test_list.py +++ b/test/make_test_list.py @@ -3,21 +3,24 @@ Generate a list of test modules in the CPython distribution. """ import os +from pathlib import Path tests = [] -TEST_DIR = "../cpython/build/3.6.4/host/lib/python3.6/test" +TEST_DIR = Path("../cpython/build/3.6.4/host/lib/python3.6/test") + for root, dirs, files in os.walk( "../cpython/build/3.6.4/host/lib/python3.6/test"): - root = os.path.relpath(root, TEST_DIR) + root = Path(root).relative_to(TEST_DIR) if root == '.': root = '' else: root = '.'.join(root.split('/')) + '.' for filename in files: - if filename.startswith("test_") and filename.endswith(".py"): - tests.append(root + os.path.splitext(filename)[0]) + filename = Path(filename) + if str(filename).startswith("test_") and filename.suffix == ".py": + tests.append(str(root / filename.stem)) tests.sort() with open("python_tests.txt", "w") as fp: diff --git a/test/test_python.py b/test/test_python.py index 777bae350..9b5423b40 100644 --- a/test/test_python.py +++ b/test/test_python.py @@ -1,5 +1,5 @@ import os -import pathlib +from pathlib import Path import time @@ -295,8 +295,7 @@ def pytest_generate_tests(metafunc): test_modules = [] if 'CIRCLECI' not in os.environ or True: with open( - str(pathlib.Path(__file__).parents[0] / - "python_tests.txt")) as fp: + Path(__file__).parent / "python_tests.txt") as fp: for line in fp: line = line.strip() if line.startswith('#'): diff --git a/tools/buildall b/tools/buildall index e8986a7f8..5866788b3 100755 --- a/tools/buildall +++ b/tools/buildall @@ -6,7 +6,7 @@ Build all of the packages in a given directory. import argparse import json -import os +from pathlib import Path import shutil @@ -14,42 +14,39 @@ import common import buildpkg -def build_package(pkgname, dependencies, packagesdir, args): +def build_package(pkgname, dependencies, packagesdir, outputdir, args): reqs = dependencies[pkgname] # Make sure all of the package's requirements are built first for req in reqs: - build_package(req, dependencies, packagesdir, args) - if not os.path.isfile( - os.path.join(packagesdir, pkgname, 'build', '.packaged')): + build_package(req, dependencies, packagesdir, outputdir, args) + if not (packagesdir / pkgname / 'build' / '.packaged').is_file(): print("BUILDING PACKAGE: " + pkgname) - buildpkg.build_package( - os.path.join(packagesdir, pkgname, 'meta.yaml'), args) + buildpkg.build_package(packagesdir / pkgname / 'meta.yaml', args) shutil.copyfile( - os.path.join(packagesdir, pkgname, 'build', pkgname + '.data'), - os.path.join(args.output[0], pkgname + '.data')) + packagesdir / pkgname / 'build' / (pkgname + '.data'), + outputdir / (pkgname + '.data')) shutil.copyfile( - os.path.join(packagesdir, pkgname, 'build', pkgname + '.js'), - os.path.join(args.output[0], pkgname + '.js')) + packagesdir / pkgname / 'build' / (pkgname + '.js'), + outputdir / (pkgname + '.js')) -def build_packages(packagesdir, args): +def build_packages(packagesdir, outputdir, args): # We have to build the packages in the correct order (dependencies first), # so first load in all of the package metadata and build a dependency map. dependencies = {} - for pkgdir in os.listdir(packagesdir): - pkgdir = os.path.join(packagesdir, pkgdir) - pkgpath = os.path.join(pkgdir, 'meta.yaml') - if os.path.isdir(pkgdir) and os.path.isfile(pkgpath): + for pkgdir in packagesdir.iterdir(): + pkgpath = pkgdir / 'meta.yaml' + if pkgdir.is_dir() and pkgpath.is_file(): pkg = common.parse_package(pkgpath) name = pkg['package']['name'] reqs = pkg.get('requirements', {}).get('run', []) dependencies[name] = reqs for pkgname in dependencies.keys(): - build_package(pkgname, dependencies, packagesdir, args) + build_package(pkgname, dependencies, packagesdir, outputdir, args) # This is done last so the main Makefile can use it as a completion token - with open(os.path.join(args.output[0], 'packages.json'), 'w') as fd: + with open(outputdir / 'packages.json', 'w') as fd: json.dump({'dependencies': dependencies}, fd) @@ -78,8 +75,9 @@ def parse_args(): def main(args): - packagesdir = os.path.abspath(args.dir[0]) - build_packages(packagesdir, args) + packagesdir = Path(args.dir[0]).resolve() + outputdir = Path(args.output[0]).resolve() + build_packages(packagesdir, outputdir, args) if __name__ == '__main__': diff --git a/tools/buildpkg.py b/tools/buildpkg.py index 0c5cb0d8d..505d8dfa9 100755 --- a/tools/buildpkg.py +++ b/tools/buildpkg.py @@ -7,6 +7,7 @@ Builds a Pyodide package. import argparse import hashlib import os +from pathlib import Path import shutil import subprocess @@ -14,7 +15,7 @@ import subprocess import common -ROOTDIR = os.path.abspath(os.path.dirname(__file__)) +ROOTDIR = Path(__file__).parent.resolve() def check_checksum(path, pkg): @@ -37,40 +38,39 @@ def check_checksum(path, pkg): def download_and_extract(buildpath, packagedir, pkg, args): - tarballpath = os.path.join( - buildpath, os.path.basename(pkg['source']['url'])) - if not os.path.isfile(tarballpath): + tarballpath = buildpath / Path(pkg['source']['url']).name + if not tarballpath.is_file(): subprocess.run([ - 'wget', '-q', '-O', tarballpath, pkg['source']['url'] + 'wget', '-q', '-O', str(tarballpath), pkg['source']['url'] ], check=True) check_checksum(tarballpath, pkg) - srcpath = os.path.join(buildpath, packagedir) - if not os.path.isdir(srcpath): - shutil.unpack_archive(tarballpath, buildpath) + srcpath = buildpath / packagedir + if not srcpath.is_dir(): + shutil.unpack_archive(str(tarballpath), str(buildpath)) return srcpath def patch(path, srcpath, pkg, args): - if os.path.isfile(os.path.join(srcpath, '.patched')): + if (srcpath / '.patched').is_file(): return # Apply all of the patches - orig_dir = os.getcwd() - pkgdir = os.path.abspath(os.path.dirname(path)) + orig_dir = Path.cwd() + pkgdir = path.parent.resolve() os.chdir(srcpath) try: for patch in pkg['source'].get('patches', []): subprocess.run([ - 'patch', '-p1', '--binary', '-i', os.path.join(pkgdir, patch) + 'patch', '-p1', '--binary', '-i', pkgdir / patch ], check=True) finally: os.chdir(orig_dir) # Add any extra files for src, dst in pkg['source'].get('extras', []): - shutil.copyfile(os.path.join(pkgdir, src), os.path.join(srcpath, dst)) + shutil.copyfile(pkgdir / src, srcpath / dst) - with open(os.path.join(srcpath, '.patched'), 'wb') as fd: + with open(srcpath / '.patched', 'wb') as fd: fd.write(b'\n') @@ -78,31 +78,31 @@ def get_libdir(srcpath, args): # Get the name of the build/lib.XXX directory that distutils wrote its # output to slug = subprocess.check_output([ - os.path.join(args.host, 'bin', 'python3'), + str(Path(args.host) / 'bin' / 'python3'), '-c', 'import sysconfig, sys; ' 'print("{}-{}.{}".format(' 'sysconfig.get_platform(), ' 'sys.version_info[0], ' 'sys.version_info[1]))']).decode('ascii').strip() - purelib = os.path.join(srcpath, 'build', 'lib') - if os.path.isdir(purelib): + purelib = srcpath / 'build' / 'lib' + if purelib.is_dir(): libdir = purelib else: - libdir = os.path.join(srcpath, 'build', 'lib.' + slug) + libdir = srcpath / 'build' / ('lib.' + slug) return libdir def compile(path, srcpath, pkg, args): - if os.path.isfile(os.path.join(srcpath, '.built')): + if (srcpath / '.built').is_file(): return - orig_dir = os.getcwd() + orig_dir = Path.cwd() os.chdir(srcpath) try: subprocess.run([ - os.path.join(args.host, 'bin', 'python3'), - os.path.join(ROOTDIR, 'pywasmcross'), + str(Path(args.host) / 'bin' / 'python3'), + str(ROOTDIR / 'pywasmcross'), '--cflags', args.cflags + ' ' + pkg.get('build', {}).get('cflags', ''), @@ -117,7 +117,7 @@ def compile(path, srcpath, pkg, args): post = pkg.get('build', {}).get('post') if post is not None: libdir = get_libdir(srcpath, args) - pkgdir = os.path.abspath(os.path.dirname(path)) + pkgdir = path.parent.resolve() env = { 'BUILD': libdir, 'PKGDIR': pkgdir @@ -125,46 +125,46 @@ def compile(path, srcpath, pkg, args): subprocess.run([ 'bash', '-c', post], env=env, check=True) - with open(os.path.join(srcpath, '.built'), 'wb') as fd: + with open(srcpath / '.built', 'wb') as fd: fd.write(b'\n') def package_files(buildpath, srcpath, pkg, args): - if os.path.isfile(os.path.join(buildpath, '.packaged')): + if (buildpath / '.pacakaged').is_file(): return name = pkg['package']['name'] libdir = get_libdir(srcpath, args) subprocess.run([ 'python2', - os.path.join(os.environ['EMSCRIPTEN'], 'tools', 'file_packager.py'), - os.path.join(buildpath, name + '.data'), + Path(os.environ['EMSCRIPTEN']) / 'tools' / 'file_packager.py', + buildpath / (name + '.data'), '--preload', '{}@/lib/python3.6/site-packages'.format(libdir), - '--js-output={}'.format(os.path.join(buildpath, name + '.js')), + '--js-output={}'.format(buildpath / (name + '.js')), '--export-name=pyodide', '--exclude', '*.wasm.pre', '--exclude', '__pycache__', '--use-preload-plugins'], check=True) subprocess.run([ 'uglifyjs', - os.path.join(buildpath, name + '.js'), + buildpath / (name + '.js'), '-o', - os.path.join(buildpath, name + '.js')], check=True) + buildpath / (name + '.js')], check=True) - with open(os.path.join(buildpath, '.packaged'), 'wb') as fd: + with open(buildpath / '.packaged', 'wb') as fd: fd.write(b'\n') def build_package(path, args): pkg = common.parse_package(path) packagedir = pkg['package']['name'] + '-' + pkg['package']['version'] - dirpath = os.path.dirname(path) - orig_path = os.getcwd() + dirpath = path.parent + orig_path = Path.cwd() os.chdir(dirpath) try: - buildpath = os.path.join(dirpath, 'build') - if not os.path.exists(buildpath): + buildpath = dirpath / 'build' + if not buildpath.is_dir(): os.makedirs(buildpath) srcpath = download_and_extract(buildpath, packagedir, pkg, args) patch(path, srcpath, pkg, args) @@ -195,7 +195,7 @@ def parse_args(): def main(args): - path = os.path.abspath(args.package[0]) + path = Path(args.package[0]).resolve() build_package(path, args) diff --git a/tools/common.py b/tools/common.py index a15b54106..477a91c73 100644 --- a/tools/common.py +++ b/tools/common.py @@ -1,11 +1,9 @@ -import os +from pathlib import Path -ROOTDIR = os.path.abspath(os.path.dirname(__file__)) -HOSTPYTHON = os.path.abspath( - os.path.join(ROOTDIR, '..', 'cpython', 'build', '3.6.4', 'host')) -TARGETPYTHON = os.path.abspath( - os.path.join(ROOTDIR, '..', 'cpython', 'installs', 'python-3.6.4')) +ROOTDIR = Path(__file__).parent.resolve() +HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.6.4' / 'host' +TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.6.4' DEFAULTCFLAGS = '' DEFAULTLDFLAGS = ' '.join([ '-O3', diff --git a/tools/pywasmcross b/tools/pywasmcross index 3da054c39..2d219359c 100755 --- a/tools/pywasmcross +++ b/tools/pywasmcross @@ -28,6 +28,7 @@ import argparse import importlib.machinery import json import os +from pathlib import Path import re import subprocess import sys @@ -36,7 +37,7 @@ import sys import common -ROOTDIR = os.path.abspath(os.path.dirname(__file__)) +ROOTDIR = Path(__file__).parent.resolve() symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc']) @@ -52,8 +53,8 @@ def collect_args(basename): # native compiler env = dict(os.environ) path = env['PATH'] - while ROOTDIR + ':' in path: - path = path.replace(ROOTDIR + ':', '') + while str(ROOTDIR) + ':' in path: + path = path.replace(str(ROOTDIR) + ':', '') env['PATH'] = path with open('build.log', 'a') as fd: @@ -71,11 +72,11 @@ def make_symlinks(env): Makes sure all of the symlinks that make this script look like a compiler exist. """ - exec_path = os.path.abspath(__file__) + exec_path = Path(__file__).resolve() for symlink in symlinks: - symlink_path = os.path.join(ROOTDIR, symlink) - if not os.path.exists(symlink_path): - os.symlink(exec_path, symlink_path) + symlink_path = ROOTDIR / symlink + if not symlink_path.exists(): + symlink_path.symlink_to(exec_path) if symlink == 'c++': var = 'CXX' else: @@ -86,15 +87,16 @@ def make_symlinks(env): def capture_compile(args): env = dict(os.environ) make_symlinks(env) - env['PATH'] = ROOTDIR + ':' + os.environ['PATH'] + env['PATH'] = str(ROOTDIR) + ':' + os.environ['PATH'] result = subprocess.run( - [os.path.join(args.host, 'bin', 'python3'), + [Path(args.host) / 'bin' / 'python3', 'setup.py', 'install'], env=env) if result.returncode != 0: - if os.path.exists('build.log'): - os.remove('build.log') + build_log_path = Path('build.log') + if build_log_path.exists(): + build_log_path.unlink() sys.exit(result.returncode) @@ -131,7 +133,7 @@ def handle_command(line, args): # Don't include any system directories if arg[2:].startswith('/usr'): continue - if (os.path.abspath(arg[2:]).startswith(args.host) and + if (str(Path(arg[2:]).resolve()).startswith(args.host) and 'site-packages' not in arg): arg = arg.replace('-I' + args.host, '-I' + args.target) # Don't include any system directories @@ -168,8 +170,9 @@ def handle_command(line, args): def replay_compile(args): # If pure Python, there will be no build.log file, which is fine -- just do # nothing - if os.path.isfile('build.log'): - with open('build.log', 'r') as fd: + build_log_path = Path('build.log') + if build_log_path.is_file(): + with open(build_log_path, 'r') as fd: for line in fd: line = json.loads(line) handle_command(line, args) @@ -178,14 +181,14 @@ def replay_compile(args): def clean_out_native_artifacts(): for root, dirs, files in os.walk('.'): for file in files: - path = os.path.join(root, file) - basename, ext = os.path.splitext(file) - if ext in ('.o', '.so', '.a'): - os.remove(path) + path = Path(root) / file + if path.suffix in ('.o', '.so', '.a'): + path.unlink() def build_wrap(args): - if not os.path.isfile('build.log'): + build_log_path = Path('build.log') + if not build_log_path.is_file(): capture_compile(args) clean_out_native_artifacts() replay_compile(args) @@ -212,7 +215,7 @@ def parse_args(): if __name__ == '__main__': - basename = os.path.basename(sys.argv[0]) + basename = Path(sys.argv[0]).name if basename in symlinks: collect_args(basename) else: