From a11f72e145d2a15db467ba4885521382ce5d8b50 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Mon, 30 May 2022 10:26:40 +0900 Subject: [PATCH] Simplify the version bump process (#2587) --- docs/conf.py | 27 ++- docs/development/building-from-sources.md | 2 +- docs/development/maintainers.md | 50 +++-- docs/project/changelog.md | 4 +- docs/usage/downloading-and-deploying.md | 8 +- docs/usage/index.md | 2 +- docs/usage/loading-packages.md | 2 +- docs/usage/quickstart.md | 6 +- docs/usage/webworker.md | 2 +- pyodide-build/setup.cfg | 2 +- requirements.txt | 2 - setup.cfg | 23 --- src/py/pyodide/__init__.py | 2 +- src/py/setup.cfg | 2 +- tools/bump_version.py | 222 ++++++++++++++++++++++ 15 files changed, 285 insertions(+), 71 deletions(-) create mode 100755 tools/bump_version.py diff --git a/docs/conf.py b/docs/conf.py index 05b2a02cd..dc81dba69 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,13 @@ from unittest import mock # -- Project information ----------------------------------------------------- project = "Pyodide" -copyright = "2019-2021, Pyodide contributors and Mozilla" +copyright = "2019-2022, Pyodide contributors and Mozilla" +pyodide_version = "0.21.0.dev0" + +if ".dev" in pyodide_version: + CDN_URL = "https://cdn.jsdelivr.net/pyodide/dev/full/" +else: + CDN_URL = f"https://cdn.jsdelivr.net/pyodide/v{pyodide_version}/full/" # -- General configuration --------------------------------------------------- @@ -37,6 +43,7 @@ extensions = [ ] myst_enable_extensions = ["substitution"] + js_language = "typescript" jsdoc_config_path = "../src/js/tsconfig.json" root_for_relative_js_paths = "../src/" @@ -130,7 +137,7 @@ except ImportError: IN_READTHEDOCS = "READTHEDOCS" in os.environ if IN_READTHEDOCS: - env = {"PYODIDE_BASE_URL": "https://cdn.jsdelivr.net/pyodide/dev/full/"} + env = {"PYODIDE_BASE_URL": CDN_URL} os.makedirs("_build/html", exist_ok=True) res = subprocess.check_output( ["make", "-C", "..", "docs/_build/html/console.html"], @@ -197,3 +204,19 @@ if IN_SPHINX: for module in mock_modules: sys.modules[module] = mock.Mock() + + +# https://github.com/sphinx-doc/sphinx/issues/4054 +def globalReplace(app, docname, source): + result = source[0] + for key in app.config.global_replacements: + result = result.replace(key, app.config.global_replacements[key]) + source[0] = result + + +global_replacements = {"{{PYODIDE_CDN_URL}}": CDN_URL} + + +def setup(app): + app.add_config_value("global_replacements", {}, True) + app.connect("source-read", globalReplace) diff --git a/docs/development/building-from-sources.md b/docs/development/building-from-sources.md index 64c1a6a90..d5211e342 100644 --- a/docs/development/building-from-sources.md +++ b/docs/development/building-from-sources.md @@ -163,7 +163,7 @@ The following environment variables additionally impact the build: - `PYODIDE_BASE_URL`: Base URL where Pyodide packages are deployed. It must end with a trailing `/`. Default: `./` to load Pyodide packages from the same base URL path as where `pyodide.js` is located. Example: - `https://cdn.jsdelivr.net/pyodide/v0.20.0/full/` + `{{PYODIDE_CDN_URL}}` - `EXTRA_CFLAGS` : Add extra compilation flags. - `EXTRA_LDFLAGS` : Add extra linker flags. diff --git a/docs/development/maintainers.md b/docs/development/maintainers.md index c1b8300f5..bde3a4024 100644 --- a/docs/development/maintainers.md +++ b/docs/development/maintainers.md @@ -12,30 +12,19 @@ the latest release branch named `stable` (due to ReadTheDocs constraints). ### Making a major release -1. Make a new PR and for all occurrences of - `https://cdn.jsdelivr.net/pyodide/dev/full/` in `./docs/` replace `dev` with - the release version `vX.Y.Z` (note the presence of the leading `v`). This - also applies to `docs/conf.py`, but you should skip this file and - `docs/usage/downloading-and-deploying.md`. +1. From the root directory of the repository, -2. Set the version in: + ```bash + ./tools/bump_version.py --new-version + # ./tools/bump_version.py --new_version --dry-run + ``` - - `src/js/package.json`, - - `docs/project/about.md` (the Zenodo citation), - - `docs/development/building-from-sources.md`, - - `docs/usage/downloading-and-deploying.md`, - - Bump version in source code files by running `bump2version` command, for example, - - ```bash - bump2version minor - ``` - - check that the diff is correct with `git diff` before committing. + check that the diff is correct with `git diff` before committing. After this, try using `ripgrep` to make sure there are no extra old versions lying around e.g., `rg -F "0.18"`, `rg -F dev0`, `rg -F dev.0`. -3. Make sure the change log is up-to-date. +2. Make sure the change log is up-to-date. - Indicate the release date in the change log. - Generate the list of contributors for the release at the end of the @@ -46,7 +35,7 @@ the latest release branch named `stable` (due to ReadTheDocs constraints). where `LAST_TAG` is the tag for the last release. Merge the PR. -4. Assuming the upstream `stable` branch exists, rename it to a release branch +3. Assuming the upstream `stable` branch exists, rename it to a release branch for the previous major version. For instance if last release was, `0.20.0`, the corresponding release branch would be `0.20.X`, ```bash @@ -56,7 +45,7 @@ the latest release branch named `stable` (due to ReadTheDocs constraints). git push upstream 0.20.X git branch -D stable # delete locally ``` -5. Create a tag `X.Y.Z` (without leading `v`) and push +4. Create a tag `X.Y.Z` (without leading `v`) and push it to upstream, ```bash @@ -73,7 +62,7 @@ the latest release branch named `stable` (due to ReadTheDocs constraints). Wait for the CI to pass and create the release on GitHub. -6. Release the Pyodide JavaScript package: +5. Release the Pyodide JavaScript package: ```bash cd dist @@ -81,10 +70,15 @@ the latest release branch named `stable` (due to ReadTheDocs constraints). npm dist-tag add pyodide@a.b.c next # Label this release as also the latest unstable release ``` -7. Revert Step 1. and increment the version in `src/py/pyodide/__init__.py` to - the next version specified by Semantic Versioning. +6. Increment the version to the next version + specified by Semantic Versioning. Set `dev` version if needed. -8. Update this file with any relevant changes. + ```sh + # For example, if you just released 0.22.0, then set the version to 0.22.1.dev0 + ./tools/bump_version.py --new-version 0.22.1.dev0 + ``` + +7. Update this file with any relevant changes. ### Making a minor release @@ -108,13 +102,13 @@ This can be done with either, ``` and indicate which commits to take from `main` in the UI. -Then follow steps 2, 3, and 6 from {ref}`making-major-release`. +Then follow steps 1, 2, 5 and 6 from {ref}`making-major-release`. ### Making an alpha release -Follow steps 2, 6, 7, and 9 from {ref}`making-major-release`. Name the first -alpha release `x.x.xa1` and in subsequent alphas increment the final number. For -the npm package the alpha should have version in the format `x.x.x-alpha.1`. For +Follow steps 1, 5, and 6 from {ref}`making-major-release`. Name the first +alpha release `x.x.xa0` and in subsequent alphas increment the final number. For +the npm package the alpha should have version in the format `x.x.x-alpha.0`. For the node package make sure to use `npm publish --tag next` to avoid setting the alpha version as the stable release. diff --git a/docs/project/changelog.md b/docs/project/changelog.md index ee45e2459..5ad61a2f4 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -79,8 +79,8 @@ substitutions: rewriting and decorators such as `pytest.mark.parametrize` and hypothesis. {pr}`2510`, {pr}`2541` -- {{ BREAKING }} `pyodide_build.testing` is removed. `run_in_pyodide` decorator - can now be accessed through `pyodide_test_runner`. +- {{ Breaking }} `pyodide_build.testing` is removed. `run_in_pyodide` + decorator can now be accessed through `pyodide_test_runner`. {pr}`2418` - {{ Enhancement }} Added the `js_id` attribute to `JsProxy` to allow using diff --git a/docs/usage/downloading-and-deploying.md b/docs/usage/downloading-and-deploying.md index c28fc6ed4..0d25b60be 100644 --- a/docs/usage/downloading-and-deploying.md +++ b/docs/usage/downloading-and-deploying.md @@ -8,10 +8,10 @@ Pyodide packages, including the `pyodide.js` file, are available from the JsDelivr CDN, -| channel | indexURL | Comments | REPL | -| ------------------- | ------------------------------------------------ | ---------------------------------------------------------------------------------------- | -------------------------------------------------- | -| Latest release | `https://cdn.jsdelivr.net/pyodide/v0.20.0/full/` | Recommended, cached by the browser | [link](https://pyodide.org/en/stable/console.html) | -| Dev (`main` branch) | `https://cdn.jsdelivr.net/pyodide/dev/full/` | Re-deployed for each commit on main, no browser caching, should only be used for testing | [link](https://pyodide.org/en/latest/console.html) | +| channel | indexURL | Comments | REPL | +| ------------------- | -------------------------------------------- | ---------------------------------------------------------------------------------------- | -------------------------------------------------- | +| Latest release | `{{PYODIDE_CDN_URL}}` | Recommended, cached by the browser | [link](https://pyodide.org/en/stable/console.html) | +| Dev (`main` branch) | `https://cdn.jsdelivr.net/pyodide/dev/full/` | Re-deployed for each commit on main, no browser caching, should only be used for testing | [link](https://pyodide.org/en/latest/console.html) | To access a particular file, append the file name to `indexURL`. For instance, `"${indexURL}pyodide.js"` in the case of `pyodide.js`. diff --git a/docs/usage/index.md b/docs/usage/index.md index 27d61511a..18b41b054 100644 --- a/docs/usage/index.md +++ b/docs/usage/index.md @@ -12,7 +12,7 @@ Pyodide with {any}`loadPyodide ` specifying an index URL - + + Pyodide test page
@@ -86,7 +86,7 @@ Create and save a test `index.html` page with the following contents: - + diff --git a/docs/usage/webworker.md b/docs/usage/webworker.md index c3ed281bb..9a42fee70 100644 --- a/docs/usage/webworker.md +++ b/docs/usage/webworker.md @@ -105,7 +105,7 @@ shown below: // Setup your project to serve `py-worker.js`. You should also serve // `pyodide.js`, and all its associated `.asm.js`, `.data`, `.json`, // and `.wasm` files as well: -importScripts("https://cdn.jsdelivr.net/pyodide/dev/full/pyodide.js"); +importScripts("{{PYODIDE_CDN_URL}}pyodide.js"); async function loadPyodideAndPackages() { self.pyodide = await loadPyodide(); diff --git a/pyodide-build/setup.cfg b/pyodide-build/setup.cfg index 9b4abb45a..b9b618893 100644 --- a/pyodide-build/setup.cfg +++ b/pyodide-build/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pyodide-build -version = 0.20.0dev0 +version = 0.21.0.dev0 author = Pyodide developers description = "Tools for building Pyodide" long_description = file: README.md diff --git a/requirements.txt b/requirements.txt index bced41197..a196ed87f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,5 +16,3 @@ pytest-xdist selenium==4.1.0 tblib - # maintenance - bump2version diff --git a/setup.cfg b/setup.cfg index d89bd7022..fe32db21d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,26 +17,3 @@ testpaths = src pyodide-build packages - - -[bumpversion] -current_version = 0.20.0 -commit = False -tag = False -tag_name = {new_version} - -[bumpversion:file:src/py/pyodide/__init__.py] -search = __version__ = "{current_version}" -replace = __version__ = "{new_version}" - -[bumpversion:file:src/py/setup.cfg] -search = version = {current_version} -replace = version = {new_version} - -[bumpversion:file:pyodide-build/setup.cfg] -search = version = {current_version} -replace = version = {new_version} - -[bumpversion:file:run_docker] -search = PYODIDE_PREBUILT_IMAGE_TAG="{current_version}" -replace = PYODIDE_PREBUILT_IMAGE_TAG="{new_version}" diff --git a/src/py/pyodide/__init__.py b/src/py/pyodide/__init__.py index 6780216bc..3a9a9acba 100644 --- a/src/py/pyodide/__init__.py +++ b/src/py/pyodide/__init__.py @@ -50,7 +50,7 @@ if IN_BROWSER: asyncio.set_event_loop_policy(WebLoopPolicy()) -__version__ = "0.20.0" +__version__ = "0.21.0.dev0" __all__ = [ "CodeRunner", diff --git a/src/py/setup.cfg b/src/py/setup.cfg index 998627f23..a39bf4dc7 100644 --- a/src/py/setup.cfg +++ b/src/py/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pyodide -version = 0.20.0 +version = 0.21.0.dev0 author = Pyodide developers description = "A Python package providing core interpreter functionality for Pyodide." long_description = file: README.md diff --git a/tools/bump_version.py b/tools/bump_version.py new file mode 100755 index 000000000..c1280aad0 --- /dev/null +++ b/tools/bump_version.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 + +import argparse +import difflib +import functools +import itertools +import pathlib +import re +from ast import Str +from collections import namedtuple +from typing import Callable + +CORE_VERSION_REGEX = r"(?P\d+)\.(?P\d+)\.(?P\d+)" + +PYTHON_VERSION_REGEX = CORE_VERSION_REGEX + ( + r"((?P
a|b|rc)(?P\d+))?" r"(\.(?Pdev)(?P\d+))?"
+)
+
+JS_VERSION_REGEX = CORE_VERSION_REGEX + (
+    r"(\-(?P
alpha|beta|rc)\.(?P\d+))?"
+    r"(\-(?Pdev)\.(?P\d+))?"
+)
+
+
+def build_version_pattern(pattern):
+    return re.compile(
+        pattern.format(
+            python_version=f"(?P{PYTHON_VERSION_REGEX})",
+            js_version=f"(?P{JS_VERSION_REGEX})",
+        )
+    )
+
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+Target = namedtuple("target", ("file", "pattern", "prerelease"))
+PYTHON_TARGETS = [
+    Target(
+        file=ROOT / "src/py/pyodide/__init__.py",
+        pattern=build_version_pattern('__version__ = "{python_version}"'),
+        prerelease=True,
+    ),
+    Target(
+        file=ROOT / "src/py/setup.cfg",
+        pattern=build_version_pattern("version = {python_version}"),
+        prerelease=True,
+    ),
+    Target(
+        ROOT / "pyodide-build/setup.cfg",
+        build_version_pattern("version = {python_version}"),
+        prerelease=True,
+    ),
+    Target(
+        ROOT / "docs/conf.py",
+        build_version_pattern('pyodide_version = "{python_version}"'),
+        prerelease=True,
+    ),
+    Target(
+        ROOT / "run_docker",
+        build_version_pattern('PYODIDE_PREBUILT_IMAGE_TAG="{python_version}"'),
+        prerelease=False,
+    ),
+    Target(
+        ROOT / "docs/project/about.md",
+        build_version_pattern(r"version\s*=\s*{{{python_version}}}"),
+        prerelease=False,
+    ),
+]
+
+JS_TARGETS = [
+    Target(
+        ROOT / "src/js/package.json",
+        build_version_pattern(r'"pyodide",\s*"version": "{js_version}"'),
+        prerelease=True,
+    ),
+    Target(
+        ROOT / "src/js/package-lock.json",
+        build_version_pattern(r'"pyodide",\s*"version": "{js_version}"'),
+        prerelease=True,
+    ),
+]
+
+
+@functools.lru_cache
+def python_version_to_js_version(version: str) -> Str:
+    """
+    Convert Python version name to JS version name
+    These two are different in prerelease or dev versions.
+    e.g. 1.2.3a0 <==> 1.2.3-alpha.0
+         4.5.6.dev2 <==> 4.5.6-dev.2
+    """
+    match = re.match(PYTHON_VERSION_REGEX, version)
+    matches = match.groupdict()
+
+    prerelease = matches["pre"] is not None
+    devrelease = matches["dev"] is not None
+
+    if prerelease and devrelease:
+        raise ValueError("Cannot have both prerelease and devrelease")
+    elif prerelease:
+        matches["pre"] = matches["pre"].replace("a", "alpha").replace("b", "beta")
+        return "{major}.{minor}.{patch}-{pre}.{preversion}".format(**matches)
+    elif devrelease:
+        return "{major}.{minor}.{patch}-{dev}.{devversion}".format(**matches)
+    else:
+        return "{major}.{minor}.{patch}".format(**matches)
+
+
+@functools.lru_cache
+def is_core_version(version: str) -> bool:
+    match = re.fullmatch(CORE_VERSION_REGEX, version)
+    if match is None:
+        return False
+
+    return True
+
+
+def parse_current_version(target: Target) -> str:
+    """Parse current version"""
+    content = target.file.read_text()
+    match = target.pattern.search(content)
+
+    if match is None:
+        raise ValueError(f"Unabled to detect version string: {target.file}")
+
+    return match.groupdict()["version"]
+
+
+def generate_updated_content(
+    target: Target, current_version: str, new_version: str
+) -> Callable:
+    file = target.file
+    pattern = target.pattern
+    content = file.read_text()
+
+    if current_version == new_version:
+        return None
+
+    # Some files only required to be bumped on core version release.
+    # For example, we don't deploy prebuilt docker images for dev release.
+    if not target.prerelease:
+        if not is_core_version(new_version):
+            print(f"[*] {file}: Skipped (not targeting a core version)")
+            return None
+
+    new_content = content
+    startpos = 0
+    while match := pattern.search(new_content, pos=startpos):
+        version = match.groupdict()["version"]
+        if version == current_version:
+            start, end = match.span()
+            new_span = new_content[start:end].replace(current_version, new_version)
+            new_content = new_content[:start] + new_span + new_content[end:]
+            startpos = end
+        elif version == new_version:
+            break
+        else:
+            raise ValueError(
+                f"'{file}' contains invalid version: expected '{current_version}' but found '{version}'"
+            )
+
+    show_diff(content, new_content, file)
+
+    return new_content
+
+
+def show_diff(before: str, after: str, file: pathlib.Path):
+    diffs = list(
+        difflib.unified_diff(
+            before.splitlines(keepends=True), after.splitlines(keepends=True), n=0
+        )
+    )[2:]
+    print(f"[*] Diff of '{file}':\n")
+    print("".join(diffs))
+
+
+def parse_args():
+    parser = argparse.ArgumentParser("Bump version strings in the Pyodide repository")
+    parser.add_argument("--new-version", help="New version")
+    parser.add_argument(
+        "--dry-run", action="store_true", help="Don't actually write anything"
+    )
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    if args.new_version is None:
+        new_version = input("New version (e.g. 0.22.0, 0.22.0a0, 0.22.0.dev0): ")
+    else:
+        new_version = args.new_version
+
+    if re.fullmatch(PYTHON_VERSION_REGEX, new_version) is None:
+        raise ValueError(f"Invalid new version: {new_version}")
+
+    new_version_py = new_version
+    new_version_js = python_version_to_js_version(new_version)
+
+    # We want to update files in all-or-nothing strategy,
+    # so we keep the queue of update functions
+    update_queue = []
+
+    targets = itertools.chain(
+        zip(PYTHON_TARGETS, [new_version_py] * len(PYTHON_TARGETS)),
+        zip(JS_TARGETS, [new_version_js] * len(JS_TARGETS)),
+    )
+    for target, new_version in targets:
+        current_version = parse_current_version(target)
+        new_content = generate_updated_content(target, current_version, new_version)
+        if new_content is not None:
+            update_queue.append((target, new_content))
+
+    if args.dry_run:
+        return
+
+    for file, content in update_queue:
+        file.write_text(content)
+
+
+if __name__ == "__main__":
+    main()