Deploy top-of-tree xbuildenv and pyodide-core to S3 (#3766)

.circleci/config.yml

@@ -543,6 +543,7 @@ jobs:
             PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/debug/" make dist/console.html
             cp dist/console.html dist-debug/console.html
             PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make dist/console.html
+
       - run:
           name: Deploy to S3
           command: |
@@ -556,6 +557,20 @@ jobs:
           command: |
             python3 tools/deploy_s3.py dist-pyc/ "dev/pyc/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=3600, public' --rm-remote-prefix
 
+      - run:
+          # Unlike the release version, we upload the dev version to S3 not to GitHub.
+          name: Deploy release files to S3
+          command: |
+            mkdir -p /tmp/ghr/dist
+            cp -r dist /tmp/ghr/pyodide
+            cp -r xbuildenv /tmp/ghr/xbuildenv
+            cd /tmp/ghr
+            tar cjf dist/pyodide-core.tar.bz2 pyodide/pyodide{.js,.mjs,.asm.js,.asm.wasm} pyodide/{package,repodata}.json pyodide/python_stdlib.zip
+            tar cjf dist/xbuildenv.tar.bz2  xbuildenv/
+
+            python3 tools/deploy_s3.py dist/ "xbuildenv/dev" --bucket "pyodide-cache" --cache-control 'max-age=3600, public' --overwrite \
+              --access-key-env "AWS_ACCESS_KEY_ID_CACHE" --secret-key-env "AWS_SECRET_ACCESS_KEY_CACHE"
+
 workflows:
   version: 2
   build-and-deploy:
@@ -854,6 +869,7 @@ workflows:
             - test-core-firefox
             - test-packages-firefox
             - build-pyodide-debug
+            - create-xbuild-env
             - build-test-pyc-packages
           filters:
             branches:

tools/deploy_s3.py

@@ -1,6 +1,7 @@
 import gzip
 import io
 import mimetypes
+import os
 import shutil
 from pathlib import Path
 
@@ -70,9 +71,19 @@ def deploy_to_s3_main(
     rm_remote_prefix: bool = typer.Option(
         False, help="Remove existing files under the remote prefix"
     ),
+    access_key_env: str = typer.Option(
+        "AWS_ACCESS_KEY_ID", help="Environment variable name for AWS access key"
+    ),
+    secret_key_env: str = typer.Option(
+        "AWS_SECRET_ACCESS_KEY", help="Environment variable name for AWS secret key"
+    ),
 ):
     """Deploy a dist folder with Pyodide packages to AWS S3"""
-    s3_client = boto3.client("s3")
+    s3_client = boto3.client(
+        "s3",
+        aws_access_key_id=os.environ[access_key_env],
+        aws_secret_access_key=os.environ[secret_key_env],
+    )
 
     typer.echo(f"Deploying {local_folder} to s3://{bucket}/{remote_prefix}")
     typer.echo("Options: ")

tools/tests/test_deploy_s3.py

@@ -43,6 +43,8 @@ def test_deploy_to_s3_overwrite(tmp_path, capsys):
         pretend=False,
         overwrite=False,
         rm_remote_prefix=False,
+        access_key_env="AWS_ACCESS_KEY_ID",
+        secret_key_env="AWS_SECRET_ACCESS_KEY",
     )
 
     def get_object_list():
@@ -61,6 +63,8 @@ def test_deploy_to_s3_overwrite(tmp_path, capsys):
             pretend=False,
             overwrite=False,
             rm_remote_prefix=False,
+            access_key_env="AWS_ACCESS_KEY_ID",
+            secret_key_env="AWS_SECRET_ACCESS_KEY",
         )
     msg = "Cannot upload .* because it already exists"
     captured = capsys.readouterr()
@@ -76,6 +80,8 @@ def test_deploy_to_s3_overwrite(tmp_path, capsys):
         pretend=False,
         overwrite=True,
         rm_remote_prefix=False,
+        access_key_env="AWS_ACCESS_KEY_ID",
+        secret_key_env="AWS_SECRET_ACCESS_KEY",
     )
     assert get_object_list() == {"dev/full/a.whl", "dev/full/b.tar", "dev/full/c.zip"}
 
@@ -90,6 +96,8 @@ def test_deploy_to_s3_overwrite(tmp_path, capsys):
         pretend=False,
         overwrite=False,
         rm_remote_prefix=True,
+        access_key_env="AWS_ACCESS_KEY_ID",
+        secret_key_env="AWS_SECRET_ACCESS_KEY",
     )
     assert get_object_list() == {"dev/full/c.zip", "dev/full/a.whl"}
 
@@ -112,6 +120,8 @@ def test_deploy_to_s3_mime_type(tmp_path, capsys):
         pretend=False,
         overwrite=False,
         rm_remote_prefix=False,
+        access_key_env="AWS_ACCESS_KEY_ID",
+        secret_key_env="AWS_SECRET_ACCESS_KEY",
     )
 
     def get_header(key, field="content-type"):