Magisk module onboarding for Android (#5547)

* Added magisk module generation

* Fixed typo

* changelog

* Fixed mypy bug

* Changed action based on ubuntu 18.04 due to https://bit.ly/3QOw87Z

* Workflow pinned to ubuntu 20.04

* Moved magisk code to utils and gen on download

* Styling

* Removed magisk from git repo

* Added tests

* Fixed dead line

* Update CHANGELOG.md

* Hardcoded hash

Co-authored-by: Joran van Apeldoorn <joran@bitsoffreedom.nl>
Co-authored-by: Maximilian Hils <github@maximilianhils.com>

.github/workflows/main.yml

@@ -112,7 +112,7 @@ jobs:
             platform: macos
           - image: windows-2019
             platform: windows
-          - image: ubuntu-18.04  # Old Ubuntu version for old glibc
+          - image: ubuntu-20.04  # Oldest avalible version so we get oldest glibc possible.
             platform: linux
     runs-on: ${{ matrix.image }}
     env:

CHANGELOG.md

@@ -16,6 +16,8 @@
 * Fix `tls_version_server_min` and `tls_version_server_max` options.
   ([#5546](https://github.com/mitmproxy/mitmproxy/issues/5546), @mhils)
 * DTLS support ([#5397](https://github.com/mitmproxy/mitmproxy/pull/5397), @kckeiks).
+* Added Magisk module generation for Android onboarding (@jorants).
+* Update Linux binary builder to Ubuntu 20.04, bumping the minimum glibc version to 2.31. (@jorants)
 
 ## 28 June 2022: mitmproxy 8.1.1
 

mitmproxy/addons/onboardingapp/__init__.py

@@ -3,6 +3,7 @@ import os
 from flask import Flask, render_template
 
 from mitmproxy.options import CONF_BASENAME, CONF_DIR
+from mitmproxy.utils.magisk import write_magisk_module
 
 app = Flask(__name__)
 # will be overridden in the addon, setting this here so that the Flask app can be run standalone.
@@ -29,6 +30,24 @@ def cer():
     return read_cert("cer", "application/x-x509-ca-cert")
 
 
+@app.route("/cert/magisk")
+def magisk():
+    filename = CONF_BASENAME + f"-magisk-module.zip"
+    p = os.path.join(app.config["CONFDIR"], filename)
+    p = os.path.expanduser(p)
+
+    if not os.path.exists(p):
+        write_magisk_module(p)
+
+    with open(p, "rb") as f:
+        cert = f.read()
+
+    return cert, {
+        "Content-Type": "application/zip",
+        "Content-Disposition": f"attachment; filename={filename}",
+    }
+
+
 def read_cert(ext, content_type):
     filename = CONF_BASENAME + f"-ca-cert.{ext}"
     p = os.path.join(app.config["CONFDIR"], filename)
@@ -38,5 +57,5 @@ def read_cert(ext, content_type):
 
     return cert, {
         "Content-Type": content_type,
-        "Content-Disposition": f"inline; filename={filename}",
+        "Content-Disposition": f"attachment; filename={filename}",
     }

mitmproxy/addons/onboardingapp/templates/index.html

@@ -93,6 +93,9 @@
             patch most apps manually
             (<a href="https://developer.android.com/training/articles/security-config">Android network security config</a>).
         </p>
+        <p>
+          Alternatively, if you have rooted the device and have Magisk installed, you can install <a href="/cert/magisk">this Magisk module</a> via the Magisk Manager app.
+        </p>
     </div>
     {% endcall %}
     {% call entry('Firefox <small>(does not use the OS root certificates)</small>', 'firefox-browser') %}

mitmproxy/utils/magisk.py

@@ -0,0 +1,109 @@
+from zipfile import ZipFile
+import hashlib
+from cryptography import x509
+from cryptography.hazmat.primitives import serialization
+
+from mitmproxy import certs, ctx
+from mitmproxy.options import CONF_BASENAME
+
+import os
+
+# The following 3 variables are for including in the magisk module as text file
+MODULE_PROP_TEXT = """id=mitmproxycert
+name=MITMProxy cert
+version=v1
+versionCode=1
+author=mitmproxy
+description=Adds the mitmproxy certificate to the system store
+template=3"""
+
+CONFIG_SH_TEXT = """
+MODID=mitmproxycert
+AUTOMOUNT=true
+PROPFILE=false
+POSTFSDATA=false
+LATESTARTSERVICE=false
+
+print_modname() {
+  ui_print "*******************************"
+  ui_print "    MITMProxy cert installer   "
+  ui_print "*******************************"
+}
+
+REPLACE="
+"
+
+set_permissions() {
+  set_perm_recursive  $MODPATH  0  0  0755  0644
+}
+"""
+
+UPDATE_BINARY_TEXT = """
+#!/sbin/sh
+
+#################
+# Initialization
+#################
+
+umask 022
+
+# echo before loading util_functions
+ui_print() { echo "$1"; }
+
+require_new_magisk() {
+  ui_print "*******************************"
+  ui_print " Please install Magisk v20.4+! "
+  ui_print "*******************************"
+  exit 1
+}
+
+OUTFD=$2
+ZIPFILE=$3
+
+mount /data 2>/dev/null
+[ -f /data/adb/magisk/util_functions.sh ] || require_new_magisk
+. /data/adb/magisk/util_functions.sh
+[ $MAGISK_VER_CODE -lt 20400 ] && require_new_magisk
+
+install_module
+exit 0
+"""
+
+
+def get_ca_from_files() -> x509.Certificate:
+    # Borrowed from tlsconfig
+    certstore_path = os.path.expanduser(ctx.options.confdir)
+    certstore = certs.CertStore.from_store(
+        path=certstore_path,
+        basename=CONF_BASENAME,
+        key_size=ctx.options.key_size,
+        passphrase=ctx.options.cert_passphrase.encode("utf8")
+        if ctx.options.cert_passphrase
+        else None,
+    )
+    return certstore.default_ca._cert
+
+
+def subject_hash_old(ca : x509.Certificate) -> str:
+    # Mimics the -subject_hash_old option of openssl used for android certificate names
+    full_hash = hashlib.md5(ca.subject.public_bytes()).digest()
+    sho = (full_hash[0] | (full_hash[1] << 8) | (full_hash[2] << 16) | full_hash[3] << 24)
+    return hex(sho)[2:]
+
+
+def write_magisk_module(path : str):
+    # Makes a zip file that can be loaded by Magisk
+    # Android certs are stored as DER files
+    ca = get_ca_from_files()
+    der_cert = ca.public_bytes(serialization.Encoding.DER)
+    with ZipFile(path, "w") as zipp:
+        # Main cert file, name is always the old subject hash with a '.0' added
+        zipp.writestr(f"system/etc/security/cacerts/{subject_hash_old(ca)}.0", der_cert)
+        zipp.writestr("module.prop", MODULE_PROP_TEXT)
+        zipp.writestr("config.sh", CONFIG_SH_TEXT)
+        zipp.writestr("META-INF/com/google/android/updater-script", "#MAGISK")
+        zipp.writestr("META-INF/com/google/android/update-binary", UPDATE_BINARY_TEXT)
+        zipp.writestr("common/file_contexts_image", "/magisk(/.*)? u:object_r:system_file:s0")
+        zipp.writestr("common/post-fs-data.sh", "MODDIR=${0%/*}")
+        zipp.writestr("common/service.sh", "MODDIR=${0%/*}")
+        zipp.writestr("common/system.prop", "")

test/mitmproxy/addons/test_onboarding.py

@@ -20,7 +20,7 @@ class TestApp:
             tctx.configure(ob)
             assert client.get("/").status_code == 200
 
-    @pytest.mark.parametrize("ext", ["pem", "p12", "cer"])
+    @pytest.mark.parametrize("ext", ["pem", "p12", "cer", "magisk"])
     def test_cert(self, client, ext, tdata):
         ob = onboarding.Onboarding()
         with taddons.context(ob) as tctx:
@@ -29,7 +29,7 @@ class TestApp:
             assert resp.status_code == 200
             assert resp.data
 
-    @pytest.mark.parametrize("ext", ["pem", "p12", "cer"])
+    @pytest.mark.parametrize("ext", ["pem", "p12", "cer", "magisk"])
     def test_head(self, client, ext, tdata):
         ob = onboarding.Onboarding()
         with taddons.context(ob) as tctx:
@@ -37,4 +37,7 @@ class TestApp:
             resp = client.head(f"http://{tctx.options.onboarding_host}/cert/{ext}")
             assert resp.status_code == 200
             assert "Content-Length" in resp.headers
+            assert "Content-Type" in resp.headers
+            assert "Content-Disposition" in resp.headers
+            assert "attachment" in resp.headers["Content-Disposition"]
             assert not resp.data

test/mitmproxy/utils/test_magisk.py

@@ -0,0 +1,30 @@
+from mitmproxy.utils import magisk
+from cryptography import x509
+from mitmproxy.test import taddons
+import os
+
+
+def test_get_ca(tdata):
+    with taddons.context() as tctx:
+        tctx.options.confdir = tdata.path("mitmproxy/data/confdir")
+        ca = magisk.get_ca_from_files()
+        assert isinstance(ca, x509.Certificate)
+
+
+def test_subject_hash_old(tdata):
+    # checks if the hash is the same as that comming form openssl
+    with taddons.context() as tctx:
+        tctx.options.confdir = tdata.path("mitmproxy/data/confdir")
+        ca = magisk.get_ca_from_files()
+        our_hash = magisk.subject_hash_old(ca)
+        assert our_hash == "efb15d7d"
+
+
+def test_magisk_write(tdata, tmp_path):
+    # checks if the hash is the same as that comming form openssl
+    with taddons.context() as tctx:
+        tctx.options.confdir = tdata.path("mitmproxy/data/confdir")
+        magisk_path = tmp_path / "mitmproxy-magisk-module.zip"
+        magisk.write_magisk_module(magisk_path)
+
+        assert os.path.exists(magisk_path)