TLS Interception Cert Generation (#362)

* Use common.pki for interception certificate generation * Fix tests * Dont use certificate fields that we dont need, it leads to certificate generation error on Ubuntu * Prepare for v2.2.0 * npm audit fix
2020-06-09 12:07:00 +05:30 · 2020-06-09 12:07:00 +05:30 · e7aa8a28f7
parent ab08901239
commit e7aa8a28f7
9 changed files with 116 additions and 78 deletions
--- a/README.md
+++ b/README.md
@ -795,8 +795,7 @@ response from the server.  Start `proxy.py` as:
 ```


-> :note: **MacOS users** also need to pass explicit CA file path
-> needed for validation of peer certificates. See --ca-file flag.
+[![NOTE](https://img.shields.io/static/v1?label=MacOS&message=note&color=yellow)](https://github.com/abhinavsingh/proxy.py#flags) Also provide explicit CA bundle path needed for validation of peer certificates. See `--ca-file` flag.


 Verify TLS interception using `curl`
@ -1327,7 +1326,7 @@ usage: pki.py [-h] [--password PASSWORD] [--private-key-path PRIVATE_KEY_PATH]
              [--public-key-path PUBLIC_KEY_PATH] [--subject SUBJECT]
              action

-proxy.py v2.1.2 : PKI Utility
+proxy.py v2.2.0 : PKI Utility

 positional arguments:
  action                Valid actions: remove_passphrase, gen_private_key,
@ -1518,7 +1517,7 @@ usage: proxy [-h] [--backlog BACKLOG] [--basic-auth BASIC_AUTH]
             [--static-server-dir STATIC_SERVER_DIR] [--threadless]
             [--timeout TIMEOUT] [--version]

-proxy.py v2.1.2
+proxy.py v2.2.0

 optional arguments:
  -h, --help            show this help message and exit
--- a/dashboard/package-lock.json
+++ b/dashboard/package-lock.json
@ -1382,9 +1382,9 @@
      "dev": true
    },
    "follow-redirects": {
-      "version": "1.10.0",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.10.0.tgz",
-      "integrity": "sha512-4eyLK6s6lH32nOvLLwlIOnr9zrL8Sm+OvW4pVTJNoXeGzYIkHVf+pADQi+OJ0E67hiuSLezPVPyBcIZO50TmmQ==",
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.11.0.tgz",
+      "integrity": "sha512-KZm0V+ll8PfBrKwMzdo5D13b1bur9Iq9Zd/RMmAoQQcl2PxxFml8cxXPaaPYVbV0RjNjq1CU7zIzAOqtUPudmA==",
      "dev": true,
      "requires": {
        "debug": "^3.0.0"
@ -1582,9 +1582,9 @@
      }
    },
    "http-proxy": {
-      "version": "1.18.0",
-      "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.0.tgz",
-      "integrity": "sha512-84I2iJM/n1d4Hdgc6y2+qY5mDaz2PUVjlg9znE9byl+q0uC3DeByqBGReQu5tpLK0TAqTIXScRUV+dg7+bUPpQ==",
+      "version": "1.18.1",
+      "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz",
+      "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==",
      "dev": true,
      "requires": {
        "eventemitter3": "^4.0.0",
@ -1593,19 +1593,19 @@
      }
    },
    "http-server": {
-      "version": "0.12.1",
-      "resolved": "https://registry.npmjs.org/http-server/-/http-server-0.12.1.tgz",
-      "integrity": "sha512-T0jB+7J7GJ2Vo+a4/T7P7SbQ3x2GPDnqRqQXdfEuPuUOmES/9NBxPnDm7dh1HGEeUWqUmLUNtGV63ZC5Uy3tGA==",
+      "version": "0.12.3",
+      "resolved": "https://registry.npmjs.org/http-server/-/http-server-0.12.3.tgz",
+      "integrity": "sha512-be0dKG6pni92bRjq0kvExtj/NrrAd28/8fCXkaI/4piTwQMSDSLMhWyW0NI1V+DBI3aa1HMlQu46/HjVLfmugA==",
      "dev": true,
      "requires": {
        "basic-auth": "^1.0.3",
-        "colors": "^1.3.3",
+        "colors": "^1.4.0",
        "corser": "^2.0.1",
        "ecstatic": "^3.3.2",
-        "http-proxy": "^1.17.0",
+        "http-proxy": "^1.18.0",
+        "minimist": "^1.2.5",
        "opener": "^1.5.1",
-        "optimist": "~0.6.1",
-        "portfinder": "^1.0.20",
+        "portfinder": "^1.0.25",
        "secure-compare": "3.0.1",
        "union": "~0.5.0"
      }
@ -2641,30 +2641,6 @@
        "pinkie-promise": "^2.0.0"
      }
    },
-    "optimist": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz",
-      "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=",
-      "dev": true,
-      "requires": {
-        "minimist": "~0.0.1",
-        "wordwrap": "~0.0.2"
-      },
-      "dependencies": {
-        "minimist": {
-          "version": "0.0.10",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz",
-          "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=",
-          "dev": true
-        },
-        "wordwrap": {
-          "version": "0.0.3",
-          "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz",
-          "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=",
-          "dev": true
-        }
-      }
-    },
    "optionator": {
      "version": "0.8.2",
      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz",
@ -2832,9 +2808,9 @@
      "dev": true
    },
    "portfinder": {
-      "version": "1.0.25",
-      "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.25.tgz",
-      "integrity": "sha512-6ElJnHBbxVA1XSLgBp7G1FiCkQdlqGzuF7DswL5tcea+E8UpuvPU7beVAjjRwCioTS9ZluNbu+ZyRvgTsmqEBg==",
+      "version": "1.0.26",
+      "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.26.tgz",
+      "integrity": "sha512-Xi7mKxJHHMI3rIUrnm/jjUgwhbYMkp/XKEcZX3aG4BrumLpq3nmoQMX+ClYnDZnZ/New7IatC1no5RX0zo1vXQ==",
      "dev": true,
      "requires": {
        "async": "^2.6.2",
@ -2873,9 +2849,9 @@
      "dev": true
    },
    "qs": {
-      "version": "6.9.2",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.2.tgz",
-      "integrity": "sha512-2eQ6zajpK7HwqrY1rRtGw5IZvjgtELXzJECaEDuzDFo2jjnIXpJSimzd4qflWZq6bLLi+Zgfj5eDrAzl/lptyg==",
+      "version": "6.9.4",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz",
+      "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==",
      "dev": true
    },
    "read-pkg": {
--- a/dashboard/package.json
+++ b/dashboard/package.json
@ -37,7 +37,7 @@
    "eslint-plugin-node": "^10.0.0",
    "eslint-plugin-promise": "^4.2.1",
    "eslint-plugin-standard": "^4.0.1",
-    "http-server": "^0.12.1",
+    "http-server": "^0.12.3",
    "jasmine": "^3.5.0",
    "jasmine-ts": "^0.3.0",
    "jquery": "^3.5.0",
--- a/helper/homebrew/stable/proxy.rb
+++ b/helper/homebrew/stable/proxy.rb
@ -5,7 +5,7 @@ class Proxy < Formula
    Network monitoring, controls & Application development, testing, debugging."
  homepage "https://github.com/abhinavsingh/proxy.py"
  url "https://github.com/abhinavsingh/proxy.py/archive/master.zip"
-  version "2.0.0"
+  version "2.1.2"

  depends_on "python"

--- a/proxy/common/version.py
+++ b/proxy/common/version.py
@ -8,5 +8,5 @@
    :copyright: (c) 2013-present by Abhinav Singh and contributors.
    :license: BSD, see LICENSE for more details.
 """
-VERSION = (2, 1, 2)
+VERSION = (2, 2, 0)
 __version__ = '.'.join(map(str, VERSION[0:3]))
--- a/proxy/http/proxy/server.py
+++ b/proxy/http/proxy/server.py
@ -8,14 +8,13 @@
    :copyright: (c) 2013-present by Abhinav Singh and contributors.
    :license: BSD, see LICENSE for more details.
 """
+import logging
 import threading
-import subprocess
 import os
 import ssl
 import socket
 import time
 import errno
-import logging
 from typing import Optional, List, Union, Dict, cast, Any, Tuple

 from .plugin import HttpProxyBasePlugin
@ -28,6 +27,7 @@ from ..methods import httpMethods
 from ...common.types import HasFileno
 from ...common.constants import PROXY_AGENT_HEADER_VALUE
 from ...common.utils import build_http_response, text_
+from ...common.pki import gen_public_key, gen_csr, sign_csr

 from ...core.event import eventNames
 from ...core.connection import TcpServerConnection, TcpConnectionUninitializedException
@ -279,7 +279,8 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin):
                        'BrokenPipeError when wrapping client')
                    return True
                except OSError as e:
-                    logger.exception('OSError when wrapping client', exc_info=e)
+                    logger.exception(
+                        'OSError when wrapping client', exc_info=e)
                    return True
                # Update all plugin connection reference
                for plugin in self.plugins.values():
@ -342,6 +343,57 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin):
                 self.response.total_size,
                 connection_time_ms))

+    def gen_ca_signed_certificate(self, cert_file_path: str) -> None:
+        '''CA signing key (default) is used for generating a public key
+        for common_name, if one already doesn't exist.  Using generated
+        public key a CSR request is generated, which is then signed by
+        CA key and secret.  Again this process only happen if signed
+        certificate doesn't already exist.
+
+        returns signed certificate path.'''
+        assert(self.request.host and self.flags.ca_cert_dir and self.flags.ca_signing_key_file and
+               self.flags.ca_key_file and self.flags.ca_cert_file)
+        public_key_path = os.path.join(self.flags.ca_cert_dir,
+                                       '{0}.{1}'.format(text_(self.request.host), 'pub'))
+        private_key_path = self.flags.ca_signing_key_file
+        private_key_password = ''
+        subject = '/CN={0}'.format(text_(self.request.host))
+        alt_subj_names = [text_(self.request.host), ]
+        validity_in_days = 365 * 2
+        timeout = 10
+
+        # Generate a public key for the common name
+        if not os.path.isfile(public_key_path):
+            logger.debug('Generating public key %s', public_key_path)
+            resp = gen_public_key(public_key_path=public_key_path, private_key_path=private_key_path,
+                                  private_key_password=private_key_password, subject=subject, alt_subj_names=alt_subj_names,
+                                  validity_in_days=validity_in_days, timeout=timeout)
+            assert(resp is True)
+
+        csr_path = os.path.join(self.flags.ca_cert_dir,
+                                '{0}.{1}'.format(text_(self.request.host), 'csr'))
+
+        # Generate a CSR request for this common name
+        if not os.path.isfile(csr_path):
+            logger.debug('Generating CSR %s', csr_path)
+            resp = gen_csr(csr_path=csr_path, key_path=private_key_path, password=private_key_password,
+                           crt_path=public_key_path, timeout=timeout)
+            assert(resp is True)
+
+        ca_key_path = self.flags.ca_key_file
+        ca_key_password = ''
+        ca_crt_path = self.flags.ca_cert_file
+        serial = self.uid.int
+
+        # Sign generated CSR
+        if not os.path.isfile(cert_file_path):
+            logger.debug('Signing CSR %s', cert_file_path)
+            resp = sign_csr(csr_path=csr_path, crt_path=cert_file_path, ca_key_path=ca_key_path,
+                            ca_key_password=ca_key_password, ca_crt_path=ca_crt_path,
+                            serial=str(serial), alt_subj_names=alt_subj_names,
+                            validity_in_days=validity_in_days, timeout=timeout)
+            assert(resp is True)
+
    @staticmethod
    def generated_cert_file_path(ca_cert_dir: str, host: str) -> str:
        return os.path.join(ca_cert_dir, '%s.pem' % host)
@ -359,21 +411,7 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin):
            self.flags.ca_cert_dir, text_(self.request.host))
        with self.lock:
            if not os.path.isfile(cert_file_path):
-                logger.debug('Generating certificates %s', cert_file_path)
-                # TODO: Parse subject from certificate
-                # Currently we only set CN= field for generated certificates.
-                gen_cert = subprocess.Popen(
-                    ['openssl', 'req', '-new', '-key', self.flags.ca_signing_key_file, '-subj',
-                     f'/C=/ST=/L=/O=/OU=/CN={ text_(self.request.host) }'],
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE)
-                sign_cert = subprocess.Popen(
-                    ['openssl', 'x509', '-req', '-days', '365', '-CA', self.flags.ca_cert_file, '-CAkey',
-                     self.flags.ca_key_file, '-set_serial', str(self.uid.int), '-out', cert_file_path],
-                    stdin=gen_cert.stdout,
-                    stderr=subprocess.PIPE)
-                # TODO: Ensure sign_cert success.
-                sign_cert.communicate(timeout=10)
+                self.gen_ca_signed_certificate(cert_file_path)
        return cert_file_path

    def wrap_server(self) -> None:
--- a/setup.py
+++ b/setup.py
@ -10,7 +10,7 @@
 """
 from setuptools import setup, find_packages

-VERSION = (2, 1, 2)
+VERSION = (2, 2, 0)
 __version__ = '.'.join(map(str, VERSION[0:3]))
 __description__ = '''⚡⚡⚡Fast, Lightweight, Pluggable, TLS interception capable proxy server
    focused on Network monitoring, controls & Application development, testing, debugging.'''
--- a/tests/http/test_http_proxy_tls_interception.py
+++ b/tests/http/test_http_proxy_tls_interception.py
@ -30,14 +30,18 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
    @mock.patch('ssl.wrap_socket')
    @mock.patch('ssl.create_default_context')
    @mock.patch('proxy.http.proxy.server.TcpServerConnection')
-    @mock.patch('subprocess.Popen')
+    @mock.patch('proxy.http.proxy.server.gen_public_key')
+    @mock.patch('proxy.http.proxy.server.gen_csr')
+    @mock.patch('proxy.http.proxy.server.sign_csr')
    @mock.patch('selectors.DefaultSelector')
    @mock.patch('socket.fromfd')
    def test_e2e(
            self,
            mock_fromfd: mock.Mock,
            mock_selector: mock.Mock,
-            mock_popen: mock.Mock,
+            mock_sign_csr: mock.Mock,
+            mock_gen_csr: mock.Mock,
+            mock_gen_public_key: mock.Mock,
            mock_server_conn: mock.Mock,
            mock_ssl_context: mock.Mock,
            mock_ssl_wrap: mock.Mock) -> None:
@ -46,11 +50,17 @@ class TestHttpProxyTlsInterception(unittest.TestCase):

        self.mock_fromfd = mock_fromfd
        self.mock_selector = mock_selector
-        self.mock_popen = mock_popen
+        self.mock_sign_csr = mock_sign_csr
+        self.mock_gen_csr = mock_gen_csr
+        self.mock_gen_public_key = mock_gen_public_key
        self.mock_server_conn = mock_server_conn
        self.mock_ssl_context = mock_ssl_context
        self.mock_ssl_wrap = mock_ssl_wrap

+        self.mock_sign_csr.return_value = True
+        self.mock_gen_csr.return_value = True
+        self.mock_gen_public_key.return_value = True
+
        ssl_connection = mock.MagicMock(spec=ssl.SSLSocket)
        self.mock_ssl_context.return_value.wrap_socket.return_value = ssl_connection
        self.mock_ssl_wrap.return_value = mock.MagicMock(spec=ssl.SSLSocket)
@ -118,6 +128,7 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
                fd=self._conn.fileno,
                events=selectors.EVENT_READ,
                data=None), selectors.EVENT_READ)], ]
+
        self.protocol_handler.run_once()

        # Assert our mocked plugins invocations
@ -142,8 +153,9 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
        self.assertEqual(plain_connection.setblocking.call_count, 2)
        self.mock_ssl_context.return_value.wrap_socket.assert_called_with(
            plain_connection, server_hostname=host)
-        # TODO: Assert Popen arguments, piping, success condition
-        self.assertEqual(self.mock_popen.call_count, 2)
+        self.assertEqual(self.mock_sign_csr.call_count, 1)
+        self.assertEqual(self.mock_gen_csr.call_count, 1)
+        self.assertEqual(self.mock_gen_public_key.call_count, 1)
        self.assertEqual(ssl_connection.setblocking.call_count, 1)
        self.assertEqual(
            self.mock_server_conn.return_value._conn,
--- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py
+++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py
@ -33,23 +33,33 @@ class TestHttpProxyPluginExamplesWithTlsInterception(unittest.TestCase):
    @mock.patch('ssl.wrap_socket')
    @mock.patch('ssl.create_default_context')
    @mock.patch('proxy.http.proxy.server.TcpServerConnection')
-    @mock.patch('subprocess.Popen')
+    @mock.patch('proxy.http.proxy.server.gen_public_key')
+    @mock.patch('proxy.http.proxy.server.gen_csr')
+    @mock.patch('proxy.http.proxy.server.sign_csr')
    @mock.patch('selectors.DefaultSelector')
    @mock.patch('socket.fromfd')
    def setUp(self,
              mock_fromfd: mock.Mock,
              mock_selector: mock.Mock,
-              mock_popen: mock.Mock,
+              mock_sign_csr: mock.Mock,
+              mock_gen_csr: mock.Mock,
+              mock_gen_public_key: mock.Mock,
              mock_server_conn: mock.Mock,
              mock_ssl_context: mock.Mock,
              mock_ssl_wrap: mock.Mock) -> None:
        self.mock_fromfd = mock_fromfd
        self.mock_selector = mock_selector
-        self.mock_popen = mock_popen
+        self.mock_sign_csr = mock_sign_csr
+        self.mock_gen_csr = mock_gen_csr
+        self.mock_gen_public_key = mock_gen_public_key
        self.mock_server_conn = mock_server_conn
        self.mock_ssl_context = mock_ssl_context
        self.mock_ssl_wrap = mock_ssl_wrap

+        self.mock_sign_csr.return_value = True
+        self.mock_gen_csr.return_value = True
+        self.mock_gen_public_key.return_value = True
+
        self.fileno = 10
        self._addr = ('127.0.0.1', 54382)
        self.flags = Flags(
@ -126,7 +136,10 @@ class TestHttpProxyPluginExamplesWithTlsInterception(unittest.TestCase):
        )
        self.protocol_handler.run_once()

-        self.mock_popen.assert_called()
+        self.assertEqual(self.mock_sign_csr.call_count, 1)
+        self.assertEqual(self.mock_gen_csr.call_count, 1)
+        self.assertEqual(self.mock_gen_public_key.call_count, 1)
+
        self.mock_server_conn.assert_called_once_with('uni.corn', 443)
        self.server.connect.assert_called()
        self.assertEqual(