From e7aa8a28f7efe446622024b9ae1db07448d9a7a3 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Tue, 9 Jun 2020 12:07:00 +0530 Subject: [PATCH] TLS Interception Cert Generation (#362) * Use common.pki for interception certificate generation * Fix tests * Dont use certificate fields that we dont need, it leads to certificate generation error on Ubuntu * Prepare for v2.2.0 * npm audit fix --- README.md | 7 +- dashboard/package-lock.json | 62 +++++----------- dashboard/package.json | 2 +- helper/homebrew/stable/proxy.rb | 2 +- proxy/common/version.py | 2 +- proxy/http/proxy/server.py | 74 ++++++++++++++----- setup.py | 2 +- .../http/test_http_proxy_tls_interception.py | 22 ++++-- ...ttp_proxy_plugins_with_tls_interception.py | 21 +++++- 9 files changed, 116 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 9140f9d1..1ce5e216 100644 --- a/README.md +++ b/README.md @@ -795,8 +795,7 @@ response from the server. Start `proxy.py` as: ``` -> :note: **MacOS users** also need to pass explicit CA file path -> needed for validation of peer certificates. See --ca-file flag. +[![NOTE](https://img.shields.io/static/v1?label=MacOS&message=note&color=yellow)](https://github.com/abhinavsingh/proxy.py#flags) Also provide explicit CA bundle path needed for validation of peer certificates. See `--ca-file` flag. Verify TLS interception using `curl` @@ -1327,7 +1326,7 @@ usage: pki.py [-h] [--password PASSWORD] [--private-key-path PRIVATE_KEY_PATH] [--public-key-path PUBLIC_KEY_PATH] [--subject SUBJECT] action -proxy.py v2.1.2 : PKI Utility +proxy.py v2.2.0 : PKI Utility positional arguments: action Valid actions: remove_passphrase, gen_private_key, @@ -1518,7 +1517,7 @@ usage: proxy [-h] [--backlog BACKLOG] [--basic-auth BASIC_AUTH] [--static-server-dir STATIC_SERVER_DIR] [--threadless] [--timeout TIMEOUT] [--version] -proxy.py v2.1.2 +proxy.py v2.2.0 optional arguments: -h, --help show this help message and exit diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 1b0c71a9..a28f3279 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -1382,9 +1382,9 @@ "dev": true }, "follow-redirects": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.10.0.tgz", - "integrity": "sha512-4eyLK6s6lH32nOvLLwlIOnr9zrL8Sm+OvW4pVTJNoXeGzYIkHVf+pADQi+OJ0E67hiuSLezPVPyBcIZO50TmmQ==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.11.0.tgz", + "integrity": "sha512-KZm0V+ll8PfBrKwMzdo5D13b1bur9Iq9Zd/RMmAoQQcl2PxxFml8cxXPaaPYVbV0RjNjq1CU7zIzAOqtUPudmA==", "dev": true, "requires": { "debug": "^3.0.0" @@ -1582,9 +1582,9 @@ } }, "http-proxy": { - "version": "1.18.0", - "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.0.tgz", - "integrity": "sha512-84I2iJM/n1d4Hdgc6y2+qY5mDaz2PUVjlg9znE9byl+q0uC3DeByqBGReQu5tpLK0TAqTIXScRUV+dg7+bUPpQ==", + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", "dev": true, "requires": { "eventemitter3": "^4.0.0", @@ -1593,19 +1593,19 @@ } }, "http-server": { - "version": "0.12.1", - "resolved": "https://registry.npmjs.org/http-server/-/http-server-0.12.1.tgz", - "integrity": "sha512-T0jB+7J7GJ2Vo+a4/T7P7SbQ3x2GPDnqRqQXdfEuPuUOmES/9NBxPnDm7dh1HGEeUWqUmLUNtGV63ZC5Uy3tGA==", + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-0.12.3.tgz", + "integrity": "sha512-be0dKG6pni92bRjq0kvExtj/NrrAd28/8fCXkaI/4piTwQMSDSLMhWyW0NI1V+DBI3aa1HMlQu46/HjVLfmugA==", "dev": true, "requires": { "basic-auth": "^1.0.3", - "colors": "^1.3.3", + "colors": "^1.4.0", "corser": "^2.0.1", "ecstatic": "^3.3.2", - "http-proxy": "^1.17.0", + "http-proxy": "^1.18.0", + "minimist": "^1.2.5", "opener": "^1.5.1", - "optimist": "~0.6.1", - "portfinder": "^1.0.20", + "portfinder": "^1.0.25", "secure-compare": "3.0.1", "union": "~0.5.0" } @@ -2641,30 +2641,6 @@ "pinkie-promise": "^2.0.0" } }, - "optimist": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", - "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", - "dev": true, - "requires": { - "minimist": "~0.0.1", - "wordwrap": "~0.0.2" - }, - "dependencies": { - "minimist": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", - "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=", - "dev": true - }, - "wordwrap": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", - "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", - "dev": true - } - } - }, "optionator": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", @@ -2832,9 +2808,9 @@ "dev": true }, "portfinder": { - "version": "1.0.25", - "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.25.tgz", - "integrity": "sha512-6ElJnHBbxVA1XSLgBp7G1FiCkQdlqGzuF7DswL5tcea+E8UpuvPU7beVAjjRwCioTS9ZluNbu+ZyRvgTsmqEBg==", + "version": "1.0.26", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.26.tgz", + "integrity": "sha512-Xi7mKxJHHMI3rIUrnm/jjUgwhbYMkp/XKEcZX3aG4BrumLpq3nmoQMX+ClYnDZnZ/New7IatC1no5RX0zo1vXQ==", "dev": true, "requires": { "async": "^2.6.2", @@ -2873,9 +2849,9 @@ "dev": true }, "qs": { - "version": "6.9.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.2.tgz", - "integrity": "sha512-2eQ6zajpK7HwqrY1rRtGw5IZvjgtELXzJECaEDuzDFo2jjnIXpJSimzd4qflWZq6bLLi+Zgfj5eDrAzl/lptyg==", + "version": "6.9.4", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz", + "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==", "dev": true }, "read-pkg": { diff --git a/dashboard/package.json b/dashboard/package.json index f5bd8bfe..0f63c92d 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -37,7 +37,7 @@ "eslint-plugin-node": "^10.0.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^4.0.1", - "http-server": "^0.12.1", + "http-server": "^0.12.3", "jasmine": "^3.5.0", "jasmine-ts": "^0.3.0", "jquery": "^3.5.0", diff --git a/helper/homebrew/stable/proxy.rb b/helper/homebrew/stable/proxy.rb index c13d221b..b7ca93fd 100644 --- a/helper/homebrew/stable/proxy.rb +++ b/helper/homebrew/stable/proxy.rb @@ -5,7 +5,7 @@ class Proxy < Formula Network monitoring, controls & Application development, testing, debugging." homepage "https://github.com/abhinavsingh/proxy.py" url "https://github.com/abhinavsingh/proxy.py/archive/master.zip" - version "2.0.0" + version "2.1.2" depends_on "python" diff --git a/proxy/common/version.py b/proxy/common/version.py index 913d8044..e8af2693 100644 --- a/proxy/common/version.py +++ b/proxy/common/version.py @@ -8,5 +8,5 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -VERSION = (2, 1, 2) +VERSION = (2, 2, 0) __version__ = '.'.join(map(str, VERSION[0:3])) diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index db03de93..83e77fb7 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -8,14 +8,13 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import logging import threading -import subprocess import os import ssl import socket import time import errno -import logging from typing import Optional, List, Union, Dict, cast, Any, Tuple from .plugin import HttpProxyBasePlugin @@ -28,6 +27,7 @@ from ..methods import httpMethods from ...common.types import HasFileno from ...common.constants import PROXY_AGENT_HEADER_VALUE from ...common.utils import build_http_response, text_ +from ...common.pki import gen_public_key, gen_csr, sign_csr from ...core.event import eventNames from ...core.connection import TcpServerConnection, TcpConnectionUninitializedException @@ -279,7 +279,8 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin): 'BrokenPipeError when wrapping client') return True except OSError as e: - logger.exception('OSError when wrapping client', exc_info=e) + logger.exception( + 'OSError when wrapping client', exc_info=e) return True # Update all plugin connection reference for plugin in self.plugins.values(): @@ -342,6 +343,57 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin): self.response.total_size, connection_time_ms)) + def gen_ca_signed_certificate(self, cert_file_path: str) -> None: + '''CA signing key (default) is used for generating a public key + for common_name, if one already doesn't exist. Using generated + public key a CSR request is generated, which is then signed by + CA key and secret. Again this process only happen if signed + certificate doesn't already exist. + + returns signed certificate path.''' + assert(self.request.host and self.flags.ca_cert_dir and self.flags.ca_signing_key_file and + self.flags.ca_key_file and self.flags.ca_cert_file) + public_key_path = os.path.join(self.flags.ca_cert_dir, + '{0}.{1}'.format(text_(self.request.host), 'pub')) + private_key_path = self.flags.ca_signing_key_file + private_key_password = '' + subject = '/CN={0}'.format(text_(self.request.host)) + alt_subj_names = [text_(self.request.host), ] + validity_in_days = 365 * 2 + timeout = 10 + + # Generate a public key for the common name + if not os.path.isfile(public_key_path): + logger.debug('Generating public key %s', public_key_path) + resp = gen_public_key(public_key_path=public_key_path, private_key_path=private_key_path, + private_key_password=private_key_password, subject=subject, alt_subj_names=alt_subj_names, + validity_in_days=validity_in_days, timeout=timeout) + assert(resp is True) + + csr_path = os.path.join(self.flags.ca_cert_dir, + '{0}.{1}'.format(text_(self.request.host), 'csr')) + + # Generate a CSR request for this common name + if not os.path.isfile(csr_path): + logger.debug('Generating CSR %s', csr_path) + resp = gen_csr(csr_path=csr_path, key_path=private_key_path, password=private_key_password, + crt_path=public_key_path, timeout=timeout) + assert(resp is True) + + ca_key_path = self.flags.ca_key_file + ca_key_password = '' + ca_crt_path = self.flags.ca_cert_file + serial = self.uid.int + + # Sign generated CSR + if not os.path.isfile(cert_file_path): + logger.debug('Signing CSR %s', cert_file_path) + resp = sign_csr(csr_path=csr_path, crt_path=cert_file_path, ca_key_path=ca_key_path, + ca_key_password=ca_key_password, ca_crt_path=ca_crt_path, + serial=str(serial), alt_subj_names=alt_subj_names, + validity_in_days=validity_in_days, timeout=timeout) + assert(resp is True) + @staticmethod def generated_cert_file_path(ca_cert_dir: str, host: str) -> str: return os.path.join(ca_cert_dir, '%s.pem' % host) @@ -359,21 +411,7 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin): self.flags.ca_cert_dir, text_(self.request.host)) with self.lock: if not os.path.isfile(cert_file_path): - logger.debug('Generating certificates %s', cert_file_path) - # TODO: Parse subject from certificate - # Currently we only set CN= field for generated certificates. - gen_cert = subprocess.Popen( - ['openssl', 'req', '-new', '-key', self.flags.ca_signing_key_file, '-subj', - f'/C=/ST=/L=/O=/OU=/CN={ text_(self.request.host) }'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - sign_cert = subprocess.Popen( - ['openssl', 'x509', '-req', '-days', '365', '-CA', self.flags.ca_cert_file, '-CAkey', - self.flags.ca_key_file, '-set_serial', str(self.uid.int), '-out', cert_file_path], - stdin=gen_cert.stdout, - stderr=subprocess.PIPE) - # TODO: Ensure sign_cert success. - sign_cert.communicate(timeout=10) + self.gen_ca_signed_certificate(cert_file_path) return cert_file_path def wrap_server(self) -> None: diff --git a/setup.py b/setup.py index 6c73e03f..0d2c6751 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ """ from setuptools import setup, find_packages -VERSION = (2, 1, 2) +VERSION = (2, 2, 0) __version__ = '.'.join(map(str, VERSION[0:3])) __description__ = '''⚡⚡⚡Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on Network monitoring, controls & Application development, testing, debugging.''' diff --git a/tests/http/test_http_proxy_tls_interception.py b/tests/http/test_http_proxy_tls_interception.py index 6f4d93e9..f8019d82 100644 --- a/tests/http/test_http_proxy_tls_interception.py +++ b/tests/http/test_http_proxy_tls_interception.py @@ -30,14 +30,18 @@ class TestHttpProxyTlsInterception(unittest.TestCase): @mock.patch('ssl.wrap_socket') @mock.patch('ssl.create_default_context') @mock.patch('proxy.http.proxy.server.TcpServerConnection') - @mock.patch('subprocess.Popen') + @mock.patch('proxy.http.proxy.server.gen_public_key') + @mock.patch('proxy.http.proxy.server.gen_csr') + @mock.patch('proxy.http.proxy.server.sign_csr') @mock.patch('selectors.DefaultSelector') @mock.patch('socket.fromfd') def test_e2e( self, mock_fromfd: mock.Mock, mock_selector: mock.Mock, - mock_popen: mock.Mock, + mock_sign_csr: mock.Mock, + mock_gen_csr: mock.Mock, + mock_gen_public_key: mock.Mock, mock_server_conn: mock.Mock, mock_ssl_context: mock.Mock, mock_ssl_wrap: mock.Mock) -> None: @@ -46,11 +50,17 @@ class TestHttpProxyTlsInterception(unittest.TestCase): self.mock_fromfd = mock_fromfd self.mock_selector = mock_selector - self.mock_popen = mock_popen + self.mock_sign_csr = mock_sign_csr + self.mock_gen_csr = mock_gen_csr + self.mock_gen_public_key = mock_gen_public_key self.mock_server_conn = mock_server_conn self.mock_ssl_context = mock_ssl_context self.mock_ssl_wrap = mock_ssl_wrap + self.mock_sign_csr.return_value = True + self.mock_gen_csr.return_value = True + self.mock_gen_public_key.return_value = True + ssl_connection = mock.MagicMock(spec=ssl.SSLSocket) self.mock_ssl_context.return_value.wrap_socket.return_value = ssl_connection self.mock_ssl_wrap.return_value = mock.MagicMock(spec=ssl.SSLSocket) @@ -118,6 +128,7 @@ class TestHttpProxyTlsInterception(unittest.TestCase): fd=self._conn.fileno, events=selectors.EVENT_READ, data=None), selectors.EVENT_READ)], ] + self.protocol_handler.run_once() # Assert our mocked plugins invocations @@ -142,8 +153,9 @@ class TestHttpProxyTlsInterception(unittest.TestCase): self.assertEqual(plain_connection.setblocking.call_count, 2) self.mock_ssl_context.return_value.wrap_socket.assert_called_with( plain_connection, server_hostname=host) - # TODO: Assert Popen arguments, piping, success condition - self.assertEqual(self.mock_popen.call_count, 2) + self.assertEqual(self.mock_sign_csr.call_count, 1) + self.assertEqual(self.mock_gen_csr.call_count, 1) + self.assertEqual(self.mock_gen_public_key.call_count, 1) self.assertEqual(ssl_connection.setblocking.call_count, 1) self.assertEqual( self.mock_server_conn.return_value._conn, diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py index 2976869d..0c1ec0a4 100644 --- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py +++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py @@ -33,23 +33,33 @@ class TestHttpProxyPluginExamplesWithTlsInterception(unittest.TestCase): @mock.patch('ssl.wrap_socket') @mock.patch('ssl.create_default_context') @mock.patch('proxy.http.proxy.server.TcpServerConnection') - @mock.patch('subprocess.Popen') + @mock.patch('proxy.http.proxy.server.gen_public_key') + @mock.patch('proxy.http.proxy.server.gen_csr') + @mock.patch('proxy.http.proxy.server.sign_csr') @mock.patch('selectors.DefaultSelector') @mock.patch('socket.fromfd') def setUp(self, mock_fromfd: mock.Mock, mock_selector: mock.Mock, - mock_popen: mock.Mock, + mock_sign_csr: mock.Mock, + mock_gen_csr: mock.Mock, + mock_gen_public_key: mock.Mock, mock_server_conn: mock.Mock, mock_ssl_context: mock.Mock, mock_ssl_wrap: mock.Mock) -> None: self.mock_fromfd = mock_fromfd self.mock_selector = mock_selector - self.mock_popen = mock_popen + self.mock_sign_csr = mock_sign_csr + self.mock_gen_csr = mock_gen_csr + self.mock_gen_public_key = mock_gen_public_key self.mock_server_conn = mock_server_conn self.mock_ssl_context = mock_ssl_context self.mock_ssl_wrap = mock_ssl_wrap + self.mock_sign_csr.return_value = True + self.mock_gen_csr.return_value = True + self.mock_gen_public_key.return_value = True + self.fileno = 10 self._addr = ('127.0.0.1', 54382) self.flags = Flags( @@ -126,7 +136,10 @@ class TestHttpProxyPluginExamplesWithTlsInterception(unittest.TestCase): ) self.protocol_handler.run_once() - self.mock_popen.assert_called() + self.assertEqual(self.mock_sign_csr.call_count, 1) + self.assertEqual(self.mock_gen_csr.call_count, 1) + self.assertEqual(self.mock_gen_public_key.call_count, 1) + self.mock_server_conn.assert_called_once_with('uni.corn', 443) self.server.connect.assert_called() self.assertEqual(