From ab1198268c9e845eccaea56b20be32c39b14c2fe Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Wed, 25 Mar 2020 13:30:19 +0530 Subject: [PATCH] Add flag to specify custom system CA Path (#321) * Fixes #320 * Update README and add codecov.yml * Update codecov.yml --- README.md | 21 ++++++++++++++----- proxy/common/constants.py | 1 + proxy/common/flags.py | 16 +++++++++++++- proxy/http/proxy/server.py | 14 ++++++------- tests/codecov.yml | 12 +++++++++++ .../http/test_http_proxy_tls_interception.py | 4 ++-- 6 files changed, 53 insertions(+), 15 deletions(-) create mode 100644 tests/codecov.yml diff --git a/README.md b/README.md index 7984b773..9140f9d1 100644 --- a/README.md +++ b/README.md @@ -777,14 +777,14 @@ TLS Interception ================= By default, `proxy.py` will not decrypt `https` traffic between client and server. -To enable TLS interception first generate CA certificates: +To enable TLS interception first generate root CA certificates: -``` -make ca-certificates +```bash +❯ make ca-certificates ``` Lets also enable `CacheResponsePlugin` so that we can verify decrypted -response from the server. Start `proxy.py` as: +response from the server. Start `proxy.py` as: ```bash ❯ proxy \ @@ -794,7 +794,16 @@ response from the server. Start `proxy.py` as: --ca-signing-key-file ca-signing-key.pem ``` -Verify using `curl -v -x localhost:8899 --cacert ca-cert.pem https://httpbin.org/get` + +> :note: **MacOS users** also need to pass explicit CA file path +> needed for validation of peer certificates. See --ca-file flag. + + +Verify TLS interception using `curl` + +```bash +❯ curl -v -x localhost:8899 --cacert ca-cert.pem https://httpbin.org/get +``` ```bash * issuer: C=US; ST=CA; L=SanFrancisco; O=proxy.py; OU=CA; CN=Proxy PY CA; emailAddress=proxyca@mailserver.com @@ -1530,6 +1539,8 @@ optional arguments: Default: None. Signing certificate to use for signing dynamically generated HTTPS certificates. If used, must also pass --ca-key-file and --ca-signing-key-file + --ca-file CA_FILE Default: None. Provide path to custom CA file for peer + certificate validation. Specially useful on MacOS. --ca-signing-key-file CA_SIGNING_KEY_FILE Default: None. CA signing key to use for dynamic generation of HTTPS certificates. If used, must also diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 72efbce5..a4a32bf1 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -44,6 +44,7 @@ DEFAULT_CA_CERT_FILE = None DEFAULT_CA_KEY_FILE = None DEFAULT_CA_SIGNING_KEY_FILE = None DEFAULT_CERT_FILE = None +DEFAULT_CA_FILE = None DEFAULT_CLIENT_RECVBUF_SIZE = DEFAULT_BUFFER_SIZE DEFAULT_DEVTOOLS_WS_PATH = b'/devtools' DEFAULT_DISABLE_HEADERS: List[bytes] = [] diff --git a/proxy/common/flags.py b/proxy/common/flags.py index 30aaf6f1..5b74173c 100644 --- a/proxy/common/flags.py +++ b/proxy/common/flags.py @@ -28,7 +28,7 @@ from .utils import text_, bytes_ from .constants import DEFAULT_LOG_LEVEL, DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_BACKLOG, DEFAULT_BASIC_AUTH from .constants import DEFAULT_TIMEOUT, DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HTTP_PROXY, DEFAULT_DISABLE_HEADERS from .constants import DEFAULT_ENABLE_STATIC_SERVER, DEFAULT_ENABLE_EVENTS, DEFAULT_ENABLE_DEVTOOLS -from .constants import DEFAULT_ENABLE_WEB_SERVER, DEFAULT_THREADLESS, DEFAULT_CERT_FILE, DEFAULT_KEY_FILE +from .constants import DEFAULT_ENABLE_WEB_SERVER, DEFAULT_THREADLESS, DEFAULT_CERT_FILE, DEFAULT_KEY_FILE, DEFAULT_CA_FILE from .constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE from .constants import DEFAULT_PAC_FILE_URL_PATH, DEFAULT_PAC_FILE, DEFAULT_PLUGINS, DEFAULT_PID_FILE, DEFAULT_PORT from .constants import DEFAULT_NUM_WORKERS, DEFAULT_VERSION, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_IPV6_HOSTNAME @@ -67,6 +67,7 @@ class Flags: ca_key_file: Optional[str] = None, ca_cert_file: Optional[str] = None, ca_signing_key_file: Optional[str] = None, + ca_file: Optional[str] = None, num_workers: int = 0, hostname: Union[ipaddress.IPv4Address, ipaddress.IPv6Address] = DEFAULT_IPV6_HOSTNAME, @@ -98,6 +99,7 @@ class Flags: self.ca_key_file: Optional[str] = ca_key_file self.ca_cert_file: Optional[str] = ca_cert_file self.ca_signing_key_file: Optional[str] = ca_signing_key_file + self.ca_file = ca_file self.num_workers: int = num_workers if num_workers > 0 else multiprocessing.cpu_count() self.hostname: Union[ipaddress.IPv4Address, ipaddress.IPv6Address] = hostname @@ -223,6 +225,11 @@ class Flags: opts.get( 'ca_signing_key_file', args.ca_signing_key_file)), + ca_file=cast( + Optional[str], + opts.get( + 'ca_file', + args.ca_file)), hostname=cast(Union[ipaddress.IPv4Address, ipaddress.IPv6Address], opts.get('hostname', ipaddress.ip_address(args.hostname))), @@ -308,6 +315,13 @@ class Flags: help='Default: None. Signing certificate to use for signing dynamically generated ' 'HTTPS certificates. If used, must also pass --ca-key-file and --ca-signing-key-file' ) + parser.add_argument( + '--ca-file', + type=str, + default=DEFAULT_CA_FILE, + help='Default: None. Provide path to custom CA file for peer certificate validation. ' + 'Specially useful on MacOS.' + ) parser.add_argument( '--ca-signing-key-file', type=str, diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 500aec47..76624145 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -89,13 +89,13 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin): logger.debug('Server is write ready, flushing buffer') try: self.server.flush() - except OSError: - logger.error('OSError when flushing buffer to server') - return True except BrokenPipeError: logger.error( 'BrokenPipeError when flushing buffer for server') return True + except OSError: + logger.error('OSError when flushing buffer to server') + return True return False def read_from_descriptors(self, r: List[Union[int, HasFileno]]) -> bool: @@ -274,13 +274,13 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin): # wrap_client also flushes client data before wrapping # sending to client can raise, handle expected exceptions self.wrap_client() - except OSError: - logger.error('OSError when wrapping client') - return True except BrokenPipeError: logger.error( 'BrokenPipeError when wrapping client') return True + except OSError: + logger.error('OSError when wrapping client') + return True # Update all plugin connection reference for plugin in self.plugins.values(): plugin.client._conn = self.client.connection @@ -380,7 +380,7 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin): assert self.server is not None assert isinstance(self.server.connection, socket.socket) ctx = ssl.create_default_context( - ssl.Purpose.SERVER_AUTH) + ssl.Purpose.SERVER_AUTH, cafile=self.flags.ca_file) ctx.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 self.server.connection.setblocking(True) self.server._conn = ctx.wrap_socket( diff --git a/tests/codecov.yml b/tests/codecov.yml new file mode 100644 index 00000000..f393de9c --- /dev/null +++ b/tests/codecov.yml @@ -0,0 +1,12 @@ +codecov: + require_ci_to_pass: yes + notify: + wait_for_ci: yes +coverage: + status: + project: + default: + threshold: 1% + patch: + default: + threshold: 1% diff --git a/tests/http/test_http_proxy_tls_interception.py b/tests/http/test_http_proxy_tls_interception.py index ff01a7c8..6f4d93e9 100644 --- a/tests/http/test_http_proxy_tls_interception.py +++ b/tests/http/test_http_proxy_tls_interception.py @@ -69,7 +69,7 @@ class TestHttpProxyTlsInterception(unittest.TestCase): self.flags = Flags( ca_cert_file='ca-cert.pem', ca_key_file='ca-key.pem', - ca_signing_key_file='ca-signing-key.pem', + ca_signing_key_file='ca-signing-key.pem' ) self.plugin = mock.MagicMock() self.proxy_plugin = mock.MagicMock() @@ -135,7 +135,7 @@ class TestHttpProxyTlsInterception(unittest.TestCase): self.mock_server_conn.return_value.connection.setblocking.assert_called_with( False) - self.mock_ssl_context.assert_called_with(ssl.Purpose.SERVER_AUTH) + self.mock_ssl_context.assert_called_with(ssl.Purpose.SERVER_AUTH, cafile=None) # self.assertEqual(self.mock_ssl_context.return_value.options, # ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | # ssl.OP_NO_TLSv1_1)