Refactor plugin base classes for plugin specific flags (#388)

* Update to latest code signing recommendations

* Move HttpProtocolHandlerPlugin into separate file

* Dont add subject attributes if not provided by upstream. Also handle subprocess.TimeoutExpired raised during certificate generation.  Instead of retries, we simply close the connection on timeout

* Remove plugin specific flag initialization methods for now

menubar/proxy.py.xcodeproj/project.pbxproj

@@ -198,7 +198,7 @@
 			isa = PBXProject;
 			attributes = {
 				LastSwiftUpdateCheck = 1120;
-				LastUpgradeCheck = 1120;
+				LastUpgradeCheck = 1150;
 				ORGANIZATIONNAME = "Abhinav Singh";
 				TargetAttributes = {
 					AD1F92A2238864240088A917 = {
@@ -432,6 +432,7 @@
 			buildSettings = {
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CODE_SIGN_ENTITLEMENTS = proxy.py/proxy_py.entitlements;
+				CODE_SIGN_IDENTITY = "-";
 				CODE_SIGN_STYLE = Automatic;
 				COMBINE_HIDPI_IMAGES = YES;
 				DEVELOPMENT_ASSET_PATHS = "\"proxy.py/Preview Content\"";
@@ -455,6 +456,7 @@
 			buildSettings = {
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CODE_SIGN_ENTITLEMENTS = proxy.py/proxy_py.entitlements;
+				CODE_SIGN_IDENTITY = "-";
 				CODE_SIGN_STYLE = Automatic;
 				COMBINE_HIDPI_IMAGES = YES;
 				DEVELOPMENT_ASSET_PATHS = "\"proxy.py/Preview Content\"";

menubar/proxy.py.xcodeproj/xcuserdata/abhinavsingh.xcuserdatad/xcschemes/xcschememanagement.plist

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>SchemeUserState</key>
+	<dict>
+		<key>proxy.py.xcscheme_^#shared#^_</key>
+		<dict>
+			<key>orderHint</key>
+			<integer>0</integer>
+		</dict>
+	</dict>
+</dict>
+</plist>

menubar/proxy.py/AppDelegate.swift

@@ -3,7 +3,8 @@
 //  proxy.py
 //
 //  Created by Abhinav Singh on 11/22/19.
-//  Copyright © 2019 Abhinav Singh. All rights reserved.
+//  Copyright © 2013-present by Abhinav Singh and contributors.
+//  All rights reserved.
 //
 
 import Cocoa
@@ -41,3 +42,9 @@ class AppDelegate: NSObject, NSApplicationDelegate {
         statusItem.menu = menu
     }
 }
+
+struct AppDelegate_Previews: PreviewProvider {
+    static var previews: some View {
+        /*@START_MENU_TOKEN@*/Text("Hello, World!")/*@END_MENU_TOKEN@*/
+    }
+}

menubar/proxy.py/ContentView.swift

@@ -3,7 +3,8 @@
 //  proxy.py
 //
 //  Created by Abhinav Singh on 11/22/19.
-//  Copyright © 2019 Abhinav Singh. All rights reserved.
+//  Copyright © 2013-present by Abhinav Singh and contributors.
+//  All rights reserved.
 //
 
 import SwiftUI

proxy/common/flags.py

@@ -145,7 +145,9 @@ class Flags:
                 'A future version of pip will drop support for Python 2.7.')
             sys.exit(1)
 
+        # Initialize core flags.
         parser = Flags.init_parser()
+        # Parse flags
         args = parser.parse_args(input_args)
 
         # Print version and exit
@@ -159,6 +161,7 @@ class Flags:
         # Setup limits
         Flags.set_open_file_limit(args.open_file_limit)
 
+        # Prepare list of plugins to load based upon --enable-* and --disable-* flags
         default_plugins: List[Tuple[str, bool]] = []
         if args.enable_dashboard:
             default_plugins.append((PLUGIN_WEB_SERVER, True))
@@ -179,6 +182,7 @@ class Flags:
         if args.pac_file is not None:
             default_plugins.append((PLUGIN_PAC_FILE, True))
 
+        # Load default plugins along with user provided --plugins
         plugins = Flags.load_plugins(
             bytes_(
                 '%s,%s' %

proxy/http/handler.py

@@ -15,9 +15,11 @@ import time
 import contextlib
 import errno
 import logging
-from abc import ABC, abstractmethod
+
 from typing import Tuple, List, Union, Optional, Generator, Dict
 from uuid import UUID
+
+from .plugin import HttpProtocolHandlerPlugin
 from .parser import HttpParser, httpParserStates, httpParserTypes
 from .exception import HttpProtocolException
 
@@ -30,83 +32,6 @@ from ..core.connection import TcpClientConnection
 logger = logging.getLogger(__name__)
 
 
-class HttpProtocolHandlerPlugin(ABC):
-    """Base HttpProtocolHandler Plugin class.
-
-    NOTE: This is an internal plugin and in most cases only useful for core contributors.
-    If you are looking for proxy server plugins see `<proxy.HttpProxyBasePlugin>`.
-
-    Implements various lifecycle events for an accepted client connection.
-    Following events are of interest:
-
-    1. Client Connection Accepted
-       A new plugin instance is created per accepted client connection.
-       Add your logic within __init__ constructor for any per connection setup.
-    2. Client Request Chunk Received
-       on_client_data is called for every chunk of data sent by the client.
-    3. Client Request Complete
-       on_request_complete is called once client request has completed.
-    4. Server Response Chunk Received
-       on_response_chunk is called for every chunk received from the server.
-    5. Client Connection Closed
-       Add your logic within `on_client_connection_close` for any per connection teardown.
-    """
-
-    def __init__(
-            self,
-            uid: UUID,
-            flags: Flags,
-            client: TcpClientConnection,
-            request: HttpParser,
-            event_queue: EventQueue):
-        self.uid: UUID = uid
-        self.flags: Flags = flags
-        self.client: TcpClientConnection = client
-        self.request: HttpParser = request
-        self.event_queue = event_queue
-        super().__init__()
-
-    def name(self) -> str:
-        """A unique name for your plugin.
-
-        Defaults to name of the class. This helps plugin developers to directly
-        access a specific plugin by its name."""
-        return self.__class__.__name__
-
-    @abstractmethod
-    def get_descriptors(
-            self) -> Tuple[List[socket.socket], List[socket.socket]]:
-        return [], []  # pragma: no cover
-
-    @abstractmethod
-    def write_to_descriptors(self, w: List[Union[int, HasFileno]]) -> bool:
-        return False  # pragma: no cover
-
-    @abstractmethod
-    def read_from_descriptors(self, r: List[Union[int, HasFileno]]) -> bool:
-        return False  # pragma: no cover
-
-    @abstractmethod
-    def on_client_data(self, raw: memoryview) -> Optional[memoryview]:
-        return raw  # pragma: no cover
-
-    @abstractmethod
-    def on_request_complete(self) -> Union[socket.socket, bool]:
-        """Called right after client request parser has reached COMPLETE state."""
-        return False  # pragma: no cover
-
-    @abstractmethod
-    def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]:
-        """Handle data chunks as received from the server.
-
-        Return optionally modified chunk to return back to client."""
-        return chunk  # pragma: no cover
-
-    @abstractmethod
-    def on_client_connection_close(self) -> None:
-        pass  # pragma: no cover
-
-
 class HttpProtocolHandler(ThreadlessWork):
     """HTTP, HTTPS, HTTP2, WebSockets protocol handler.
 

proxy/http/plugin.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import socket
+
+from abc import ABC, abstractmethod
+from uuid import UUID
+from typing import Tuple, List, Union, Optional
+
+from .parser import HttpParser
+
+from ..common.flags import Flags
+from ..common.types import HasFileno
+from ..core.event import EventQueue
+from ..core.connection import TcpClientConnection
+
+
+class HttpProtocolHandlerPlugin(ABC):
+    """Base HttpProtocolHandler Plugin class.
+
+    NOTE: This is an internal plugin and in most cases only useful for core contributors.
+    If you are looking for proxy server plugins see `<proxy.HttpProxyBasePlugin>`.
+
+    Implements various lifecycle events for an accepted client connection.
+    Following events are of interest:
+
+    1. Client Connection Accepted
+       A new plugin instance is created per accepted client connection.
+       Add your logic within __init__ constructor for any per connection setup.
+    2. Client Request Chunk Received
+       on_client_data is called for every chunk of data sent by the client.
+    3. Client Request Complete
+       on_request_complete is called once client request has completed.
+    4. Server Response Chunk Received
+       on_response_chunk is called for every chunk received from the server.
+    5. Client Connection Closed
+       Add your logic within `on_client_connection_close` for any per connection teardown.
+    """
+
+    def __init__(
+            self,
+            uid: UUID,
+            flags: Flags,
+            client: TcpClientConnection,
+            request: HttpParser,
+            event_queue: EventQueue):
+        self.uid: UUID = uid
+        self.flags: Flags = flags
+        self.client: TcpClientConnection = client
+        self.request: HttpParser = request
+        self.event_queue = event_queue
+        super().__init__()
+
+    def name(self) -> str:
+        """A unique name for your plugin.
+
+        Defaults to name of the class. This helps plugin developers to directly
+        access a specific plugin by its name."""
+        return self.__class__.__name__
+
+    @abstractmethod
+    def get_descriptors(
+            self) -> Tuple[List[socket.socket], List[socket.socket]]:
+        return [], []  # pragma: no cover
+
+    @abstractmethod
+    def write_to_descriptors(self, w: List[Union[int, HasFileno]]) -> bool:
+        return False  # pragma: no cover
+
+    @abstractmethod
+    def read_from_descriptors(self, r: List[Union[int, HasFileno]]) -> bool:
+        return False  # pragma: no cover
+
+    @abstractmethod
+    def on_client_data(self, raw: memoryview) -> Optional[memoryview]:
+        return raw  # pragma: no cover
+
+    @abstractmethod
+    def on_request_complete(self) -> Union[socket.socket, bool]:
+        """Called right after client request parser has reached COMPLETE state."""
+        return False  # pragma: no cover
+
+    @abstractmethod
+    def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]:
+        """Handle data chunks as received from the server.
+
+        Return optionally modified chunk to return back to client."""
+        return chunk  # pragma: no cover
+
+    @abstractmethod
+    def on_client_connection_close(self) -> None:
+        pass  # pragma: no cover

proxy/http/proxy/server.py

@@ -10,6 +10,7 @@
 """
 import logging
 import threading
+import subprocess
 import os
 import ssl
 import socket
@@ -18,7 +19,7 @@ import errno
 from typing import Optional, List, Union, Dict, cast, Any, Tuple
 
 from .plugin import HttpProxyBasePlugin
-from ..handler import HttpProtocolHandlerPlugin
+from ..plugin import HttpProtocolHandlerPlugin
 from ..exception import HttpProtocolException, ProxyConnectionFailed, ProxyAuthenticationFailed
 from ..codes import httpStatusCodes
 from ..parser import HttpParser, httpParserStates, httpParserTypes
@@ -287,6 +288,9 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin):
                     # wrap_client also flushes client data before wrapping
                     # sending to client can raise, handle expected exceptions
                     self.wrap_client()
+                except subprocess.TimeoutExpired as e:  # Popen communicate timeout
+                    logger.exception('TimeoutExpired during certificate generation', exc_info=e)
+                    return True
                 except BrokenPipeError:
                     logger.error(
                         'BrokenPipeError when wrapping client')
@@ -372,13 +376,19 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin):
                                        '{0}.{1}'.format(text_(self.request.host), 'pub'))
         private_key_path = self.flags.ca_signing_key_file
         private_key_password = ''
-        subject = '/CN={0}/C={1}/ST={2}/L={3}/O={4}/OU={5}'.format(
+        # Build certificate subject
-            upstream_subject.get('commonName', text_(self.request.host)),
+        keys = {
-            upstream_subject.get('countryName', 'NA'),
+            'CN': 'commonName',
-            upstream_subject.get('stateOrProvinceName', 'Unavailable'),
+            'C': 'countryName',
-            upstream_subject.get('localityName', 'Unavailable'),
+            'ST': 'stateOrProvinceName',
-            upstream_subject.get('organizationName', 'Unavailable'),
+            'L': 'localityName',
-            upstream_subject.get('organizationalUnitName', 'Unavailable'))
+            'O': 'organizationName',
+            'OU': 'organizationalUnitName',
+        }
+        subject = ''
+        for key in keys:
+            if upstream_subject.get(keys[key], None):
+                subject += '/{0}={1}'.format(key, upstream_subject.get(keys[key]))
         alt_subj_names = [text_(self.request.host), ]
         validity_in_days = 365 * 2
         timeout = 10
@@ -458,9 +468,10 @@ class HttpProxyPlugin(HttpProtocolHandlerPlugin):
         self.client._conn = ssl.wrap_socket(
             self.client.connection,
             server_side=True,
+            # ca_certs=self.flags.ca_cert_file,
             certfile=generated_cert,
             keyfile=self.flags.ca_signing_key_file,
-            ssl_version=ssl.PROTOCOL_TLSv1_2)
+            ssl_version=ssl.PROTOCOL_TLS)
         self.client.connection.setblocking(False)
         logger.debug(
             'TLS interception using %s', generated_cert)

proxy/http/server/web.py

@@ -23,7 +23,7 @@ from ..exception import HttpProtocolException
 from ..websocket import WebsocketFrame, websocketOpcodes
 from ..codes import httpStatusCodes
 from ..parser import HttpParser, httpParserStates, httpParserTypes
-from ..handler import HttpProtocolHandlerPlugin
+from ..plugin import HttpProtocolHandlerPlugin
 
 from ...common.utils import bytes_, text_, build_http_response, build_websocket_handshake_response
 from ...common.constants import PROXY_AGENT_HEADER_VALUE

tests/http/test_http_proxy_tls_interception.py

@@ -169,7 +169,7 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
             keyfile=self.flags.ca_signing_key_file,
             certfile=HttpProxyPlugin.generated_cert_file_path(
                 self.flags.ca_cert_dir, host),
-            ssl_version=ssl.PROTOCOL_TLSv1_2
+            ssl_version=ssl.PROTOCOL_TLS
         )
         self.assertEqual(self._conn.setblocking.call_count, 2)
         self.assertEqual(