remove old code
This commit is contained in:
Maximilian Hils
parent
08655cb956
commit
3873e08339
|
|
@ -1,37 +0,0 @@
|
|||
# This script makes mitmproxy switch to passthrough mode for all HTTP
|
||||
# responses with "Connection: Upgrade" header. This is useful to make
|
||||
# WebSockets work in untrusted environments.
|
||||
#
|
||||
# Note: Chrome (and possibly other browsers), when explicitly configured
|
||||
# to use a proxy (i.e. mitmproxy's regular mode), send a CONNECT request
|
||||
# to the proxy before they initiate the websocket connection.
|
||||
# To make WebSockets work in these cases, supply
|
||||
# `--ignore :80$` as an additional parameter.
|
||||
# (see http://mitmproxy.org/doc/features/passthrough.html)
|
||||
|
||||
import netlib.http.semantics
|
||||
|
||||
from libmproxy.protocol.tcp import TCPHandler
|
||||
from libmproxy.protocol import KILL
|
||||
from libmproxy.script import concurrent
|
||||
|
||||
|
||||
def start(context, argv):
|
||||
netlib.http.semantics.Request._headers_to_strip_off.remove("Connection")
|
||||
netlib.http.semantics.Request._headers_to_strip_off.remove("Upgrade")
|
||||
|
||||
|
||||
def done(context):
|
||||
netlib.http.semantics.Request._headers_to_strip_off.append("Connection")
|
||||
netlib.http.semantics.Request._headers_to_strip_off.append("Upgrade")
|
||||
|
||||
|
||||
@concurrent
|
||||
def response(context, flow):
|
||||
value = flow.response.headers.get_first("Connection", None)
|
||||
if value and value.upper() == "UPGRADE":
|
||||
# We need to send the response manually now...
|
||||
flow.client_conn.send(flow.client_conn.protocol.assemble(flow.response))
|
||||
# ...and then delegate to tcp passthrough.
|
||||
TCPHandler(flow.live.c, log=False).handle_messages()
|
||||
flow.reply(KILL)
|
||||
|
|
@ -8,15 +8,16 @@ import Cookie
|
|||
import cookielib
|
||||
import os
|
||||
import re
|
||||
from libmproxy.protocol.http import HTTPFlow
|
||||
from libmproxy.protocol2.http_replay import RequestReplayThread
|
||||
|
||||
from netlib import odict, wsgi, tcp
|
||||
from netlib import odict, wsgi
|
||||
from netlib.http.semantics import CONTENT_MISSING
|
||||
import netlib.http
|
||||
|
||||
from . import controller, protocol, tnetstring, filt, script, version
|
||||
from .onboarding import app
|
||||
from .protocol import http, handle
|
||||
from .protocol import http
|
||||
from .proxy.config import HostMatcher
|
||||
from .proxy.connection import ClientConnection, ServerConnection
|
||||
import urlparse
|
||||
|
|
@ -1090,7 +1091,7 @@ class FlowReader:
|
|||
"Incompatible serialized data version: %s" % v
|
||||
)
|
||||
off = self.fo.tell()
|
||||
yield handle.protocols[data["type"]]["flow"].from_state(data)
|
||||
yield HTTPFlow.from_state(data)
|
||||
except ValueError as v:
|
||||
# Error is due to EOF
|
||||
if self.fo.tell() == off and self.fo.read() == '':
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
from __future__ import absolute_import
|
||||
from . import http, tcp
|
||||
|
||||
protocols = {
|
||||
'http': dict(handler=http.HTTPHandler, flow=http.HTTPFlow),
|
||||
'tcp': dict(handler=tcp.TCPHandler)
|
||||
}
|
||||
|
||||
|
||||
def protocol_handler(protocol):
|
||||
"""
|
||||
@type protocol: str
|
||||
@returns: libmproxy.protocol.primitives.ProtocolHandler
|
||||
"""
|
||||
if protocol in protocols:
|
||||
return protocols[protocol]["handler"]
|
||||
|
||||
raise NotImplementedError(
|
||||
"Unknown Protocol: %s" %
|
||||
protocol) # pragma: nocover
|
||||
|
|
@ -1,62 +1,9 @@
|
|||
from __future__ import absolute_import
|
||||
import Cookie
|
||||
import copy
|
||||
import threading
|
||||
import time
|
||||
import urllib
|
||||
import urlparse
|
||||
from email.utils import parsedate_tz, formatdate, mktime_tz
|
||||
|
||||
import netlib
|
||||
from netlib import http, tcp, odict, utils, encoding
|
||||
from netlib.http import cookies, http1, http2
|
||||
from netlib.http.http1 import HTTP1Protocol
|
||||
from netlib.http.semantics import CONTENT_MISSING
|
||||
|
||||
from .tcp import TCPHandler
|
||||
from .primitives import KILL, ProtocolHandler, Flow, Error
|
||||
from ..proxy.connection import ServerConnection
|
||||
from .. import utils, controller, stateobject, proxy
|
||||
from .primitives import Flow
|
||||
|
||||
from .http_wrappers import decoded, HTTPRequest, HTTPResponse
|
||||
|
||||
|
||||
class KillSignal(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def send_connect_request(conn, host, port, update_state=True):
|
||||
upstream_request = HTTPRequest(
|
||||
"authority",
|
||||
"CONNECT",
|
||||
None,
|
||||
host,
|
||||
port,
|
||||
None,
|
||||
(1, 1),
|
||||
odict.ODictCaseless(),
|
||||
""
|
||||
)
|
||||
|
||||
# we currently only support HTTP/1 CONNECT requests
|
||||
protocol = http1.HTTP1Protocol(conn)
|
||||
|
||||
conn.send(protocol.assemble(upstream_request))
|
||||
resp = HTTPResponse.from_protocol(protocol, upstream_request.method)
|
||||
if resp.status_code != 200:
|
||||
raise proxy.ProxyError(resp.status_code,
|
||||
"Cannot establish SSL " +
|
||||
"connection with upstream proxy: \r\n" +
|
||||
repr(resp))
|
||||
if update_state:
|
||||
conn.state.append(("http", {
|
||||
"state": "connect",
|
||||
"host": host,
|
||||
"port": port}
|
||||
))
|
||||
return resp
|
||||
|
||||
|
||||
class HTTPFlow(Flow):
|
||||
"""
|
||||
A HTTPFlow is a collection of objects representing a single HTTP
|
||||
|
|
@ -143,556 +90,3 @@ class HTTPFlow(Flow):
|
|||
if self.response:
|
||||
c += self.response.replace(pattern, repl, *args, **kwargs)
|
||||
return c
|
||||
|
||||
|
||||
class HTTPHandler(ProtocolHandler):
|
||||
"""
|
||||
HTTPHandler implements mitmproxys understanding of the HTTP protocol.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, c):
|
||||
super(HTTPHandler, self).__init__(c)
|
||||
self.expected_form_in = c.config.mode.http_form_in
|
||||
self.expected_form_out = c.config.mode.http_form_out
|
||||
self.skip_authentication = False
|
||||
|
||||
def handle_messages(self):
|
||||
while self.handle_flow():
|
||||
pass
|
||||
|
||||
def get_response_from_server(self, flow):
|
||||
self.c.establish_server_connection()
|
||||
|
||||
for attempt in (0, 1):
|
||||
try:
|
||||
if not self.c.server_conn.protocol:
|
||||
# instantiate new protocol if connection does not have one yet
|
||||
# TODO: select correct protocol based on ALPN (?)
|
||||
self.c.server_conn.protocol = http1.HTTP1Protocol(self.c.server_conn)
|
||||
# self.c.server_conn.protocol = http2.HTTP2Protocol(self.c.server_conn)
|
||||
# self.c.server_conn.protocol.perform_connection_preface()
|
||||
|
||||
self.c.server_conn.send(self.c.server_conn.protocol.assemble(flow.request))
|
||||
|
||||
# Only get the headers at first...
|
||||
flow.response = HTTPResponse.from_protocol(
|
||||
self.c.server_conn.protocol,
|
||||
flow.request.method,
|
||||
body_size_limit=self.c.config.body_size_limit,
|
||||
include_body=False,
|
||||
)
|
||||
break
|
||||
except (tcp.NetLibError, http.HttpErrorConnClosed) as v:
|
||||
self.c.log(
|
||||
"error in server communication: %s" % repr(v),
|
||||
level="debug"
|
||||
)
|
||||
if attempt == 0:
|
||||
# In any case, we try to reconnect at least once. This is
|
||||
# necessary because it might be possible that we already
|
||||
# initiated an upstream connection after clientconnect that
|
||||
# has already been expired, e.g consider the following event
|
||||
# log:
|
||||
# > clientconnect (transparent mode destination known)
|
||||
# > serverconnect
|
||||
# > read n% of large request
|
||||
# > server detects timeout, disconnects
|
||||
# > read (100-n)% of large request
|
||||
# > send large request upstream
|
||||
self.c.server_reconnect()
|
||||
else:
|
||||
raise
|
||||
|
||||
# call the appropriate script hook - this is an opportunity for an
|
||||
# inline script to set flow.stream = True
|
||||
flow = self.c.channel.ask("responseheaders", flow)
|
||||
if flow is None or flow == KILL:
|
||||
raise KillSignal()
|
||||
else:
|
||||
# now get the rest of the request body, if body still needs to be
|
||||
# read but not streaming this response
|
||||
if flow.response.stream:
|
||||
flow.response.content = CONTENT_MISSING
|
||||
else:
|
||||
if isinstance(self.c.server_conn.protocol, http1.HTTP1Protocol):
|
||||
# streaming is only supported with HTTP/1 at the moment
|
||||
flow.response.content = self.c.server_conn.protocol.read_http_body(
|
||||
flow.response.headers,
|
||||
self.c.config.body_size_limit,
|
||||
flow.request.method,
|
||||
flow.response.code,
|
||||
False
|
||||
)
|
||||
flow.response.timestamp_end = utils.timestamp()
|
||||
|
||||
def handle_flow(self):
|
||||
flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live)
|
||||
|
||||
try:
|
||||
try:
|
||||
if not flow.client_conn.protocol:
|
||||
# instantiate new protocol if connection does not have one yet
|
||||
# the first request might be a CONNECT - which is currently only supported with HTTP/1
|
||||
flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn)
|
||||
|
||||
req = HTTPRequest.from_protocol(
|
||||
flow.client_conn.protocol,
|
||||
body_size_limit=self.c.config.body_size_limit
|
||||
)
|
||||
except tcp.NetLibError:
|
||||
# don't throw an error for disconnects that happen
|
||||
# before/between requests.
|
||||
return False
|
||||
|
||||
self.c.log(
|
||||
"request",
|
||||
"debug",
|
||||
[repr(req)]
|
||||
)
|
||||
ret = self.process_request(flow, req)
|
||||
if ret:
|
||||
# instantiate new protocol if connection does not have one yet
|
||||
# TODO: select correct protocol based on ALPN (?)
|
||||
flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn)
|
||||
# flow.client_conn.protocol = http2.HTTP2Protocol(self.c.client_conn, is_server=True)
|
||||
if ret is not None:
|
||||
return ret
|
||||
|
||||
# Be careful NOT to assign the request to the flow before
|
||||
# process_request completes. This is because the call can raise an
|
||||
# exception. If the request object is already attached, this results
|
||||
# in an Error object that has an attached request that has not been
|
||||
# sent through to the Master.
|
||||
flow.request = req
|
||||
request_reply = self.c.channel.ask("request", flow)
|
||||
if request_reply is None or request_reply == KILL:
|
||||
raise KillSignal()
|
||||
|
||||
# The inline script may have changed request.host
|
||||
self.process_server_address(flow)
|
||||
|
||||
if isinstance(request_reply, HTTPResponse):
|
||||
flow.response = request_reply
|
||||
else:
|
||||
self.get_response_from_server(flow)
|
||||
|
||||
# no further manipulation of self.c.server_conn beyond this point
|
||||
# we can safely set it as the final attribute value here.
|
||||
flow.server_conn = self.c.server_conn
|
||||
|
||||
self.c.log(
|
||||
"response",
|
||||
"debug",
|
||||
[repr(flow.response)]
|
||||
)
|
||||
response_reply = self.c.channel.ask("response", flow)
|
||||
if response_reply is None or response_reply == KILL:
|
||||
raise KillSignal()
|
||||
|
||||
self.send_response_to_client(flow)
|
||||
|
||||
if self.check_close_connection(flow):
|
||||
return False
|
||||
|
||||
# We sent a CONNECT request to an upstream proxy.
|
||||
if flow.request.form_in == "authority" and flow.response.code == 200:
|
||||
# TODO: Possibly add headers (memory consumption/usefulness
|
||||
# tradeoff) Make sure to add state info before the actual
|
||||
# processing of the CONNECT request happens. During an SSL
|
||||
# upgrade, we may receive an SNI indication from the client,
|
||||
# which resets the upstream connection. If this is the case, we
|
||||
# must already re-issue the CONNECT request at this point.
|
||||
self.c.server_conn.state.append(
|
||||
(
|
||||
"http", {
|
||||
"state": "connect",
|
||||
"host": flow.request.host,
|
||||
"port": flow.request.port
|
||||
}
|
||||
)
|
||||
)
|
||||
if not self.process_connect_request(
|
||||
(flow.request.host, flow.request.port)):
|
||||
return False
|
||||
|
||||
# If the user has changed the target server on this connection,
|
||||
# restore the original target server
|
||||
flow.live.restore_server()
|
||||
|
||||
return True # Next flow please.
|
||||
except (
|
||||
http.HttpAuthenticationError,
|
||||
http.HttpError,
|
||||
proxy.ProxyError,
|
||||
tcp.NetLibError,
|
||||
) as e:
|
||||
self.handle_error(e, flow)
|
||||
except KillSignal:
|
||||
self.c.log("Connection killed", "info")
|
||||
finally:
|
||||
flow.live = None # Connection is not live anymore.
|
||||
return False
|
||||
|
||||
def handle_server_reconnect(self, state):
|
||||
if state["state"] == "connect":
|
||||
send_connect_request(
|
||||
self.c.server_conn,
|
||||
state["host"],
|
||||
state["port"],
|
||||
update_state=False
|
||||
)
|
||||
else: # pragma: nocover
|
||||
raise RuntimeError("Unknown State: %s" % state["state"])
|
||||
|
||||
def handle_error(self, error, flow=None):
|
||||
message = repr(error)
|
||||
message_debug = None
|
||||
|
||||
if isinstance(error, tcp.NetLibError):
|
||||
message = None
|
||||
message_debug = "TCP connection closed unexpectedly."
|
||||
elif "tlsv1 alert unknown ca" in message:
|
||||
message = "TLSv1 Alert Unknown CA: The client does not trust the proxy's certificate."
|
||||
elif "handshake error" in message:
|
||||
message_debug = message
|
||||
message = "SSL handshake error: The client may not trust the proxy's certificate."
|
||||
|
||||
if message:
|
||||
self.c.log(message, level="info")
|
||||
if message_debug:
|
||||
self.c.log(message_debug, level="debug")
|
||||
|
||||
if flow:
|
||||
# TODO: no flows without request or with both request and response
|
||||
# at the moment.
|
||||
if flow.request and not flow.response:
|
||||
flow.error = Error(message or message_debug)
|
||||
self.c.channel.ask("error", flow)
|
||||
try:
|
||||
status_code = getattr(error, "code", 502)
|
||||
headers = getattr(error, "headers", None)
|
||||
|
||||
html_message = message or ""
|
||||
if message_debug:
|
||||
html_message += "<pre>%s</pre>" % message_debug
|
||||
self.send_error(status_code, html_message, headers)
|
||||
except:
|
||||
pass
|
||||
|
||||
def send_error(self, status_code, message, headers):
|
||||
response = http.status_codes.RESPONSES.get(status_code, "Unknown")
|
||||
body = """
|
||||
<html>
|
||||
<head>
|
||||
<title>%d %s</title>
|
||||
</head>
|
||||
<body>%s</body>
|
||||
</html>
|
||||
""" % (status_code, response, message)
|
||||
|
||||
if not headers:
|
||||
headers = odict.ODictCaseless()
|
||||
assert isinstance(headers, odict.ODictCaseless)
|
||||
|
||||
headers["Server"] = [self.c.config.server_version]
|
||||
headers["Connection"] = ["close"]
|
||||
headers["Content-Length"] = [len(body)]
|
||||
headers["Content-Type"] = ["text/html"]
|
||||
|
||||
resp = HTTPResponse(
|
||||
(1, 1), # if HTTP/2 is used, this value is ignored anyway
|
||||
status_code,
|
||||
response,
|
||||
headers,
|
||||
body,
|
||||
)
|
||||
|
||||
# if no protocol is assigned yet - just assume HTTP/1
|
||||
# TODO: maybe check ALPN and use HTTP/2 if required?
|
||||
protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn)
|
||||
self.c.client_conn.send(protocol.assemble(resp))
|
||||
|
||||
def process_request(self, flow, request):
|
||||
"""
|
||||
@returns:
|
||||
True, if the request should not be sent upstream
|
||||
False, if the connection should be aborted
|
||||
None, if the request should be sent upstream
|
||||
(a status code != None should be returned directly by handle_flow)
|
||||
"""
|
||||
|
||||
if not self.skip_authentication:
|
||||
self.authenticate(request)
|
||||
|
||||
# Determine .scheme, .host and .port attributes
|
||||
# For absolute-form requests, they are directly given in the request.
|
||||
# For authority-form requests, we only need to determine the request scheme.
|
||||
# For relative-form requests, we need to determine host and port as
|
||||
# well.
|
||||
if not request.scheme:
|
||||
request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http"
|
||||
if not request.host:
|
||||
# Host/Port Complication: In upstream mode, use the server we CONNECTed to,
|
||||
# not the upstream proxy.
|
||||
if flow.server_conn:
|
||||
for s in flow.server_conn.state:
|
||||
if s[0] == "http" and s[1]["state"] == "connect":
|
||||
request.host, request.port = s[1]["host"], s[1]["port"]
|
||||
if not request.host and flow.server_conn:
|
||||
request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port
|
||||
|
||||
|
||||
# Now we can process the request.
|
||||
if request.form_in == "authority":
|
||||
if self.c.client_conn.ssl_established:
|
||||
raise http.HttpError(
|
||||
400,
|
||||
"Must not CONNECT on already encrypted connection"
|
||||
)
|
||||
|
||||
if self.c.config.mode == "regular":
|
||||
self.c.set_server_address((request.host, request.port))
|
||||
# Update server_conn attribute on the flow
|
||||
flow.server_conn = self.c.server_conn
|
||||
|
||||
# since we currently only support HTTP/1 CONNECT requests
|
||||
# the response must be HTTP/1 as well
|
||||
self.c.client_conn.send(
|
||||
('HTTP/%s.%s 200 ' % (request.httpversion[0], request.httpversion[1])) +
|
||||
'Connection established\r\n' +
|
||||
'Content-Length: 0\r\n' +
|
||||
('Proxy-agent: %s\r\n' % self.c.config.server_version) +
|
||||
'\r\n'
|
||||
)
|
||||
return self.process_connect_request(self.c.server_conn.address)
|
||||
elif self.c.config.mode == "upstream":
|
||||
return None
|
||||
else:
|
||||
# CONNECT should never occur if we don't expect absolute-form
|
||||
# requests
|
||||
pass
|
||||
|
||||
elif request.form_in == self.expected_form_in:
|
||||
request.form_out = self.expected_form_out
|
||||
if request.form_in == "absolute":
|
||||
if request.scheme != "http":
|
||||
raise http.HttpError(
|
||||
400,
|
||||
"Invalid request scheme: %s" % request.scheme
|
||||
)
|
||||
if self.c.config.mode == "regular":
|
||||
# Update info so that an inline script sees the correct
|
||||
# value at flow.server_conn
|
||||
self.c.set_server_address((request.host, request.port))
|
||||
flow.server_conn = self.c.server_conn
|
||||
|
||||
elif request.form_in == "relative":
|
||||
if self.c.config.mode == "spoof":
|
||||
# Host header
|
||||
h = request.pretty_host(hostheader=True)
|
||||
if h is None:
|
||||
raise http.HttpError(
|
||||
400,
|
||||
"Invalid request: No host information"
|
||||
)
|
||||
p = netlib.utils.parse_url("http://" + h)
|
||||
request.scheme = p[0]
|
||||
request.host = p[1]
|
||||
request.port = p[2]
|
||||
self.c.set_server_address((request.host, request.port))
|
||||
flow.server_conn = self.c.server_conn
|
||||
|
||||
if self.c.config.mode == "sslspoof":
|
||||
# SNI is processed in server.py
|
||||
if not (flow.server_conn and flow.server_conn.ssl_established):
|
||||
raise http.HttpError(
|
||||
400,
|
||||
"Invalid request: No host information"
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
raise http.HttpError(
|
||||
400, "Invalid HTTP request form (expected: %s, got: %s)" % (
|
||||
self.expected_form_in, request.form_in
|
||||
)
|
||||
)
|
||||
|
||||
def process_server_address(self, flow):
|
||||
# Depending on the proxy mode, server handling is entirely different
|
||||
# We provide a mostly unified API to the user, which needs to be
|
||||
# unfiddled here
|
||||
# ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 )
|
||||
address = tcp.Address((flow.request.host, flow.request.port))
|
||||
|
||||
ssl = (flow.request.scheme == "https")
|
||||
|
||||
if self.c.config.mode == "upstream":
|
||||
# The connection to the upstream proxy may have a state we may need
|
||||
# to take into account.
|
||||
connected_to = None
|
||||
for s in flow.server_conn.state:
|
||||
if s[0] == "http" and s[1]["state"] == "connect":
|
||||
connected_to = tcp.Address((s[1]["host"], s[1]["port"]))
|
||||
|
||||
# We need to reconnect if the current flow either requires a
|
||||
# (possibly impossible) change to the connection state, e.g. the
|
||||
# host has changed but we already CONNECTed somewhere else.
|
||||
needs_server_change = (
|
||||
ssl != self.c.server_conn.ssl_established
|
||||
or
|
||||
# HTTP proxying is "stateless", CONNECT isn't.
|
||||
(connected_to and address != connected_to)
|
||||
)
|
||||
|
||||
if needs_server_change:
|
||||
# force create new connection to the proxy server to reset
|
||||
# state
|
||||
self.live.change_server(self.c.server_conn.address, force=True)
|
||||
if ssl:
|
||||
send_connect_request(
|
||||
self.c.server_conn,
|
||||
address.host,
|
||||
address.port
|
||||
)
|
||||
self.c.establish_ssl(server=True)
|
||||
else:
|
||||
# If we're not in upstream mode, we just want to update the host
|
||||
# and possibly establish TLS. This is a no op if the addresses
|
||||
# match.
|
||||
self.live.change_server(address, ssl=ssl)
|
||||
|
||||
flow.server_conn = self.c.server_conn
|
||||
|
||||
def send_response_to_client(self, flow):
|
||||
if not flow.response.stream:
|
||||
# no streaming:
|
||||
# we already received the full response from the server and can
|
||||
# send it to the client straight away.
|
||||
self.c.client_conn.send(self.c.client_conn.protocol.assemble(flow.response))
|
||||
else:
|
||||
if isinstance(self.c.client_conn.protocol, http2.HTTP2Protocol):
|
||||
raise NotImplementedError("HTTP streaming with HTTP/2 is currently not supported.")
|
||||
|
||||
|
||||
# streaming:
|
||||
# First send the headers and then transfer the response
|
||||
# incrementally:
|
||||
h = self.c.client_conn.protocol._assemble_response_first_line(flow.response)
|
||||
self.c.client_conn.send(h + "\r\n")
|
||||
h = self.c.client_conn.protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True)
|
||||
self.c.client_conn.send(h + "\r\n")
|
||||
|
||||
chunks = self.c.server_conn.protocol.read_http_body_chunked(
|
||||
flow.response.headers,
|
||||
self.c.config.body_size_limit,
|
||||
flow.request.method,
|
||||
flow.response.code,
|
||||
False,
|
||||
4096
|
||||
)
|
||||
|
||||
if callable(flow.response.stream):
|
||||
chunks = flow.response.stream(chunks)
|
||||
|
||||
for chunk in chunks:
|
||||
for part in chunk:
|
||||
self.c.client_conn.wfile.write(part)
|
||||
self.c.client_conn.wfile.flush()
|
||||
|
||||
flow.response.timestamp_end = utils.timestamp()
|
||||
|
||||
def check_close_connection(self, flow):
|
||||
"""
|
||||
Checks if the connection should be closed depending on the HTTP
|
||||
semantics. Returns True, if so.
|
||||
"""
|
||||
|
||||
# TODO: add logic for HTTP/2
|
||||
|
||||
close_connection = (
|
||||
http1.HTTP1Protocol.connection_close(
|
||||
flow.request.httpversion,
|
||||
flow.request.headers
|
||||
) or http1.HTTP1Protocol.connection_close(
|
||||
flow.response.httpversion,
|
||||
flow.response.headers
|
||||
) or http1.HTTP1Protocol.expected_http_body_size(
|
||||
flow.response.headers,
|
||||
False,
|
||||
flow.request.method,
|
||||
flow.response.code) == -1
|
||||
)
|
||||
if close_connection:
|
||||
if flow.request.form_in == "authority" and flow.response.code == 200:
|
||||
# Workaround for
|
||||
# https://github.com/mitmproxy/mitmproxy/issues/313: Some
|
||||
# proxies (e.g. Charles) send a CONNECT response with HTTP/1.0
|
||||
# and no Content-Length header
|
||||
pass
|
||||
else:
|
||||
return True
|
||||
return False
|
||||
|
||||
def process_connect_request(self, address):
|
||||
"""
|
||||
Process a CONNECT request.
|
||||
Returns True if the CONNECT request has been processed successfully.
|
||||
Returns False, if the connection should be closed immediately.
|
||||
"""
|
||||
address = tcp.Address.wrap(address)
|
||||
if self.c.config.check_ignore(address):
|
||||
self.c.log("Ignore host: %s:%s" % address(), "info")
|
||||
TCPHandler(self.c, log=False).handle_messages()
|
||||
return False
|
||||
else:
|
||||
self.expected_form_in = "relative"
|
||||
self.expected_form_out = "relative"
|
||||
self.skip_authentication = True
|
||||
|
||||
# In practice, nobody issues a CONNECT request to send unencrypted
|
||||
# HTTP requests afterwards. If we don't delegate to TCP mode, we
|
||||
# should always negotiate a SSL connection.
|
||||
#
|
||||
# FIXME: Turns out the previous statement isn't entirely true.
|
||||
# Chrome on Windows CONNECTs to :80 if an explicit proxy is
|
||||
# configured and a websocket connection should be established. We
|
||||
# don't support websocket at the moment, so it fails anyway, but we
|
||||
# should come up with a better solution to this if we start to
|
||||
# support WebSockets.
|
||||
should_establish_ssl = (
|
||||
address.port in self.c.config.ssl_ports
|
||||
or
|
||||
not self.c.config.check_tcp(address)
|
||||
)
|
||||
|
||||
if should_establish_ssl:
|
||||
self.c.log(
|
||||
"Received CONNECT request to SSL port. "
|
||||
"Upgrading to SSL...", "debug"
|
||||
)
|
||||
server_ssl = not self.c.config.no_upstream_cert
|
||||
if server_ssl:
|
||||
self.c.establish_server_connection()
|
||||
self.c.establish_ssl(server=server_ssl, client=True)
|
||||
self.c.log("Upgrade to SSL completed.", "debug")
|
||||
|
||||
if self.c.config.check_tcp(address):
|
||||
self.c.log(
|
||||
"Generic TCP mode for host: %s:%s" % address(),
|
||||
"info"
|
||||
)
|
||||
TCPHandler(self.c).handle_messages()
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def authenticate(self, request):
|
||||
if self.c.config.authenticator:
|
||||
if self.c.config.authenticator.authenticate(request.headers):
|
||||
self.c.config.authenticator.clean(request.headers)
|
||||
else:
|
||||
raise http.HttpAuthenticationError(
|
||||
self.c.config.authenticator.auth_challenge_headers())
|
||||
return request.headers
|
||||
|
|
@ -1,20 +1,12 @@
|
|||
from __future__ import absolute_import
|
||||
import Cookie
|
||||
import copy
|
||||
import threading
|
||||
import time
|
||||
import urllib
|
||||
import urlparse
|
||||
from email.utils import parsedate_tz, formatdate, mktime_tz
|
||||
|
||||
import netlib
|
||||
from netlib import http, tcp, odict, utils, encoding
|
||||
from netlib.http import cookies, semantics, http1
|
||||
|
||||
from .tcp import TCPHandler
|
||||
from .primitives import KILL, ProtocolHandler, Flow, Error
|
||||
from ..proxy.connection import ServerConnection
|
||||
from .. import utils, controller, stateobject, proxy
|
||||
from netlib import odict, encoding
|
||||
from netlib.http import semantics, CONTENT_MISSING
|
||||
from .. import utils, stateobject
|
||||
|
||||
|
||||
class decoded(object):
|
||||
|
|
@ -170,19 +162,19 @@ class HTTPRequest(MessageMixin, semantics.Request):
|
|||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
form_in,
|
||||
method,
|
||||
scheme,
|
||||
host,
|
||||
port,
|
||||
path,
|
||||
httpversion,
|
||||
headers,
|
||||
body,
|
||||
timestamp_start=None,
|
||||
timestamp_end=None,
|
||||
form_out=None,
|
||||
self,
|
||||
form_in,
|
||||
method,
|
||||
scheme,
|
||||
host,
|
||||
port,
|
||||
path,
|
||||
httpversion,
|
||||
headers,
|
||||
body,
|
||||
timestamp_start=None,
|
||||
timestamp_end=None,
|
||||
form_out=None,
|
||||
):
|
||||
semantics.Request.__init__(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
from __future__ import absolute_import
|
||||
import copy
|
||||
import uuid
|
||||
import netlib.tcp
|
||||
|
||||
from .. import stateobject, utils, version
|
||||
from ..proxy.connection import ClientConnection, ServerConnection
|
||||
|
||||
|
||||
KILL = 0 # const for killed requests
|
||||
|
||||
|
||||
|
|
@ -165,130 +164,3 @@ class Flow(stateobject.StateObject):
|
|||
self.intercepted = False
|
||||
self.reply()
|
||||
master.handle_accept_intercept(self)
|
||||
|
||||
|
||||
|
||||
class ProtocolHandler(object):
|
||||
"""
|
||||
A ProtocolHandler implements an application-layer protocol, e.g. HTTP.
|
||||
See: libmproxy.protocol.http.HTTPHandler
|
||||
"""
|
||||
|
||||
def __init__(self, c):
|
||||
self.c = c
|
||||
"""@type: libmproxy.proxy.server.ConnectionHandler"""
|
||||
self.live = LiveConnection(c)
|
||||
"""@type: LiveConnection"""
|
||||
|
||||
def handle_messages(self):
|
||||
"""
|
||||
This method gets called if a client connection has been made. Depending
|
||||
on the proxy settings, a server connection might already exist as well.
|
||||
"""
|
||||
raise NotImplementedError # pragma: nocover
|
||||
|
||||
def handle_server_reconnect(self, state):
|
||||
"""
|
||||
This method gets called if a server connection needs to reconnect and
|
||||
there's a state associated with the server connection (e.g. a
|
||||
previously-sent CONNECT request or a SOCKS proxy request). This method
|
||||
gets called after the connection has been restablished but before SSL is
|
||||
established.
|
||||
"""
|
||||
raise NotImplementedError # pragma: nocover
|
||||
|
||||
def handle_error(self, error):
|
||||
"""
|
||||
This method gets called should there be an uncaught exception during the
|
||||
connection. This might happen outside of handle_messages, e.g. if the
|
||||
initial SSL handshake fails in transparent mode.
|
||||
"""
|
||||
raise error # pragma: nocover
|
||||
|
||||
|
||||
class LiveConnection(object):
|
||||
"""
|
||||
This facade allows interested parties (FlowMaster, inline scripts) to
|
||||
interface with a live connection, without exposing the internals
|
||||
of the ConnectionHandler.
|
||||
"""
|
||||
|
||||
def __init__(self, c):
|
||||
self.c = c
|
||||
"""@type: libmproxy.proxy.server.ConnectionHandler"""
|
||||
self._backup_server_conn = None
|
||||
"""@type: libmproxy.proxy.connection.ServerConnection"""
|
||||
|
||||
def change_server(
|
||||
self,
|
||||
address,
|
||||
ssl=None,
|
||||
sni=None,
|
||||
force=False,
|
||||
persistent_change=False):
|
||||
"""
|
||||
Change the server connection to the specified address.
|
||||
@returns:
|
||||
True, if a new connection has been established,
|
||||
False, if an existing connection has been used
|
||||
"""
|
||||
address = netlib.tcp.Address.wrap(address)
|
||||
|
||||
ssl_mismatch = (
|
||||
ssl is not None and
|
||||
(
|
||||
(self.c.server_conn.connection and ssl != self.c.server_conn.ssl_established)
|
||||
or
|
||||
(sni is not None and sni != self.c.server_conn.sni)
|
||||
)
|
||||
)
|
||||
address_mismatch = (address != self.c.server_conn.address)
|
||||
|
||||
if persistent_change:
|
||||
self._backup_server_conn = None
|
||||
|
||||
if ssl_mismatch or address_mismatch or force:
|
||||
|
||||
self.c.log(
|
||||
"Change server connection: %s:%s -> %s:%s [persistent: %s]" % (
|
||||
self.c.server_conn.address.host,
|
||||
self.c.server_conn.address.port,
|
||||
address.host,
|
||||
address.port,
|
||||
persistent_change
|
||||
),
|
||||
"debug"
|
||||
)
|
||||
|
||||
if not self._backup_server_conn and not persistent_change:
|
||||
self._backup_server_conn = self.c.server_conn
|
||||
self.c.server_conn = None
|
||||
else:
|
||||
# This is at least the second temporary change. We can kill the
|
||||
# current connection.
|
||||
self.c.del_server_connection()
|
||||
|
||||
self.c.set_server_address(address)
|
||||
self.c.establish_server_connection(ask=False)
|
||||
if ssl:
|
||||
self.c.establish_ssl(server=True, sni=sni)
|
||||
return True
|
||||
return False
|
||||
|
||||
def restore_server(self):
|
||||
# TODO: Similar to _backup_server_conn, introduce _cache_server_conn,
|
||||
# which keeps the changed connection open This may be beneficial if a
|
||||
# user is rewriting all requests from http to https or similar.
|
||||
if not self._backup_server_conn:
|
||||
return
|
||||
|
||||
self.c.log("Restore original server connection: %s:%s -> %s:%s" % (
|
||||
self.c.server_conn.address.host,
|
||||
self.c.server_conn.address.port,
|
||||
self._backup_server_conn.address.host,
|
||||
self._backup_server_conn.address.port
|
||||
), "debug")
|
||||
|
||||
self.c.del_server_connection()
|
||||
self.c.server_conn = self._backup_server_conn
|
||||
self._backup_server_conn = None
|
||||
|
|
|
|||
|
|
@ -1,97 +0,0 @@
|
|||
from __future__ import absolute_import
|
||||
import select
|
||||
import socket
|
||||
from .primitives import ProtocolHandler
|
||||
from netlib.utils import cleanBin
|
||||
from netlib.tcp import NetLibError
|
||||
|
||||
|
||||
class TCPHandler(ProtocolHandler):
|
||||
"""
|
||||
TCPHandler acts as a generic TCP forwarder.
|
||||
Data will be .log()ed, but not stored any further.
|
||||
"""
|
||||
|
||||
chunk_size = 4096
|
||||
|
||||
def __init__(self, c, log=True):
|
||||
super(TCPHandler, self).__init__(c)
|
||||
self.log = log
|
||||
|
||||
def handle_messages(self):
|
||||
self.c.establish_server_connection()
|
||||
|
||||
server = "%s:%s" % self.c.server_conn.address()[:2]
|
||||
buf = memoryview(bytearray(self.chunk_size))
|
||||
conns = [self.c.client_conn.rfile, self.c.server_conn.rfile]
|
||||
|
||||
try:
|
||||
while True:
|
||||
r, _, _ = select.select(conns, [], [], 10)
|
||||
for rfile in r:
|
||||
if self.c.client_conn.rfile == rfile:
|
||||
src, dst = self.c.client_conn, self.c.server_conn
|
||||
direction = "-> tcp ->"
|
||||
src_str, dst_str = "client", server
|
||||
else:
|
||||
dst, src = self.c.client_conn, self.c.server_conn
|
||||
direction = "<- tcp <-"
|
||||
dst_str, src_str = "client", server
|
||||
|
||||
closed = False
|
||||
if src.ssl_established:
|
||||
# Unfortunately, pyOpenSSL lacks a recv_into function.
|
||||
# We need to read a single byte before .pending()
|
||||
# becomes usable
|
||||
contents = src.rfile.read(1)
|
||||
contents += src.rfile.read(src.connection.pending())
|
||||
if not contents:
|
||||
closed = True
|
||||
else:
|
||||
size = src.connection.recv_into(buf)
|
||||
if not size:
|
||||
closed = True
|
||||
|
||||
if closed:
|
||||
conns.remove(src.rfile)
|
||||
# Shutdown connection to the other peer
|
||||
if dst.ssl_established:
|
||||
# We can't half-close a connection, so we just close everything here.
|
||||
# Sockets will be cleaned up on a higher level.
|
||||
return
|
||||
else:
|
||||
dst.connection.shutdown(socket.SHUT_WR)
|
||||
|
||||
if len(conns) == 0:
|
||||
return
|
||||
continue
|
||||
|
||||
if src.ssl_established or dst.ssl_established:
|
||||
# if one of the peers is over SSL, we need to send
|
||||
# bytes/strings
|
||||
if not src.ssl_established:
|
||||
# we revc'd into buf but need bytes/string now.
|
||||
contents = buf[:size].tobytes()
|
||||
if self.log:
|
||||
self.c.log(
|
||||
"%s %s\r\n%s" % (
|
||||
direction, dst_str, cleanBin(contents)
|
||||
),
|
||||
"info"
|
||||
)
|
||||
# Do not use dst.connection.send here, which may raise
|
||||
# OpenSSL-specific errors.
|
||||
dst.send(contents)
|
||||
else:
|
||||
# socket.socket.send supports raw bytearrays/memoryviews
|
||||
if self.log:
|
||||
self.c.log(
|
||||
"%s %s\r\n%s" % (
|
||||
direction, dst_str, cleanBin(buf.tobytes())
|
||||
),
|
||||
"info"
|
||||
)
|
||||
dst.connection.send(buf[:size])
|
||||
except (socket.error, NetLibError) as e:
|
||||
self.c.log("TCP connection closed unexpectedly.", "debug")
|
||||
return
|
||||
Loading…
Reference in New Issue