Rooba
/
mitmproxy
mirror of https://github.com/mitmproxy/mitmproxy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Whitespace and formatting

This commit is contained in:
Aldo Cortesi 2014-10-25 14:24:05 +13:00
parent 5aadf92767
commit d6ee532711
2 changed files with 125 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
libpathod/language.py
View File

@ -1,4 +1,10 @@
import operator, string, random, mmap, os, time, copy import operator
import string
import random
import mmap
import os
import time
import copy
import abc import abc
from email.utils import formatdate from email.utils import formatdate
import contrib.pyparsing as pp import contrib.pyparsing as pp
@ -9,7 +15,9 @@ import utils
BLOCKSIZE = 1024 BLOCKSIZE = 1024
TRUNCATE = 1024 TRUNCATE = 1024
class FileAccessDenied(Exception): pass
class FileAccessDenied(Exception):
pass
class ParseException(Exception): class ParseException(Exception):
@ -20,7 +28,7 @@ class ParseException(Exception):
self.col = col self.col = col
def marked(self): def marked(self):
return "%s\n%s"%(self.s, " "*(self.col-1) + "^") return "%s\n%s"%(self.s, " "*(self.col - 1) + "^")
def __str__(self): def __str__(self):
return "%s at char %s"%(self.msg, self.col) return "%s at char %s"%(self.msg, self.col)
@ -40,7 +48,9 @@ def send_chunk(fp, val, blocksize, start, end):
def write_values(fp, vals, actions, sofar=0, skip=0, blocksize=BLOCKSIZE): def write_values(fp, vals, actions, sofar=0, skip=0, blocksize=BLOCKSIZE):
""" """
vals: A list of values, which may be strings or Value objects. vals: A list of values, which may be strings or Value objects.
actions: A list of (offset, action, arg) tuples. Action may be "pause" or "disconnect".
actions: A list of (offset, action, arg) tuples. Action may be "pause"
or "disconnect".
Both vals and actions are in reverse order, with the first items last. Both vals and actions are in reverse order, with the first items last.
@ -53,7 +63,13 @@ def write_values(fp, vals, actions, sofar=0, skip=0, blocksize=BLOCKSIZE):
offset = 0 offset = 0
while actions and actions[-1][0] < (sofar + len(v)): while actions and actions[-1][0] < (sofar + len(v)):
a = actions.pop() a = actions.pop()
offset += send_chunk(fp, v, blocksize, offset, a[0]-sofar-offset) offset += send_chunk(
fp,
v,
blocksize,
offset,
a[0]-sofar-offset
)
if a[1] == "pause": if a[1] == "pause":
time.sleep(a[2]) time.sleep(a[2])
elif a[1] == "disconnect": elif a[1] == "disconnect":
@ -128,8 +144,18 @@ v_integer = pp.Regex(r"\d+")\
v_literal = pp.MatchFirst( v_literal = pp.MatchFirst(
[ [
pp.QuotedString("\"", escChar="\\", unquoteResults=True, multiline=True), pp.QuotedString(
pp.QuotedString("'", escChar="\\", unquoteResults=True, multiline=True), "\"",
escChar="\\",
unquoteResults=True,
multiline=True
),
pp.QuotedString(
"'",
escChar="\\",
unquoteResults=True,
multiline=True
),
] ]
) )
@ -202,6 +228,7 @@ class _Token(object):
A specification token. Tokens are immutable. A specification token. Tokens are immutable.
""" """
__metaclass__ = abc.ABCMeta __metaclass__ = abc.ABCMeta
@abc.abstractmethod @abc.abstractmethod
def expr(klass): # pragma: no cover def expr(klass): # pragma: no cover
""" """
@ -278,7 +305,10 @@ class ValueGenerate(_Token):
def expr(klass): def expr(klass):
e = pp.Literal("@").suppress() + v_integer e = pp.Literal("@").suppress() + v_integer
u = reduce(operator.or_, [pp.Literal(i) for i in utils.SIZE_UNITS.keys()]) u = reduce(
operator.or_,
[pp.Literal(i) for i in utils.SIZE_UNITS.keys()]
)
e = e + pp.Optional(u, default=None) e = e + pp.Optional(u, default=None)
s = pp.Literal(",").suppress() s = pp.Literal(",").suppress()
@ -318,7 +348,9 @@ class ValueFile(_Token):
s = os.path.expanduser(self.path) s = os.path.expanduser(self.path)
s = os.path.normpath(os.path.abspath(os.path.join(sd, s))) s = os.path.normpath(os.path.abspath(os.path.join(sd, s)))
if not uf and not s.startswith(sd): if not uf and not s.startswith(sd):
raise FileAccessDenied("File access outside of configured directory") raise FileAccessDenied(
"File access outside of configured directory"
)
if not os.path.isfile(s): if not os.path.isfile(s):
raise FileAccessDenied("File not readable") raise FileAccessDenied("File not readable")
return FileGenerator(s) return FileGenerator(s)
@ -347,12 +379,12 @@ NakedValue = pp.MatchFirst(
Offset = pp.MatchFirst( Offset = pp.MatchFirst(
[ [
v_integer, v_integer,
pp.Literal("r"), pp.Literal("r"),
pp.Literal("a") pp.Literal("a")
] ]
) )
class Raw(_Token): class Raw(_Token):
@ -392,11 +424,11 @@ class _Header(_Component):
def values(self, settings): def values(self, settings):
return [ return [
self.key.get_generator(settings), self.key.get_generator(settings),
": ", ": ",
self.value.get_generator(settings), self.value.get_generator(settings),
"\r\n", "\r\n",
] ]
class Header(_Header): class Header(_Header):
@ -459,7 +491,10 @@ class ShortcutUserAgent(_Header):
@classmethod @classmethod
def expr(klass): def expr(klass):
e = pp.Literal("u").suppress() e = pp.Literal("u").suppress()
u = reduce(operator.or_, [pp.Literal(i[1]) for i in http_uastrings.UASTRINGS]) u = reduce(
operator.or_,
[pp.Literal(i[1]) for i in http_uastrings.UASTRINGS]
)
e += u | Value e += u | Value
return e.setParseAction(lambda x: klass(*x)) return e.setParseAction(lambda x: klass(*x))
@ -470,7 +505,6 @@ class ShortcutUserAgent(_Header):
return ShortcutUserAgent(self.value.freeze(settings)) return ShortcutUserAgent(self.value.freeze(settings))
class Body(_Component): class Body(_Component):
def __init__(self, value): def __init__(self, value):
self.value = value self.value = value
@ -483,8 +517,8 @@ class Body(_Component):
def values(self, settings): def values(self, settings):
return [ return [
self.value.get_generator(settings), self.value.get_generator(settings),
] ]
def spec(self): def spec(self):
return "b%s"%(self.value.spec()) return "b%s"%(self.value.spec())
@ -506,8 +540,8 @@ class Path(_Component):
def values(self, settings): def values(self, settings):
return [ return [
self.value.get_generator(settings), self.value.get_generator(settings),
] ]
def spec(self): def spec(self):
return "%s"%(self.value.spec()) return "%s"%(self.value.spec())
@ -527,6 +561,7 @@ class Method(_Component):
"trace", "trace",
"connect", "connect",
] ]
def __init__(self, value): def __init__(self, value):
# If it's a string, we were passed one of the methods, so we upper-case # If it's a string, we were passed one of the methods, so we upper-case
# it to be canonical. The user can specify a different case by using a # it to be canonical. The user can specify a different case by using a
@ -645,11 +680,11 @@ class PauseAt(_Action):
e += Offset e += Offset
e += pp.Literal(",").suppress() e += pp.Literal(",").suppress()
e += pp.MatchFirst( e += pp.MatchFirst(
[ [
v_integer, v_integer,
pp.Literal("f") pp.Literal("f")
] ]
) )
return e.setParseAction(lambda x: klass(*x)) return e.setParseAction(lambda x: klass(*x))
def spec(self): def spec(self):
@ -700,10 +735,10 @@ class InjectAt(_Action):
def intermediate(self, settings): def intermediate(self, settings):
return ( return (
self.offset, self.offset,
"inject", "inject",
self.value.get_generator(settings) self.value.get_generator(settings)
) )
def freeze(self, settings): def freeze(self, settings):
return InjectAt(self.offset, self.value.freeze(settings)) return InjectAt(self.offset, self.value.freeze(settings))
@ -712,6 +747,7 @@ class InjectAt(_Action):
class _Message(object): class _Message(object):
__metaclass__ = abc.ABCMeta __metaclass__ = abc.ABCMeta
version = "HTTP/1.1" version = "HTTP/1.1"
def __init__(self, tokens): def __init__(self, tokens):
self.tokens = tokens self.tokens = tokens
@ -741,7 +777,8 @@ class _Message(object):
def length(self, settings): def length(self, settings):
""" """
Calculate the length of the base message without any applied actions. Calculate the length of the base message without any applied
actions.
""" """
return sum(len(x) for x in self.values(settings)) return sum(len(x) for x in self.values(settings))
@ -754,7 +791,8 @@ class _Message(object):
def maximum_length(self, settings): def maximum_length(self, settings):
""" """
Calculate the maximum length of the base message with all applied actions. Calculate the maximum length of the base message with all applied
actions.
""" """
l = self.length(settings) l = self.length(settings)
for i in self.actions: for i in self.actions:
@ -786,7 +824,13 @@ class _Message(object):
tokens.append( tokens.append(
Header( Header(
ValueLiteral("Date"), ValueLiteral("Date"),
ValueLiteral(formatdate(timeval=None, localtime=False, usegmt=True)) ValueLiteral(
formatdate(
timeval=None,
localtime=False,
usegmt=True
)
)
) )
) )
intermediate = self.__class__(tokens) intermediate = self.__class__(tokens)
@ -807,7 +851,8 @@ class _Message(object):
ret = {} ret = {}
for i in self.logattrs: for i in self.logattrs:
v = getattr(self, i) v = getattr(self, i)
# Careful not to log any VALUE specs without sanitizing them first. We truncate at 1k. # Careful not to log any VALUE specs without sanitizing them first.
# We truncate at 1k.
if hasattr(v, "values"): if hasattr(v, "values"):
v = [x[:TRUNCATE] for x in v.values(settings)] v = [x[:TRUNCATE] for x in v.values(settings)]
v = "".join(v).encode("string_escape") v = "".join(v).encode("string_escape")
@ -838,6 +883,7 @@ class _Message(object):
Sep = pp.Optional(pp.Literal(":")).suppress() Sep = pp.Optional(pp.Literal(":")).suppress()
class Response(_Message): class Response(_Message):
comps = ( comps = (
Body, Body,
@ -851,6 +897,7 @@ class Response(_Message):
Reason Reason
) )
logattrs = ["code", "reason", "version", "body"] logattrs = ["code", "reason", "version", "body"]
@property @property
def code(self): def code(self):
return self._get_token(Code) return self._get_token(Code)
@ -866,7 +913,14 @@ class Response(_Message):
if self.reason: if self.reason:
l.extend(self.reason.values(settings)) l.extend(self.reason.values(settings))
else: else:
l.append(LiteralGenerator(http_status.RESPONSES.get(int(self.code.code), "Unknown code"))) l.append(
LiteralGenerator(
http_status.RESPONSES.get(
int(self.code.code),
"Unknown code"
)
)
)
return l return l
@classmethod @classmethod
@ -897,6 +951,7 @@ class Request(_Message):
Raw Raw
) )
logattrs = ["method", "path", "body"] logattrs = ["method", "path", "body"]
@property @property
def method(self): def method(self):
return self._get_token(Method) return self._get_token(Method)
@ -944,7 +999,7 @@ def make_error_response(reason, body=None):
] ]
return PathodErrorResponse(tokens) return PathodErrorResponse(tokens)
FILESTART = "+"
def read_file(settings, s): def read_file(settings, s):
uf = settings.get("unconstrained_file_access") uf = settings.get("unconstrained_file_access")
sd = settings.get("staticdir") sd = settings.get("staticdir")

45
test/test_language.py
View File

@ -1,4 +1,5 @@
import os, cStringIO import os
import cStringIO
from libpathod import language, utils from libpathod import language, utils
import tutils import tutils
@ -475,7 +476,6 @@ class TestRequest:
assert r.path.string() == "/foo" assert r.path.string() == "/foo"
assert r.actions assert r.actions
l = """ l = """
GET GET
@ -611,6 +611,11 @@ class TestResponse:
r = language.parse_response("400:m'msg'") r = language.parse_response("400:m'msg'")
assert language.serve(r, s, {}) assert language.serve(r, s, {})
r = language.parse_response("400:p0,100:dr")
assert "p0" in r.spec()
s = r.preview_safe()
assert "p0" not in s.spec()
def test_raw(self): def test_raw(self):
s = cStringIO.StringIO() s = cStringIO.StringIO()
r = language.parse_response("400:b'foo'") r = language.parse_response("400:b'foo'")
@ -651,12 +656,6 @@ class TestResponse:
r = language.parse_response("400:m'msg':b@100:d0:i0,'foo'") r = language.parse_response("400:m'msg':b@100:d0:i0,'foo'")
testlen(r) testlen(r)
def test_render(self):
r = language.parse_response("400:p0,100:dr")
assert "p0" in r.spec()
s = r.preview_safe()
assert not "p0" in s.spec()
def test_parse_err(self): def test_parse_err(self):
tutils.raises(language.ParseException, language.parse_response, "400:msg,b:") tutils.raises(language.ParseException, language.parse_response, "400:msg,b:")
try: try:
@ -685,9 +684,10 @@ class TestResponse:
assert r.actions[0].spec() == "pr,10" assert r.actions[0].spec() == "pr,10"
def test_parse_stress(self): def test_parse_stress(self):
# While larger values are known to work on linux, # While larger values are known to work on linux, len() technically
# len() technically returns an int and a python 2.7 int on windows has 32bit precision. # returns an int and a python 2.7 int on windows has 32bit precision.
# Therefore, we should keep the body length < 2147483647 bytes in our tests. # Therefore, we should keep the body length < 2147483647 bytes in our
# tests.
r = language.parse_response("400:b@1g") r = language.parse_response("400:b@1g")
assert r.length({}) assert r.length({})
@ -700,16 +700,29 @@ class TestResponse:
rt("400:da") rt("400:da")
def test_read_file(): def test_read_file():
tutils.raises(language.FileAccessDenied, language.read_file, {}, "=/foo") tutils.raises(language.FileAccessDenied, language.read_file, {}, "=/foo")
p = tutils.test_data.path("data") p = tutils.test_data.path("data")
d = dict(staticdir=p) d = dict(staticdir=p)
assert language.read_file(d, "+./file").strip() == "testfile" assert language.read_file(d, "+./file").strip() == "testfile"
assert language.read_file(d, "+file").strip() == "testfile" assert language.read_file(d, "+file").strip() == "testfile"
tutils.raises(language.FileAccessDenied, language.read_file, d, "+./nonexistent") tutils.raises(
tutils.raises(language.FileAccessDenied, language.read_file, d, "+/nonexistent") language.FileAccessDenied,
language.read_file,
tutils.raises(language.FileAccessDenied, language.read_file, d, "+../test_language.py") d,
"+./nonexistent"
)
tutils.raises(
language.FileAccessDenied,
language.read_file,
d,
"+/nonexistent"
)
tutils.raises(
language.FileAccessDenied,
language.read_file,
d,
"+../test_language.py"
)
d["unconstrained_file_access"] = True d["unconstrained_file_access"] = True
assert language.read_file(d, "+../test_language.py") assert language.read_file(d, "+../test_language.py")