From 8de09bd184b219bcbfe2fd7dbb62c3c76815d870 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Thu, 19 May 2016 22:14:48 -0700 Subject: [PATCH 1/5] adding preliminary wrap_trace functionality. still need to make action configurable --- TODO.rst | 2 + boltons/debugutils.py | 98 ++++++++++++++++++++++++++++++++++ tests/test_debugutils_trace.py | 88 ++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 tests/test_debugutils_trace.py diff --git a/TODO.rst b/TODO.rst index d0d3e24..253eca0 100644 --- a/TODO.rst +++ b/TODO.rst @@ -27,6 +27,8 @@ jsonutils misc? ----- +- wrap_trace debug utility. Takes an object, looks at its dir, wraps + everything callable, with a hook. Needs an enable/disable flag. - Tracking proxy. An object that always succeeds for all operations, saving the call history. - Top/Bottom singletons (greater than and less than everything) diff --git a/boltons/debugutils.py b/boltons/debugutils.py index cec147a..298cd72 100644 --- a/boltons/debugutils.py +++ b/boltons/debugutils.py @@ -64,3 +64,101 @@ def pdb_on_exception(limit=100): pdb.post_mortem(exc_tb) sys.excepthook = pdb_excepthook + return + + +import time + + +def tw_print_hook(name, a, kw): + print time.time(), name, a, kw + + +def wrap_trace(obj, which=None): + # other actions: pdb.set_trace, print, aggregate, aggregate_return + # (like aggregate but with the return value) Q: should aggregate + # includ self? + + # TODO: how to handle creating the instance + # Specifically, getting around the namedtuple problem + # TODO: test classmethod/staticmethod + + if isinstance(which, basestring): + which_func = lambda attr_name, attr_val: attr_name == which + else: # if callable(which): + which_func = which + + def wrap_method(attr_name, func): + def wrapped(*a, **kw): + a = a[1:] + print (time.time(), attr_name, a, kw) + return func(*a, **kw) + wrapped.__name__ = func.__name__ + wrapped.__doc__ = func.__doc__ + try: + wrapped.__module__ = func.__module__ + except Exception: + pass + try: + if func.__dict__: + wrapped.__dict__.update(func.__dict__) + except Exception: + pass + return wrapped + + def wrap_getattribute(): + def __getattribute__(self, attr_name): + print (time.time(), '__getattribute__', (attr_name,)) + ret = type(obj).__getattribute__(obj, attr_name) + if callable(ret): + ret = type(obj).__getattribute__(self, attr_name) + return ret + return __getattribute__ + + attrs = {} + for attr_name in dir(obj): + try: + attr_val = getattr(obj, attr_name) + except Exception: + continue + + if not callable(attr_val) or attr_name in ('__new__',): + continue + elif which_func and not which_func(attr_name, attr_val): + continue + + if attr_name == '__getattribute__': + wrapped_method = wrap_getattribute() + else: + wrapped_method = wrap_method(attr_name, attr_val) + + attrs[attr_name] = wrapped_method + + cls_name = obj.__class__.__name__ + if cls_name == cls_name.lower(): + type_name = 'wrapped_' + cls_name + else: + type_name = 'Wrapped' + cls_name + + if hasattr(obj, '__mro__'): + bases = (obj.__class__,) + else: + # need new-style class for even basic wrapping of callables to + # work. getattribute won't work for old-style classes of course. + bases = (obj.__class__, object) + + trace_type = type(type_name, bases, attrs) + for cls in trace_type.__mro__: + try: + return cls.__new__(trace_type) + except Exception: + pass + raise TypeError('unable to wrap_trace %r instance %r' + % (obj.__class__, obj)) + + +if __name__ == '__main__': + obj = wrap_trace({}) + obj['hi'] = 'hello' + obj.fail + import pdb;pdb.set_trace() diff --git a/tests/test_debugutils_trace.py b/tests/test_debugutils_trace.py new file mode 100644 index 0000000..a69708b --- /dev/null +++ b/tests/test_debugutils_trace.py @@ -0,0 +1,88 @@ + +from collections import namedtuple + +from pytest import raises + +from boltons.debugutils import wrap_trace + + +def test_trace_dict(): + target = {} + wrapped = wrap_trace(target) + + assert target is not wrapped + assert isinstance(wrapped, dict) + + wrapped['a'] = 'A' + assert target['a'] == 'A' + assert len(wrapped) == len(target) + + wrapped.pop('a') + assert 'a' not in target + + with raises(AttributeError): + wrapped.nonexistent_attr = 'nope' + + return + + +def test_trace_str(): + target = u'Hello'.encode('ascii') + + wrapped = wrap_trace(target) + + assert target is not wrapped + assert isinstance(wrapped, str) + + assert len(wrapped) == len(target) + assert wrapped.decode('utf-8') == u'Hello' + assert wrapped.lower() == target.lower() + + +def test_trace_exc(): + class TestException(Exception): + pass + + target = TestException('exceptions can be a good thing') + wrapped = wrap_trace(target) + + try: + raise wrapped + except TestException as te: + assert te.args == target.args + + +def test_trace_which(): + class Config(object): + def __init__(self, value): + self.value = value + + config = Config('first') + wrapped = wrap_trace(config, which='__setattr__') + + wrapped.value = 'second' + assert config.value == 'second' + + +def test_trace_namedtuple(): + TargetType = namedtuple('TargetType', 'x y z') + target = TargetType(1, 2, 3) + + wrapped = wrap_trace(target) + + assert wrapped == (1, 2, 3) + + +def test_trace_oldstyle(): + class Oldie: + test = object() + + def get_test(self): + return self.test + + oldie = Oldie() + + wrapped = wrap_trace(oldie) + assert wrapped.get_test() is oldie.test + + return From 926e91bef0c40e27b6cb8b90554198ecd91e2fe4 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Fri, 20 May 2016 01:46:28 -0700 Subject: [PATCH 2/5] pluggable hook for wrap_trace --- boltons/debugutils.py | 29 ++++++++++++++++++++--------- tests/test_debugutils_trace.py | 4 ++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/boltons/debugutils.py b/boltons/debugutils.py index 298cd72..d03ebd7 100644 --- a/boltons/debugutils.py +++ b/boltons/debugutils.py @@ -5,6 +5,13 @@ applications. Currently this focuses on ways to use :mod:`pdb`, the built-in Python debugger. """ +import time + +try: + basestring +except NameError: + basestring = (str, bytes) # py3 + __all__ = ['pdb_on_signal', 'pdb_on_exception'] @@ -67,14 +74,18 @@ def pdb_on_exception(limit=100): return -import time +def wt_print_hook(obj, name, args, kwargs): + args = (hex(id(obj)), time.time(), obj.__class__.__name__, name, + ', '.join([repr(a) for a in args])) + tmpl = '@%s %r - %s.%s(%s)' + if kwargs: + args += (', '.join(['%s=%r' % (k, v) for k, v in kwargs.items()]),) + tmpl = '%s - %s@%s.%s(%s, %s)' + + print(tmpl % args) -def tw_print_hook(name, a, kw): - print time.time(), name, a, kw - - -def wrap_trace(obj, which=None): +def wrap_trace(obj, hook=wt_print_hook, which=None): # other actions: pdb.set_trace, print, aggregate, aggregate_return # (like aggregate but with the return value) Q: should aggregate # includ self? @@ -88,10 +99,10 @@ def wrap_trace(obj, which=None): else: # if callable(which): which_func = which - def wrap_method(attr_name, func): + def wrap_method(attr_name, func, _hook=hook): def wrapped(*a, **kw): a = a[1:] - print (time.time(), attr_name, a, kw) + hook(obj, attr_name, a, kw) return func(*a, **kw) wrapped.__name__ = func.__name__ wrapped.__doc__ = func.__doc__ @@ -108,7 +119,7 @@ def wrap_trace(obj, which=None): def wrap_getattribute(): def __getattribute__(self, attr_name): - print (time.time(), '__getattribute__', (attr_name,)) + hook(obj, '__getattribute__', (attr_name,), {}) ret = type(obj).__getattribute__(obj, attr_name) if callable(ret): ret = type(obj).__getattribute__(self, attr_name) diff --git a/tests/test_debugutils_trace.py b/tests/test_debugutils_trace.py index a69708b..5960bec 100644 --- a/tests/test_debugutils_trace.py +++ b/tests/test_debugutils_trace.py @@ -26,13 +26,13 @@ def test_trace_dict(): return -def test_trace_str(): +def test_trace_bytes(): target = u'Hello'.encode('ascii') wrapped = wrap_trace(target) assert target is not wrapped - assert isinstance(wrapped, str) + assert isinstance(wrapped, bytes) assert len(wrapped) == len(target) assert wrapped.decode('utf-8') == u'Hello' From a4409e8f88451e8365efb6ff15322b83ef8c1951 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Fri, 20 May 2016 19:13:22 -0700 Subject: [PATCH 3/5] a few trace TODOs --- TODO.rst | 3 +++ boltons/debugutils.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/TODO.rst b/TODO.rst index 253eca0..037679c 100644 --- a/TODO.rst +++ b/TODO.rst @@ -29,6 +29,9 @@ misc? - wrap_trace debug utility. Takes an object, looks at its dir, wraps everything callable, with a hook. Needs an enable/disable flag. + - get/set/call/return/exception + - __slots__ + - Tracking proxy. An object that always succeeds for all operations, saving the call history. - Top/Bottom singletons (greater than and less than everything) diff --git a/boltons/debugutils.py b/boltons/debugutils.py index d03ebd7..c0514a7 100644 --- a/boltons/debugutils.py +++ b/boltons/debugutils.py @@ -92,7 +92,9 @@ def wrap_trace(obj, hook=wt_print_hook, which=None): # TODO: how to handle creating the instance # Specifically, getting around the namedtuple problem - # TODO: test classmethod/staticmethod + # TODO: test classmethod/staticmethod/property + # TODO: label for object + # TODO: wrap __dict__ for old-style classes? if isinstance(which, basestring): which_func = lambda attr_name, attr_val: attr_name == which From 1cd0828d07036ed48c7d3dcb0be911e1f10b16c6 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Mon, 23 May 2016 00:46:10 -0700 Subject: [PATCH 4/5] filtration, events, and better output for wrap_trace --- boltons/debugutils.py | 129 ++++++++++++++++++++++++++++++++---------- 1 file changed, 99 insertions(+), 30 deletions(-) diff --git a/boltons/debugutils.py b/boltons/debugutils.py index c0514a7..f0678ba 100644 --- a/boltons/debugutils.py +++ b/boltons/debugutils.py @@ -5,12 +5,21 @@ applications. Currently this focuses on ways to use :mod:`pdb`, the built-in Python debugger. """ +import sys import time try: basestring + from repr import Repr except NameError: basestring = (str, bytes) # py3 + from reprlib import Repr + +try: + from typeutils import make_sentinel + _UNSET = make_sentinel(var_name='_UNSET') +except ImportError: + _UNSET = object() __all__ = ['pdb_on_signal', 'pdb_on_exception'] @@ -73,39 +82,83 @@ def pdb_on_exception(limit=100): sys.excepthook = pdb_excepthook return - -def wt_print_hook(obj, name, args, kwargs): - args = (hex(id(obj)), time.time(), obj.__class__.__name__, name, - ', '.join([repr(a) for a in args])) - tmpl = '@%s %r - %s.%s(%s)' - if kwargs: - args += (', '.join(['%s=%r' % (k, v) for k, v in kwargs.items()]),) - tmpl = '%s - %s@%s.%s(%s, %s)' - - print(tmpl % args) +_repr_obj = Repr() +_repr_obj.maxstring = 50 +_repr_obj.maxother = 50 +brief_repr = _repr_obj.repr -def wrap_trace(obj, hook=wt_print_hook, which=None): +# events: call, return, get, set, del, raise +def wt_print_hook(event, label, obj, attr_name, + args=(), kwargs={}, result=_UNSET): + fargs = (event.ljust(6), time.time(), label.rjust(10), + obj.__class__.__name__, attr_name) + if event == 'get': + tmpl = '%s %s - %s - %s.%s -> %s' + fargs += (brief_repr(result),) + elif event == 'set': + tmpl = '%s %s - %s - %s.%s = %s' + fargs += (brief_repr(args[0]),) + elif event == 'del': + tmpl = '%s %s - %s - %s.%s' + else: # call/return/raise + tmpl = '%s %s - %s - %s.%s(%s)' + fargs += (', '.join([brief_repr(a) for a in args]),) + if kwargs: + tmpl = '%s %s - %s - %s.%s(%s, %s)' + fargs += (', '.join(['%s=%s' % (k, brief_repr(v)) + for k, v in kwargs.items()]),) + if result is not _UNSET: + tmpl += ' -> %s' + fargs += (brief_repr(result),) + print(tmpl % fargs) + return + + +def wrap_trace(obj, hook=wt_print_hook, which=None, events=None, label=None): # other actions: pdb.set_trace, print, aggregate, aggregate_return # (like aggregate but with the return value) Q: should aggregate # includ self? - # TODO: how to handle creating the instance - # Specifically, getting around the namedtuple problem # TODO: test classmethod/staticmethod/property - # TODO: label for object # TODO: wrap __dict__ for old-style classes? if isinstance(which, basestring): which_func = lambda attr_name, attr_val: attr_name == which else: # if callable(which): which_func = which + label = label or hex(id(obj)) - def wrap_method(attr_name, func, _hook=hook): + if isinstance(events, basestring): + events = [events] + do_get = not events or 'get' in events + do_set = not events or 'set' in events + do_del = not events or 'del' in events + do_call = not events or 'call' in events + do_raise = not events or 'raise' in events + do_return = not events or 'return' in events + + def wrap_method(attr_name, func, _hook=hook, _label=label): def wrapped(*a, **kw): a = a[1:] - hook(obj, attr_name, a, kw) - return func(*a, **kw) + if do_call: + hook(event='call', label=_label, obj=obj, + attr_name=attr_name, args=a, kwargs=kw) + if do_raise: + try: + ret = func(*a, **kw) + except: + if not hook(event='raise', label=_label, obj=obj, + attr_name=attr_name, args=a, kwargs=kw, + result=sys.exc_info()): + raise + else: + ret = func(*a, **kw) + if do_return: + hook(event='return', label=_label, obj=obj, + attr_name=attr_name, args=a, kwargs=kw, result=ret) + return ret + wrapped.__name__ = func.__name__ wrapped.__doc__ = func.__doc__ try: @@ -119,17 +172,37 @@ def wrap_trace(obj, hook=wt_print_hook, which=None): pass return wrapped - def wrap_getattribute(): - def __getattribute__(self, attr_name): - hook(obj, '__getattribute__', (attr_name,), {}) - ret = type(obj).__getattribute__(obj, attr_name) - if callable(ret): - ret = type(obj).__getattribute__(self, attr_name) - return ret - return __getattribute__ + def __getattribute__(self, attr_name): + ret = type(obj).__getattribute__(obj, attr_name) + if callable(ret): # wrap any bound methods + ret = type(obj).__getattribute__(self, attr_name) + if do_get: + hook('get', label, obj, attr_name, (), {}, result=ret) + return ret + + def __setattr__(self, attr_name, value): + type(obj).__setattr__(obj, attr_name, value) + if do_set: + hook('set', label, obj, attr_name, (value,), {}) + return + + def __delattr__(self, attr_name): + type(obj).__delattr__(obj, attr_name) + if do_del: + hook('del', label, obj, attr_name, (), {}) + return attrs = {} for attr_name in dir(obj): + if attr_name == '__getattribute__': + attrs[attr_name] = __getattribute__ + continue + elif attr_name == '__setattr__': + attrs[attr_name] = __setattr__ + continue + elif attr_name == '__delattr__': + attrs[attr_name] = __delattr__ + continue try: attr_val = getattr(obj, attr_name) except Exception: @@ -140,11 +213,7 @@ def wrap_trace(obj, hook=wt_print_hook, which=None): elif which_func and not which_func(attr_name, attr_val): continue - if attr_name == '__getattribute__': - wrapped_method = wrap_getattribute() - else: - wrapped_method = wrap_method(attr_name, attr_val) - + wrapped_method = wrap_method(attr_name, attr_val) attrs[attr_name] = wrapped_method cls_name = obj.__class__.__name__ From c1699cc8cdc134df52ede405b8ac048a1ee56be1 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Mon, 23 May 2016 01:11:26 -0700 Subject: [PATCH 5/5] wrap_trace docstring and other minor tweaks and error checking --- boltons/debugutils.py | 64 +++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/boltons/debugutils.py b/boltons/debugutils.py index f0678ba..a8605ab 100644 --- a/boltons/debugutils.py +++ b/boltons/debugutils.py @@ -89,8 +89,8 @@ brief_repr = _repr_obj.repr # events: call, return, get, set, del, raise -def wt_print_hook(event, label, obj, attr_name, - args=(), kwargs={}, result=_UNSET): +def trace_print_hook(event, label, obj, attr_name, + args=(), kwargs={}, result=_UNSET): fargs = (event.ljust(6), time.time(), label.rjust(10), obj.__class__.__name__, attr_name) if event == 'get': @@ -115,18 +115,48 @@ def wt_print_hook(event, label, obj, attr_name, return -def wrap_trace(obj, hook=wt_print_hook, which=None, events=None, label=None): +def wrap_trace(obj, hook=trace_print_hook, + which=None, events=None, label=None): + """Monitor an object for interactions. Whenever code calls a method, + gets an attribute, or sets an attribute, an event is called. By + default the trace output is printed, but a custom tracing *hook* + can be passed. + + Args: + obj (object): New- or old-style object to be traced. Built-in + objects like lists and dicts also supported. + hook (callable): A function called once for every event. See + below for details. + which (str): One or more attribute names to trace, or a + function accepting attribute name and value, and returing + True/False. + events (str): One or more kinds of events to call *hook* + on. Expected values are ``['get', 'set', 'del', 'call', + 'raise', 'return']``. Defaults to all events. + label (str): A name to associate with the traced object + Defaults to hexadecimal memory address, similar to repr. + + The object returned is not the same object as the one passed + in. It will not pass identity checks. However, it will pass + :func:`isinstance` checks, as it is a new instance of a new + subtype of the object passed. + + """ # other actions: pdb.set_trace, print, aggregate, aggregate_return - # (like aggregate but with the return value) Q: should aggregate - # includ self? + # (like aggregate but with the return value) # TODO: test classmethod/staticmethod/property # TODO: wrap __dict__ for old-style classes? if isinstance(which, basestring): which_func = lambda attr_name, attr_val: attr_name == which - else: # if callable(which): + elif callable(getattr(which, '__contains__', None)): + which_func = lambda attr_name, attr_val: attr_name in which + elif which is None or callable(which): which_func = which + else: + raise TypeError('expected attr name(s) or callable, not: %r' % which) + label = label or hex(id(obj)) if isinstance(events, basestring): @@ -194,15 +224,6 @@ def wrap_trace(obj, hook=wt_print_hook, which=None, events=None, label=None): attrs = {} for attr_name in dir(obj): - if attr_name == '__getattribute__': - attrs[attr_name] = __getattribute__ - continue - elif attr_name == '__setattr__': - attrs[attr_name] = __setattr__ - continue - elif attr_name == '__delattr__': - attrs[attr_name] = __delattr__ - continue try: attr_val = getattr(obj, attr_name) except Exception: @@ -213,14 +234,21 @@ def wrap_trace(obj, hook=wt_print_hook, which=None, events=None, label=None): elif which_func and not which_func(attr_name, attr_val): continue - wrapped_method = wrap_method(attr_name, attr_val) + if attr_name == '__getattribute__': + wrapped_method = __getattribute__ + elif attr_name == '__setattr__': + wrapped_method = __setattr__ + elif attr_name == '__delattr__': + wrapped_method = __delattr__ + else: + wrapped_method = wrap_method(attr_name, attr_val) attrs[attr_name] = wrapped_method cls_name = obj.__class__.__name__ if cls_name == cls_name.lower(): - type_name = 'wrapped_' + cls_name + type_name = 'traced_' + cls_name else: - type_name = 'Wrapped' + cls_name + type_name = 'Traced' + cls_name if hasattr(obj, '__mro__'): bases = (obj.__class__,)