From 63ee222406af499c082411ab5bcdab9597ea40cd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 14 Aug 2016 00:16:17 +0100 Subject: [PATCH] Many docs updates. --- docs/api.rst | 70 ++++++++++++++++++++++++++++++++++++++++---- docs/howitworks.rst | 52 ++++++++++++++++++++++++++++---- docs/index.rst | 17 ++++++----- econtext/__init__.py | 26 ++++++++++++++++ econtext/core.py | 2 +- econtext/utils.py | 18 ++++++++++++ 6 files changed, 165 insertions(+), 20 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 1f68a45b..16baf822 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,19 +1,79 @@ API Reference -============= +************* + + +econtext Package +================ + +.. automodule:: econtext + +.. autodata:: econtext.slave econtext.core -############# +============= .. automodule:: econtext.core + + +Exceptions +---------- + +.. autoclass:: econtext.core.Error +.. autoclass:: econtext.core.CallError +.. autoclass:: econtext.core.ChannelError +.. autoclass:: econtext.core.StreamError +.. autoclass:: econtext.core.TimeoutError + + +Context Class +------------- + +.. autoclass:: econtext.core.Context + :members: + + +Channel Class +------------- + +.. autoclass:: econtext.core.Channel :members: - :undoc-members: econtext.master -############### +=============== .. automodule:: econtext.master + + +Helper Functions +---------------- + +.. autofunction:: econtext.master.create_child +.. autofunction:: econtext.master.get_child_modules +.. autofunction:: econtext.master.minimize_source + + +Context Class +------------- + +.. autoclass:: econtext.master.Context :members: - :undoc-members: + + +Stream Classes +-------------- + +.. autoclass:: econtext.master.LocalStream + :members: + +.. autoclass:: econtext.master.SSHStream + :members: + + +econtext.utils +============== + +.. automodule:: econtext.utils + :members: diff --git a/docs/howitworks.rst b/docs/howitworks.rst index c7a0d75a..1e0bb6dc 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -93,31 +93,61 @@ bootstrap. After the script source code is prepared, it is passed through :py:func:`econtext.master.minimize_source` to strip it of docstrings and -comments, while preserving original line numbers. This reduces the compressed -payload size by around 20%. +comments, while preserving line numbers. This reduces the compressed payload +by around 20%. Signalling Success ################## +Once the first stage has decompressed and written the bootstrap source code to +its parent Python interpreter, it writes the string ``OK\n`` to ``stdout`` +before exitting. The master process waits for this string before considering +bootstrap successful and the child's ``stdio`` ready to receive messages. -ExternalContext main() + +ExternalContext.main() ---------------------- - -Reaping The First Stage -####################### +.. automethod:: econtext.core.ExternalContext.main Generating A Synthetic `econtext` Package ######################################### +Since the bootstrap consists of the :py:mod:`econtext.core` source code, and +this code is loaded by Python by way of its main script (``__main__`` module), +initially the module layout in the slave will be incorrect. + +The first step taken after bootstrap is to rearrange ``sys.modules`` slightly +so that :py:mod:`econtext.core` appears in the correct location, and all +classes defined in that module have their ``__module__`` attribute fixed up +such that :py:mod:`cPickle` correctly serializes instance module names. + +Once a synthetic :py:mod:`econtext` package and :py:mod:`econtext.core` module +have been generated, the bootstrap **deletes** `sys.modules['__main__']`, so +that any attempt to import it (by :py:mod:`cPickle`) will cause the import to +be satisfied by fetching the econtext master's actual ``__main__`` module. This +is necessary to allow master programs to be written as a self-contained Python +script. + Setup The Broker And Master Context ################################### +Reaping The First Stage +####################### + +After the bootstrap has called :py:func:`os.dup` on the copy of the ``stdin`` +file descriptor saved by the first stage, it is closed. + +Additionally, since the first stage was forked prior to re-executing the Python +interpreter, it will exist as a zombie process until the parent process reaps +it. Therefore the bootstrap must call :py:func:`os.wait` soon after startup. + + Setup Logging ############# @@ -133,12 +163,22 @@ Standard IO Redirection Function Call Dispatch ###################### +After all initialization is complete, the slave's main thread sits in a loop +reading from a :py:class:`Channel ` connected to the +``CALL_FUNCTION`` handle. This handle is written to by +:py:meth:`call_with_deadline() ` and +:py:meth:`call() `. + Stream Protocol --------------- +Use of Pickle +############# + + Use of HMAC ########### diff --git a/docs/index.rst b/docs/index.rst index 8872f10f..4b833b6a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,7 +5,7 @@ Python Execution Contexts **4.98KiB of sugar and no fat!** .. toctree:: - :maxdepth: 1 + :maxdepth: 2 self howitworks @@ -32,8 +32,8 @@ and efficient low-level API on which tools like **Salt** or **Ansible** can be built, and while the API is quite friendly and similar in scope to **Fabric**, ultimately it should not be used directly by consumer software. -The primary focus is to centralize and perfect the intricate dance required to -run Python code safely and efficiently on a remote machine, while avoiding +The focus is to centralize and perfect the intricate dance required to run +Python code safely and efficiently on a remote machine, while avoiding temporary files or large chunks of error-prone shell scripts. @@ -45,7 +45,8 @@ communicate with new Python programs under its control running on remote machines, **using only an existing installed Python interpreter and SSH client**, something that by default can be found on almost all contemporary machines in the wild. To accomplish bootstrap, econtext uses a single 500 byte -SSH command line and 5KB of data sent to stdin of the remote SSH connection. +SSH command line and 5KB of its own source code sent to stdin of the remote SSH +connection. .. code:: @@ -107,9 +108,9 @@ configuration. Logging Forwarder ################# -The 5KB bootstrap configures the remote process's Python logging package to -forward all logs back to the local process, enabling management of program logs -in one location. +The bootstrap configures the remote process's Python logging package to forward +all logs back to the local process, enabling management of program logs in one +location. .. code:: @@ -120,7 +121,7 @@ in one location. Stdio Forwarder ############### -To ease porting of crusty old infrastructure code to pure Python, the bootstrap +To ease porting of crusty old infrastructure scripts to Python, the bootstrap redirects stdio for itself and any child processes back into the logging framework. This allows use of functions as basic as **os.system('hostname; uptime')** without further need to capture or manage output. diff --git a/econtext/__init__.py b/econtext/__init__.py index 37031917..576bb09d 100644 --- a/econtext/__init__.py +++ b/econtext/__init__.py @@ -1,2 +1,28 @@ +""" +On the econtext master, this is imported from ``econtext/__init__.py`` as would +be expected. On the slave, it is built dynamically during startup. + +As a convenience, the econtext package exports all of the functions and +variables from :py:mod:`econtext.core`. +""" + +#: This is ``True`` in slave contexts. It is used in single-file Python +#: programs to avoid reexecuting the program's :py:func:`main` function in the +#: slave. For example: +#: +#: .. code-block:: python +#: +#: def do_work(): +#: os.system('hostname') +#: +#: def main(broker): +#: context = broker.get_local() +#: context.call(do_work) # Causes slave to import __main__. +#: +#: if __name__ == '__main__' and not econtext.slave: +#: import econtext.utils +#: econtext.utils.run_with_broker(main) +#: slave = False + from econtext.core import * # NOQA diff --git a/econtext/core.py b/econtext/core.py index 714520d1..57f2ad16 100644 --- a/econtext/core.py +++ b/econtext/core.py @@ -42,7 +42,7 @@ class Error(Exception): class CallError(Error): - """Raised when .call() fails""" + """Raised when .call() fails.""" def __init__(self, e): name = '%s.%s' % (type(e).__module__, type(e).__name__) tb = sys.exc_info()[2] diff --git a/econtext/utils.py b/econtext/utils.py index 9305916b..c355f1ce 100644 --- a/econtext/utils.py +++ b/econtext/utils.py @@ -1,3 +1,6 @@ +""" +A random assortment of utility functions useful on masters and slaves. +""" import logging @@ -7,6 +10,8 @@ import econtext.master def log_to_file(path, level=logging.DEBUG): + """Install a new :py:class:`logging.Handler` writing applications logs to + the filesystem. Useful when debugging slave IO problems.""" log = logging.getLogger('') fp = open(path, 'w', 1) econtext.core.set_cloexec(fp.fileno()) @@ -15,6 +20,9 @@ def log_to_file(path, level=logging.DEBUG): def run_with_broker(func, *args, **kwargs): + """Arrange for `func(broker, *args, **kwargs)` to run with a temporary + :py:class:`econtext.master.Broker`, ensuring the broker is correctly + shut down during normal or exceptional return.""" broker = econtext.master.Broker() try: return func(broker, *args, **kwargs) @@ -24,6 +32,16 @@ def run_with_broker(func, *args, **kwargs): def with_broker(func): + """Decorator version of :py:func:`run_with_broker`. Example: + + .. code-block:: python + + @with_broker + def do_stuff(broker, arg): + pass + + do_stuff(blah, 123) + """ def wrapper(*args, **kwargs): return run_with_broker(*args, **kwargs) wrapper.func_name = func.func_name