From e856b7fc789b018e1d35988c731326f02bfaa525 Mon Sep 17 00:00:00 2001 From: Jack O'Connor Date: Tue, 13 Jan 2015 17:22:43 -0800 Subject: [PATCH] use `sync` instead of `fetch` in the plugin interface Summary: This makes the plugin interface easier to understand. There are lots of instances where we still use "fetch" to describe what's going on, but this diff at least gets all the meaningful constants in the interface. Reviewers: sean Reviewed By: sean Subscribers: sean Differential Revision: https://phabricator.buildinspace.com/D170 --- README.md | 2 +- docs/architecture.md | 14 +++++++------- peru/plugin.py | 16 ++++++++-------- peru/resources/plugins/cp/cp_plugin.py | 2 +- peru/resources/plugins/cp/plugin.yaml | 2 +- peru/resources/plugins/curl/curl_plugin.py | 8 ++++---- peru/resources/plugins/curl/plugin.yaml | 2 +- peru/resources/plugins/empty/plugin.yaml | 2 +- peru/resources/plugins/git/git_plugin.py | 8 ++++---- peru/resources/plugins/git/plugin.yaml | 2 +- peru/resources/plugins/hg/hg_plugin.py | 8 ++++---- peru/resources/plugins/hg/plugin.yaml | 2 +- peru/resources/plugins/noop_cache/plugin.yaml | 2 +- peru/resources/plugins/print/plugin.yaml | 2 +- peru/resources/plugins/rsync/plugin.yaml | 2 +- peru/resources/plugins/rsync/rsync_plugin.sh | 2 +- peru/resources/plugins/svn/plugin.yaml | 2 +- peru/resources/plugins/svn/svn_plugin.py | 8 ++++---- tests/test_plugins.py | 2 +- 19 files changed, 44 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 343ecfa..c3e7adf 100644 --- a/README.md +++ b/README.md @@ -227,7 +227,7 @@ example implemented in Bash. ## Creating New Module Types Module type plugins are as-dumb-as-possible scripts that only know how to -fetch, and optionally reup. Peru shells out to them and then handles most of +sync, and optionally reup. Peru shells out to them and then handles most of the caching magic itself, though plugins can also do their own caching as appropriate. For example, the git and hg plugins keep track of repos they clone. Peru itself doesn't need to know how to do that. For all the details, diff --git a/docs/architecture.md b/docs/architecture.md index e386c3b..e76e172 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -52,7 +52,7 @@ type's fields, and how the executable(s) should be invoked. Here's the `plugin.yaml` from `peru/resources/plugins/git`: ```yaml -fetch exe: git_plugin.py +sync exe: git_plugin.py reup exe: git_plugin.py required fields: - url @@ -65,10 +65,10 @@ cache fields: - The name of the plugin directory determines the name of the module type. -- `fetch exe` is required, and it tells peru what to execute when it - wants the plugin to fetch. +- `sync exe` is required, and it tells peru what to execute when it + wants the plugin to sync. - `reup exe` is optional; it declares that the plugin supports reup and - how to execute it. This can be the same script as `fetch exe`, as it + how to execute it. This can be the same script as `sync exe`, as it is here, in which case the script should decide what to do based on the `PERU_PLUGIN_COMMAND` environment variable described below. - `required fields` is required, and it tells peru which fields are @@ -89,7 +89,7 @@ The other part of a plugin definition is the executable script(s). These are invoked with no arguments, and several environment variables are defined to tell the plugin what to do: -- `PERU_PLUGIN_COMMAND` is either `fetch` or `reup`, depending on what +- `PERU_PLUGIN_COMMAND` is either `sync` or `reup`, depending on what peru needs the plugin to do. - `PERU_PLUGIN_CACHE` points to the plugin's cache directory. If `plugin.yaml` doesn't include `cache fields`, this path will be @@ -100,8 +100,8 @@ defined to tell the plugin what to do: form. For example, the git plugin gets its `url` field as `PERU_MODULE_URL`. The variables for optional fields that aren't present in the module are defined but empty. -- `PERU_FETCH_DEST` points to the temporary directory where the plugin - should put the files it downloads. This is only defined for fetch +- `PERU_SYNC_DEST` points to the temporary directory where the plugin + should put the files it downloads. This is only defined for sync jobs. - `PERU_REUP_OUTPUT` points to the temporary file where the plugin should write updated field values, formatted as YAML. This is only diff --git a/peru/plugin.py b/peru/plugin.py index 9706078..f6df9ee 100644 --- a/peru/plugin.py +++ b/peru/plugin.py @@ -21,7 +21,7 @@ DEBUG_PARALLEL_MAX = 0 PluginDefinition = namedtuple( 'PluginDefinition', - ['type', 'fetch_exe', 'reup_exe', 'fields', 'required_fields', + ['type', 'sync_exe', 'reup_exe', 'fields', 'required_fields', 'optional_fields', 'cache_fields']) PluginContext = namedtuple( @@ -33,8 +33,8 @@ PluginContext = namedtuple( @asyncio.coroutine def plugin_fetch(plugin_context, module_type, module_fields, dest, display_handle): - env = {'PERU_FETCH_DEST': dest} - yield from _plugin_job(plugin_context, module_type, module_fields, 'fetch', + env = {'PERU_SYNC_DEST': dest} + yield from _plugin_job(plugin_context, module_type, module_fields, 'sync', env, display_handle) @@ -112,8 +112,8 @@ def _plugin_job(plugin_context, module_type, module_fields, command, env, def _get_plugin_exe(definition, command): - if command == 'fetch': - exe = definition.fetch_exe + if command == 'sync': + exe = definition.sync_exe elif command == 'reup': exe = definition.reup_exe else: @@ -170,7 +170,7 @@ def _plugin_env(plugin_context, plugin_definition, module_fields, command, # be careful about this. env['PYTHONUNBUFFERED'] = 'true' - # For plugins that use the same exe for fetch and reup, make the command + # For plugins that use the same exe for sync and reup, make the command # name available in the environment. env['PERU_PLUGIN_COMMAND'] = command @@ -227,7 +227,7 @@ def _get_plugin_definition(module_type, module_fields, command): # Read the metadata document. with open(metadata_path) as metafile: metadoc = yaml.safe_load(metafile) or {} - fetch_exe = os.path.join(root, metadoc.pop('fetch exe')) + sync_exe = os.path.join(root, metadoc.pop('sync exe')) reup_exe = (None if 'reup exe' not in metadoc else os.path.join(root, metadoc.pop('reup exe'))) required_fields = frozenset(metadoc.pop('required fields')) @@ -249,7 +249,7 @@ def _get_plugin_definition(module_type, module_fields, command): str(invalid)) definition = PluginDefinition( - module_type, fetch_exe, reup_exe, fields, required_fields, + module_type, sync_exe, reup_exe, fields, required_fields, optional_fields, cache_fields) _validate_plugin_definition(definition, module_fields) return definition diff --git a/peru/resources/plugins/cp/cp_plugin.py b/peru/resources/plugins/cp/cp_plugin.py index ce528aa..7a829da 100755 --- a/peru/resources/plugins/cp/cp_plugin.py +++ b/peru/resources/plugins/cp/cp_plugin.py @@ -6,5 +6,5 @@ import os distutils.dir_util.copy_tree( os.environ['PERU_MODULE_PATH'], - os.environ['PERU_FETCH_DEST'], + os.environ['PERU_SYNC_DEST'], preserve_symlinks=True) diff --git a/peru/resources/plugins/cp/plugin.yaml b/peru/resources/plugins/cp/plugin.yaml index b78670e..63f2b5f 100644 --- a/peru/resources/plugins/cp/plugin.yaml +++ b/peru/resources/plugins/cp/plugin.yaml @@ -1,3 +1,3 @@ -fetch exe: cp_plugin.py +sync exe: cp_plugin.py required fields: - path diff --git a/peru/resources/plugins/curl/curl_plugin.py b/peru/resources/plugins/curl/curl_plugin.py index 66f633c..4104fd6 100755 --- a/peru/resources/plugins/curl/curl_plugin.py +++ b/peru/resources/plugins/curl/curl_plugin.py @@ -67,9 +67,9 @@ def download_file(request, output_file, stdout=sys.stdout): return digest.hexdigest() -def plugin_fetch(url, sha1): +def plugin_sync(url, sha1): unpack = os.environ['PERU_MODULE_UNPACK'] - dest = os.environ['PERU_FETCH_DEST'] + dest = os.environ['PERU_SYNC_DEST'] if unpack: # Download to the tmp dir for later unpacking. download_dir = os.environ['PERU_PLUGIN_TMP'] @@ -123,8 +123,8 @@ def main(): url = os.environ['PERU_MODULE_URL'] sha1 = os.environ['PERU_MODULE_SHA1'] command = os.environ['PERU_PLUGIN_COMMAND'] - if command == 'fetch': - plugin_fetch(url, sha1) + if command == 'sync': + plugin_sync(url, sha1) elif command == 'reup': plugin_reup(url, sha1) else: diff --git a/peru/resources/plugins/curl/plugin.yaml b/peru/resources/plugins/curl/plugin.yaml index 94477d5..ed80f63 100644 --- a/peru/resources/plugins/curl/plugin.yaml +++ b/peru/resources/plugins/curl/plugin.yaml @@ -1,4 +1,4 @@ -fetch exe: curl_plugin.py +sync exe: curl_plugin.py reup exe: curl_plugin.py required fields: - url diff --git a/peru/resources/plugins/empty/plugin.yaml b/peru/resources/plugins/empty/plugin.yaml index 804763e..8eb3cd5 100644 --- a/peru/resources/plugins/empty/plugin.yaml +++ b/peru/resources/plugins/empty/plugin.yaml @@ -1,2 +1,2 @@ -fetch exe: empty_plugin.py +sync exe: empty_plugin.py required fields: [] diff --git a/peru/resources/plugins/git/git_plugin.py b/peru/resources/plugins/git/git_plugin.py index cd4a4cd..12f3acf 100755 --- a/peru/resources/plugins/git/git_plugin.py +++ b/peru/resources/plugins/git/git_plugin.py @@ -124,8 +124,8 @@ def checkout_subrepos(repo_path, rev, work_tree): checkout_tree(sub_url, sub_rev, sub_full_path) -def plugin_fetch(): - checkout_tree(URL, REV, os.environ['PERU_FETCH_DEST']) +def plugin_sync(): + checkout_tree(URL, REV, os.environ['PERU_SYNC_DEST']) def plugin_reup(): @@ -137,8 +137,8 @@ def plugin_reup(): print('rev:', output.strip(), file=out_file) command = os.environ['PERU_PLUGIN_COMMAND'] -if command == 'fetch': - plugin_fetch() +if command == 'sync': + plugin_sync() elif command == 'reup': plugin_reup() else: diff --git a/peru/resources/plugins/git/plugin.yaml b/peru/resources/plugins/git/plugin.yaml index 4be4283..c207a1f 100644 --- a/peru/resources/plugins/git/plugin.yaml +++ b/peru/resources/plugins/git/plugin.yaml @@ -1,4 +1,4 @@ -fetch exe: git_plugin.py +sync exe: git_plugin.py reup exe: git_plugin.py required fields: - url diff --git a/peru/resources/plugins/hg/hg_plugin.py b/peru/resources/plugins/hg/hg_plugin.py index c598297..a6acee7 100755 --- a/peru/resources/plugins/hg/hg_plugin.py +++ b/peru/resources/plugins/hg/hg_plugin.py @@ -82,8 +82,8 @@ def already_has_rev(repo, rev): return output.split()[0] == rev -def plugin_fetch(): - dest = os.environ['PERU_FETCH_DEST'] +def plugin_sync(): + dest = os.environ['PERU_SYNC_DEST'] clone_if_needed(URL, verbose=True) if not already_has_rev(CACHE_PATH, REV): hg_pull(URL, CACHE_PATH) @@ -104,8 +104,8 @@ def plugin_reup(): command = os.environ['PERU_PLUGIN_COMMAND'] -if command == 'fetch': - plugin_fetch() +if command == 'sync': + plugin_sync() elif command == 'reup': plugin_reup() else: diff --git a/peru/resources/plugins/hg/plugin.yaml b/peru/resources/plugins/hg/plugin.yaml index 4c41026..6721be7 100644 --- a/peru/resources/plugins/hg/plugin.yaml +++ b/peru/resources/plugins/hg/plugin.yaml @@ -1,4 +1,4 @@ -fetch exe: hg_plugin.py +sync exe: hg_plugin.py reup exe: hg_plugin.py required fields: - url diff --git a/peru/resources/plugins/noop_cache/plugin.yaml b/peru/resources/plugins/noop_cache/plugin.yaml index b261416..a0beae5 100644 --- a/peru/resources/plugins/noop_cache/plugin.yaml +++ b/peru/resources/plugins/noop_cache/plugin.yaml @@ -1,6 +1,6 @@ # This plugin is for testing purposes. The cache dir is not used. The nonce is # there to let us change module fields without changing plugin cache fields. -fetch exe: noop_cache_plugin.py +sync exe: noop_cache_plugin.py required fields: - path optional fields: diff --git a/peru/resources/plugins/print/plugin.yaml b/peru/resources/plugins/print/plugin.yaml index 922f77f..4ca0589 100644 --- a/peru/resources/plugins/print/plugin.yaml +++ b/peru/resources/plugins/print/plugin.yaml @@ -1,3 +1,3 @@ -fetch exe: print_plugin.py +sync exe: print_plugin.py required fields: - nonce diff --git a/peru/resources/plugins/rsync/plugin.yaml b/peru/resources/plugins/rsync/plugin.yaml index 1746213..fe96027 100644 --- a/peru/resources/plugins/rsync/plugin.yaml +++ b/peru/resources/plugins/rsync/plugin.yaml @@ -1,3 +1,3 @@ -fetch exe: rsync_plugin.sh +sync exe: rsync_plugin.sh required fields: - path diff --git a/peru/resources/plugins/rsync/rsync_plugin.sh b/peru/resources/plugins/rsync/rsync_plugin.sh index a5320c7..3fe82bf 100755 --- a/peru/resources/plugins/rsync/rsync_plugin.sh +++ b/peru/resources/plugins/rsync/rsync_plugin.sh @@ -17,4 +17,4 @@ fi # Do the copy. Always append a trailing slash to the path, so that the # contents are copied rather than the directory itself. -rsync -r "$PERU_MODULE_PATH/" "$PERU_FETCH_DEST" +rsync -r "$PERU_MODULE_PATH/" "$PERU_SYNC_DEST" diff --git a/peru/resources/plugins/svn/plugin.yaml b/peru/resources/plugins/svn/plugin.yaml index 1c95a75..45464cc 100644 --- a/peru/resources/plugins/svn/plugin.yaml +++ b/peru/resources/plugins/svn/plugin.yaml @@ -1,4 +1,4 @@ -fetch exe: svn_plugin.py +sync exe: svn_plugin.py reup exe: svn_plugin.py required fields: - url diff --git a/peru/resources/plugins/svn/svn_plugin.py b/peru/resources/plugins/svn/svn_plugin.py index 4578461..83a121a 100755 --- a/peru/resources/plugins/svn/svn_plugin.py +++ b/peru/resources/plugins/svn/svn_plugin.py @@ -39,7 +39,7 @@ def remote_head_rev(url): sys.exit(1) -def plugin_fetch(): +def plugin_sync(): # Just fetch the target revision and strip the metadata. # Plugin-level caching for Subversion is futile. svn( @@ -48,7 +48,7 @@ def plugin_fetch(): '--revision', os.environ['PERU_MODULE_REV'] or 'HEAD', os.environ['PERU_MODULE_URL'], - os.environ['PERU_FETCH_DEST']) + os.environ['PERU_SYNC_DEST']) def plugin_reup(): @@ -61,8 +61,8 @@ def plugin_reup(): command = os.environ['PERU_PLUGIN_COMMAND'] -if command == 'fetch': - plugin_fetch() +if command == 'sync': + plugin_sync() elif command == 'reup': plugin_reup() else: diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 4e45e28..e58db3d 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -343,7 +343,7 @@ class PluginsTest(unittest.TestCase): print("name: val", file=open(outfile, 'w')) '''), plugin_yaml_file: textwrap.dedent('''\ - fetch exe: fetch.py + sync exe: fetch.py reup exe: reup.py required fields: [] ''')})