use `sync` instead of `fetch` in the plugin interface

Summary:
This makes the plugin interface easier to understand. There are lots of
instances where we still use "fetch" to describe what's going on, but
this diff at least gets all the meaningful constants in the interface.

Reviewers: sean

Reviewed By: sean

Subscribers: sean

Differential Revision: https://phabricator.buildinspace.com/D170

README.md

@@ -227,7 +227,7 @@ example implemented in Bash.
 
 ## Creating New Module Types
 Module type plugins are as-dumb-as-possible scripts that only know how to
-fetch, and optionally reup. Peru shells out to them and then handles most of
+sync, and optionally reup. Peru shells out to them and then handles most of
 the caching magic itself, though plugins can also do their own caching as
 appropriate.  For example, the git and hg plugins keep track of repos they
 clone. Peru itself doesn't need to know how to do that. For all the details,

docs/architecture.md

@@ -52,7 +52,7 @@ type's fields, and how the executable(s) should be invoked. Here's the
 `plugin.yaml` from `peru/resources/plugins/git`:
 
 ```yaml
-fetch exe: git_plugin.py
+sync exe: git_plugin.py
 reup exe: git_plugin.py
 required fields:
     - url
@@ -65,10 +65,10 @@ cache fields:
 
 - The name of the plugin directory determines the name of the module
   type.
-- `fetch exe` is required, and it tells peru what to execute when it
-  wants the plugin to fetch.
+- `sync exe` is required, and it tells peru what to execute when it
+  wants the plugin to sync.
 - `reup exe` is optional; it declares that the plugin supports reup and
-  how to execute it. This can be the same script as `fetch exe`, as it
+  how to execute it. This can be the same script as `sync exe`, as it
   is here, in which case the script should decide what to do based on
   the `PERU_PLUGIN_COMMAND` environment variable described below.
 - `required fields` is required, and it tells peru which fields are
@@ -89,7 +89,7 @@ The other part of a plugin definition is the executable script(s). These
 are invoked with no arguments, and several environment variables are
 defined to tell the plugin what to do:
 
-- `PERU_PLUGIN_COMMAND` is either `fetch` or `reup`, depending on what
+- `PERU_PLUGIN_COMMAND` is either `sync` or `reup`, depending on what
   peru needs the plugin to do.
 - `PERU_PLUGIN_CACHE` points to the plugin's cache directory. If
   `plugin.yaml` doesn't include `cache fields`, this path will be
@@ -100,8 +100,8 @@ defined to tell the plugin what to do:
   form. For example, the git plugin gets its `url` field as
   `PERU_MODULE_URL`. The variables for optional fields that aren't
   present in the module are defined but empty.
-- `PERU_FETCH_DEST` points to the temporary directory where the plugin
-  should put the files it downloads. This is only defined for fetch
+- `PERU_SYNC_DEST` points to the temporary directory where the plugin
+  should put the files it downloads. This is only defined for sync
   jobs.
 - `PERU_REUP_OUTPUT` points to the temporary file where the plugin
   should write updated field values, formatted as YAML. This is only

peru/plugin.py

@@ -21,7 +21,7 @@ DEBUG_PARALLEL_MAX = 0
 
 PluginDefinition = namedtuple(
     'PluginDefinition',
-    ['type', 'fetch_exe', 'reup_exe', 'fields', 'required_fields',
+    ['type', 'sync_exe', 'reup_exe', 'fields', 'required_fields',
      'optional_fields', 'cache_fields'])
 
 PluginContext = namedtuple(
@@ -33,8 +33,8 @@ PluginContext = namedtuple(
 @asyncio.coroutine
 def plugin_fetch(plugin_context, module_type, module_fields, dest,
                  display_handle):
-    env = {'PERU_FETCH_DEST': dest}
-    yield from _plugin_job(plugin_context, module_type, module_fields, 'fetch',
+    env = {'PERU_SYNC_DEST': dest}
+    yield from _plugin_job(plugin_context, module_type, module_fields, 'sync',
                            env, display_handle)
 
 
@@ -112,8 +112,8 @@ def _plugin_job(plugin_context, module_type, module_fields, command, env,
 
 
 def _get_plugin_exe(definition, command):
-    if command == 'fetch':
-        exe = definition.fetch_exe
+    if command == 'sync':
+        exe = definition.sync_exe
     elif command == 'reup':
         exe = definition.reup_exe
     else:
@@ -170,7 +170,7 @@ def _plugin_env(plugin_context, plugin_definition, module_fields, command,
     # be careful about this.
     env['PYTHONUNBUFFERED'] = 'true'
 
-    # For plugins that use the same exe for fetch and reup, make the command
+    # For plugins that use the same exe for sync and reup, make the command
     # name available in the environment.
     env['PERU_PLUGIN_COMMAND'] = command
 
@@ -227,7 +227,7 @@ def _get_plugin_definition(module_type, module_fields, command):
     # Read the metadata document.
     with open(metadata_path) as metafile:
         metadoc = yaml.safe_load(metafile) or {}
-    fetch_exe = os.path.join(root, metadoc.pop('fetch exe'))
+    sync_exe = os.path.join(root, metadoc.pop('sync exe'))
     reup_exe = (None if 'reup exe' not in metadoc
                 else os.path.join(root, metadoc.pop('reup exe')))
     required_fields = frozenset(metadoc.pop('required fields'))
@@ -249,7 +249,7 @@ def _get_plugin_definition(module_type, module_fields, command):
             str(invalid))
 
     definition = PluginDefinition(
-        module_type, fetch_exe, reup_exe, fields, required_fields,
+        module_type, sync_exe, reup_exe, fields, required_fields,
         optional_fields, cache_fields)
     _validate_plugin_definition(definition, module_fields)
     return definition

peru/resources/plugins/cp/cp_plugin.py

@@ -6,5 +6,5 @@ import os
 
 distutils.dir_util.copy_tree(
     os.environ['PERU_MODULE_PATH'],
-    os.environ['PERU_FETCH_DEST'],
+    os.environ['PERU_SYNC_DEST'],
     preserve_symlinks=True)

peru/resources/plugins/cp/plugin.yaml

@@ -1,3 +1,3 @@
-fetch exe: cp_plugin.py
+sync exe: cp_plugin.py
 required fields:
     - path

peru/resources/plugins/curl/curl_plugin.py

@@ -67,9 +67,9 @@ def download_file(request, output_file, stdout=sys.stdout):
     return digest.hexdigest()
 
 
-def plugin_fetch(url, sha1):
+def plugin_sync(url, sha1):
     unpack = os.environ['PERU_MODULE_UNPACK']
-    dest = os.environ['PERU_FETCH_DEST']
+    dest = os.environ['PERU_SYNC_DEST']
     if unpack:
         # Download to the tmp dir for later unpacking.
         download_dir = os.environ['PERU_PLUGIN_TMP']
@@ -123,8 +123,8 @@ def main():
     url = os.environ['PERU_MODULE_URL']
     sha1 = os.environ['PERU_MODULE_SHA1']
     command = os.environ['PERU_PLUGIN_COMMAND']
-    if command == 'fetch':
-        plugin_fetch(url, sha1)
+    if command == 'sync':
+        plugin_sync(url, sha1)
     elif command == 'reup':
         plugin_reup(url, sha1)
     else:

peru/resources/plugins/curl/plugin.yaml

@@ -1,4 +1,4 @@
-fetch exe: curl_plugin.py
+sync exe: curl_plugin.py
 reup exe: curl_plugin.py
 required fields:
     - url

peru/resources/plugins/empty/plugin.yaml

@@ -1,2 +1,2 @@
-fetch exe: empty_plugin.py
+sync exe: empty_plugin.py
 required fields: []

peru/resources/plugins/git/git_plugin.py

@@ -124,8 +124,8 @@ def checkout_subrepos(repo_path, rev, work_tree):
         checkout_tree(sub_url, sub_rev, sub_full_path)
 
 
-def plugin_fetch():
-    checkout_tree(URL, REV, os.environ['PERU_FETCH_DEST'])
+def plugin_sync():
+    checkout_tree(URL, REV, os.environ['PERU_SYNC_DEST'])
 
 
 def plugin_reup():
@@ -137,8 +137,8 @@ def plugin_reup():
         print('rev:', output.strip(), file=out_file)
 
 command = os.environ['PERU_PLUGIN_COMMAND']
-if command == 'fetch':
-    plugin_fetch()
+if command == 'sync':
+    plugin_sync()
 elif command == 'reup':
     plugin_reup()
 else:

peru/resources/plugins/git/plugin.yaml

@@ -1,4 +1,4 @@
-fetch exe: git_plugin.py
+sync exe: git_plugin.py
 reup exe: git_plugin.py
 required fields:
     - url

peru/resources/plugins/hg/hg_plugin.py

@@ -82,8 +82,8 @@ def already_has_rev(repo, rev):
     return output.split()[0] == rev
 
 
-def plugin_fetch():
-    dest = os.environ['PERU_FETCH_DEST']
+def plugin_sync():
+    dest = os.environ['PERU_SYNC_DEST']
     clone_if_needed(URL, verbose=True)
     if not already_has_rev(CACHE_PATH, REV):
         hg_pull(URL, CACHE_PATH)
@@ -104,8 +104,8 @@ def plugin_reup():
 
 
 command = os.environ['PERU_PLUGIN_COMMAND']
-if command == 'fetch':
-    plugin_fetch()
+if command == 'sync':
+    plugin_sync()
 elif command == 'reup':
     plugin_reup()
 else:

peru/resources/plugins/hg/plugin.yaml

@@ -1,4 +1,4 @@
-fetch exe: hg_plugin.py
+sync exe: hg_plugin.py
 reup exe: hg_plugin.py
 required fields:
     - url

peru/resources/plugins/noop_cache/plugin.yaml

@@ -1,6 +1,6 @@
 # This plugin is for testing purposes. The cache dir is not used. The nonce is
 # there to let us change module fields without changing plugin cache fields.
-fetch exe: noop_cache_plugin.py
+sync exe: noop_cache_plugin.py
 required fields:
     - path
 optional fields:

peru/resources/plugins/print/plugin.yaml

@@ -1,3 +1,3 @@
-fetch exe: print_plugin.py
+sync exe: print_plugin.py
 required fields:
     - nonce

peru/resources/plugins/rsync/plugin.yaml

@@ -1,3 +1,3 @@
-fetch exe: rsync_plugin.sh
+sync exe: rsync_plugin.sh
 required fields:
     - path

peru/resources/plugins/rsync/rsync_plugin.sh

@@ -17,4 +17,4 @@ fi
 
 # Do the copy. Always append a trailing slash to the path, so that the
 # contents are copied rather than the directory itself.
-rsync -r "$PERU_MODULE_PATH/" "$PERU_FETCH_DEST"
+rsync -r "$PERU_MODULE_PATH/" "$PERU_SYNC_DEST"

peru/resources/plugins/svn/plugin.yaml

@@ -1,4 +1,4 @@
-fetch exe: svn_plugin.py
+sync exe: svn_plugin.py
 reup exe: svn_plugin.py
 required fields:
     - url

peru/resources/plugins/svn/svn_plugin.py

@@ -39,7 +39,7 @@ def remote_head_rev(url):
     sys.exit(1)
 
 
-def plugin_fetch():
+def plugin_sync():
     # Just fetch the target revision and strip the metadata.
     # Plugin-level caching for Subversion is futile.
     svn(
@@ -48,7 +48,7 @@ def plugin_fetch():
         '--revision',
         os.environ['PERU_MODULE_REV'] or 'HEAD',
         os.environ['PERU_MODULE_URL'],
-        os.environ['PERU_FETCH_DEST'])
+        os.environ['PERU_SYNC_DEST'])
 
 
 def plugin_reup():
@@ -61,8 +61,8 @@ def plugin_reup():
 
 
 command = os.environ['PERU_PLUGIN_COMMAND']
-if command == 'fetch':
-    plugin_fetch()
+if command == 'sync':
+    plugin_sync()
 elif command == 'reup':
     plugin_reup()
 else:

tests/test_plugins.py

@@ -343,7 +343,7 @@ class PluginsTest(unittest.TestCase):
                 print("name: val", file=open(outfile, 'w'))
                 '''),
             plugin_yaml_file: textwrap.dedent('''\
-                fetch exe: fetch.py
+                sync exe: fetch.py
                 reup exe: reup.py
                 required fields: []
                 ''')})