cflite: Support private repos (#6592)

Pass github token during cloning.

Fixes #6584

infra/cifuzz/config_utils.py

@@ -82,6 +82,11 @@ class BaseCiEnvironment:
     """Returns the Git SHA to diff against."""
     raise NotImplementedError('Child class must implment method.')
 
+  @property
+  def actor(self):
+    """Name of the actor for the CI."""
+    raise NotImplementedError('Child class must implment method.')
+
   @property
   def token(self):
     """Returns the CI API token."""
@@ -119,6 +124,11 @@ class GenericCiEnvironment(BaseCiEnvironment):
     """Returns the CI API token."""
     return os.getenv('TOKEN')
 
+  @property
+  def actor(self):
+    """Name of the actor for the CI."""
+    return os.getenv('ACTOR')
+
   @property
   def project_repo_owner_and_name(self):
     """Returns a tuple containing the project repo owner and None."""
@@ -145,6 +155,11 @@ class GithubEnvironment(BaseCiEnvironment):
     """Returns the Git SHA to diff against."""
     return os.getenv('GITHUB_SHA')
 
+  @property
+  def actor(self):
+    """Name of the actor for the CI."""
+    return os.getenv('GITHUB_ACTOR')
+
   @property
   def token(self):
     """Returns the CI API token."""
@@ -215,6 +230,7 @@ class BaseConfig:
     self.language = _get_language()
     self.low_disk_space = environment.get_bool('LOW_DISK_SPACE', False)
 
+    self.actor = self._ci_env.actor
     self.token = self._ci_env.token
     self.git_store_repo = os.environ.get('GIT_STORE_REPO')
     self.git_store_branch = os.environ.get('GIT_STORE_BRANCH')

infra/cifuzz/continuous_integration.py

@@ -194,7 +194,11 @@ class InternalGithub(GithubCiMixin, BaseCi):
 
     # Checkout project's repo in the shared volume.
     manager = repo_manager.clone_repo_and_get_manager(
-        inferred_url, self.workspace.repo_storage, repo_name=image_repo_name)
+        inferred_url,
+        self.workspace.repo_storage,
+        repo_name=image_repo_name,
+        username=self.config.actor,
+        password=self.config.token)
     checkout_specified_commit(manager, self.config.pr_ref,
                               self.config.commit_sha)
 
@@ -302,7 +306,9 @@ class ExternalGithub(GithubCiMixin, BaseCi):
     manager = repo_manager.clone_repo_and_get_manager(
         self.config.git_url,
         self.workspace.repo_storage,
-        repo_name=self.config.project_repo_name)
+        repo_name=self.config.project_repo_name,
+        username=self.config.actor,
+        password=self.config.token)
     checkout_specified_commit(manager, self.config.pr_ref,
                               self.config.commit_sha)
 

infra/repo_manager.py

@@ -26,6 +26,8 @@ import logging
 import os
 import shutil
 
+import urllib.parse
+
 import utils
 
 
@@ -226,7 +228,11 @@ class RepoManager:
       shutil.rmtree(self.repo_dir)
 
 
-def clone_repo_and_get_manager(repo_url, base_dir, repo_name=None):
+def clone_repo_and_get_manager(repo_url,
+                               base_dir,
+                               repo_name=None,
+                               username=None,
+                               password=None):
   """Clones a repo and constructs a repo manager class.
 
     Args:
@@ -240,17 +246,23 @@ def clone_repo_and_get_manager(repo_url, base_dir, repo_name=None):
   manager = RepoManager(repo_dir)
 
   if not os.path.exists(repo_dir):
-    _clone(repo_url, base_dir, repo_name)
+    _clone(repo_url, base_dir, repo_name, username=username, password=password)
 
   return manager
 
 
-def _clone(repo_url, base_dir, repo_name):
+def _clone(repo_url, base_dir, repo_name, username=None, password=None):
   """Creates a clone of the repo in the specified directory.
 
      Raises:
        ValueError: when the repo is not able to be cloned.
   """
+  if username and password:
+    parsed_url = urllib.parse.urlparse(repo_url)
+    new_netloc = f'{username}:{password}@{parsed_url.netloc}'
+    repo_url = urllib.parse.urlunparse(parsed_url._replace(netloc=new_netloc))
+
   utils.execute(['git', 'clone', repo_url, repo_name],
                 location=base_dir,
-                check_result=True)
+                check_result=True,
+                log_command=not password)

infra/repo_manager_test.py

@@ -55,6 +55,21 @@ class CloneTest(unittest.TestCase):
         repo_manager._clone('https://github.com/oss-fuzz-not-real.git', tmp_dir,
                             'oss-fuzz')
 
+  @mock.patch('utils.execute')
+  def test_clone_with_username(self, mock_execute):  # pylint: disable=no-self-use
+    """Test clone with username."""
+    repo_manager._clone('https://github.com/fake/repo.git',
+                        '/',
+                        'name',
+                        username='user',
+                        password='password')
+    mock_execute.assert_called_once_with([
+        'git', 'clone', 'https://user:password@github.com/fake/repo.git', 'name'
+    ],
+                                         location='/',
+                                         check_result=True,
+                                         log_command=False)
+
 
 @unittest.skipIf(not os.getenv('INTEGRATION_TESTS'),
                  'INTEGRATION_TESTS=1 not set')

infra/utils.py

@@ -48,7 +48,11 @@ def command_to_string(command):
   return shlex.join(command)
 
 
-def execute(command, env=None, location=None, check_result=False):
+def execute(command,
+            env=None,
+            location=None,
+            check_result=False,
+            log_command=True):
   """Runs a shell command in the specified directory location.
 
   Args:
@@ -75,12 +79,18 @@ def execute(command, env=None, location=None, check_result=False):
   out = out.decode('utf-8', errors='ignore')
   err = err.decode('utf-8', errors='ignore')
 
-  command_str = command_to_string(command)
+  if log_command:
+    command_str = command_to_string(command)
+    display_err = err
+  else:
+    command_str = 'redacted'
+    display_err = 'redacted'
+
   if err:
-    logging.debug('Stderr of command "%s" is: %s.', command_str, err)
+    logging.debug('Stderr of command "%s" is: %s.', command_str, display_err)
   if check_result and process.returncode:
     raise RuntimeError('Executing command "{0}" failed with error: {1}.'.format(
-        command_str, err))
+        command_str, display_err))
   return out, err, process.returncode