build: Project sync fixes. (#4194)

- Add some more logging.
- Use GitHub client ID/secret rather than personal access token.
- Fix function deploy wrt "--project" argument.

infra/build/functions/datastore_entities.py

@@ -27,9 +27,10 @@ class Project(ndb.Model):
 
 
 # pylint: disable=too-few-public-methods
-class GitAuth(ndb.Model):
-  """Represents Github access token entity."""
-  access_token = ndb.StringProperty()
+class GithubCreds(ndb.Model):
+  """Represents GitHub credentials."""
+  client_id = ndb.StringProperty()
+  client_secret = ndb.StringProperty()
 
 
 # pylint: disable=too-few-public-methods

infra/build/functions/deploy.sh

@@ -1,3 +1,4 @@
+#!/bin/bash -ex
 # Copyright 2020 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -58,16 +59,16 @@ function deploy_scheduler {
 	if gcloud scheduler jobs describe $scheduler_name --project $project ;
 		then
 			gcloud scheduler jobs update pubsub $scheduler_name \
+			--project $project \
 			--schedule "$schedule" \
 			--topic $topic \
-			--message-body "$message" \
-			--project $project
+			--message-body "$message"
 		else
 			gcloud scheduler jobs create pubsub $scheduler_name \
+			--project $project \
 			--schedule "$schedule" \
 			--topic $topic \
-			--message-body "$message" \
-			--project $project
+			--message-body "$message"
 	fi
 }
 

infra/build/functions/project_sync.py

@@ -26,7 +26,7 @@ from google.api_core import exceptions
 from google.cloud import ndb
 from google.cloud import scheduler_v1
 
-from datastore_entities import GitAuth
+from datastore_entities import GithubCreds
 from datastore_entities import Project
 
 VALID_PROJECT_NAME = re.compile(r'^[a-zA-Z0-9_-]+$')
@@ -92,6 +92,7 @@ def sync_projects(cloud_scheduler_client, projects):
     if project.name in projects:
       continue
 
+    logging.info('Deleting project %s', project.name)
     try:
       delete_scheduler(cloud_scheduler_client, project.name)
       project.key.delete()
@@ -119,10 +120,13 @@ def sync_projects(cloud_scheduler_client, projects):
   for project in Project.query():
     if project.name not in projects:
       continue
+
+    logging.info('Setting up project %s', project.name)
     project_metadata = projects[project.name]
     project_changed = False
     if project.schedule != project_metadata.schedule:
       try:
+        logging.info('Schedule changed.')
         update_scheduler(cloud_scheduler_client, project,
                          projects[project.name].schedule)
         project.schedule = project_metadata.schedule
@@ -197,12 +201,12 @@ def get_projects(repo):
   return projects
 
 
-def get_access_token():
-  """Retrieves Github's Access token from Cloud Datastore."""
-  token = GitAuth.query().get()
-  if token is None:
-    raise RuntimeError('No access token available')
-  return token.access_token
+def get_github_creds():
+  """Retrieves GitHub client credentials."""
+  git_creds = GithubCreds.query().get()
+  if git_creds is None:
+    raise RuntimeError('Git credentials not available.')
+  return git_creds
 
 
 def sync(event, context):
@@ -210,7 +214,8 @@ def sync(event, context):
   del event, context  #unused
 
   with ndb.Client().context():
-    github_client = Github(get_access_token())
+    git_creds = get_github_creds()
+    github_client = Github(git_creds.client_id, git_creds.client_secret)
     repo = github_client.get_repo('google/oss-fuzz')
     projects = get_projects(repo)
     cloud_scheduler_client = scheduler_v1.CloudSchedulerClient()

infra/build/functions/project_sync_test.py

@@ -21,7 +21,7 @@ import unittest
 from google.cloud import ndb
 
 from datastore_entities import Project
-from project_sync import get_access_token
+from project_sync import get_github_creds
 from project_sync import get_projects
 from project_sync import ProjectMetadata
 from project_sync import sync_projects
@@ -276,10 +276,10 @@ class TestDataSync(unittest.TestCase):
 
     self.assertEqual(get_projects(repo), {})
 
-  def test_get_access_token(self):
-    """Testing get_access_token()."""
+  def test_get_github_creds(self):
+    """Testing get_github_creds()."""
     with ndb.Client().context():
-      self.assertRaises(RuntimeError, get_access_token)
+      self.assertRaises(RuntimeError, get_github_creds)
 
   @classmethod
   def tearDownClass(cls):