From a80fd730e4b7ac89726055894c746b674fbe2370 Mon Sep 17 00:00:00 2001 From: Dongge Liu Date: Mon, 25 Mar 2024 23:39:11 +1100 Subject: [PATCH] Revert "target_experiment: don't use gcloud. (#11701)" (#11727) This reverts commit 66bcb5af8103b2dddcccd59951a11359aa951cc2 to help investigate a recent slowness and errors in OSS-Fuzz-Gen. Related: https://github.com/google/oss-fuzz-gen/pull/177. Specially, here is [the error](https://pantheon.corp.google.com/logs/query;cursorTimestamp=2024-03-25T06:45:55.502596773Z;duration=P7D;query=resource.type%3D%22k8s_container%22%0Aresource.labels.project_id%3D%22oss-fuzz%22%0Aresource.labels.location%3D%22us-central1-c%22%0Aresource.labels.cluster_name%3D%22llm-experiment%22%0Aresource.labels.namespace_name%3D%22default%22%0Alabels.k8s-pod%2Fbatch_kubernetes_io%2Fcontroller-uid%3D%226d6e5719-618b-4e3d-a161-065c116ccfc0%22%20severity%3E%3DDEFAULT%0A--Hide%20similar%20entries%0A-%2528textPayload%3D~%22WARNING:root:Missing%20result%20JSON%20of%20benchmark%20instance:%20%5B%5E%20%3D%5Ct%5Cn%5Cr%5Cf%5C%22%5C%2528%5C%2529%5C%5B%5C%5D%5C%7C'%5D%2B%20-%20%2528%2528%3F:%5Cd%5B,.%5D%3F%2529*%5Cd%2529%22%2529%0A--End%20of%20hide%20similar%20entries%0Atimestamp%3D%222024-03-25T06:45:55.502596773Z%22%0AinsertId%3D%22fw412us1tsys1vfh%22?project=oss-fuzz) shown in recent experiments: ```json { "textPayload": "Traceback (most recent call last):\t File \"/tmp/tmpi8uy26ij/infra/build/functions/target_experiment.py\", line 296, in \t main()\t File \"/tmp/tmpi8uy26ij/infra/build/functions/target_experiment.py\", line 289, in main\t run_experiment(args.project, args.target, args.args, args.upload_output_log,\t File \"/tmp/tmpi8uy26ij/infra/build/functions/target_experiment.py\", line 255, in run_experiment\t build_lib.wait_for_build(build_id, credentials, 'oss-fuzz')\t File \"/tmp/tmpi8uy26ij/infra/build/functions/build_lib.py\", line 627, in wait_for_build\t id=build_id).execute()\t ^^^^^^^^^\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/googleapiclient/_helpers.py\", line 134, in positional_wrapper\t return wrapped(*args, **kwargs)\t ^^^^^^^^^^^^^^^^^^^^^^^^\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/googleapiclient/http.py\", line 892, in execute\t resp, content = _retry_request(\t ^^^^^^^^^^^^^^^\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/googleapiclient/http.py\", line 204, in _retry_request\t raise exception\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/googleapiclient/http.py\", line 177, in _retry_request\t resp, content = http.request(uri, method, *args, **kwargs)\t ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/google_auth_httplib2.py\", line 209, in request\t self.credentials.before_request(self._request, method, uri, request_headers)\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/google/auth/credentials.py\", line 133, in before_request\t self.refresh(request)\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/google/auth/compute_engine/credentials.py\", line 99, in refresh\t self._retrieve_info(request)\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/google/auth/compute_engine/credentials.py\", line 79, in _retrieve_info\t info = _metadata.get_service_account_info(\t ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/google/auth/compute_engine/_metadata.py\", line 227, in get_service_account_info\t return get(\t ^^^^\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/google/auth/compute_engine/_metadata.py\", line 146, in get\t response = request(url=url, method=\"GET\", headers=_METADATA_HEADERS)\t ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/google_auth_httplib2.py\", line 119, in __call__\t response, data = self.http.request(\t ^^^^^^^^^^^^^^^^^^\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/httplib2/__init__.py\", line 1724, in request\t (response, content) = self._request(\t ^^^^^^^^^^^^^^\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/httplib2/__init__.py\", line 1444, in _request\t (response, content) = self._conn_request(conn, request_uri, method, body, headers)\t ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\t File \"/tmp/tmpi8uy26ij/venv/lib/python3.11/site-packages/httplib2/__init__.py\", line 1367, in _conn_request\t conn.request(method, request_uri, body, headers)\t File \"/usr/lib/python3.11/http/client.py\", line 1282, in request\t self._send_request(method, url, body, headers, encode_chunked)\t File \"/usr/lib/python3.11/http/client.py\", line 1328, in _send_request\t self.endheaders(body, encode_chunked=encode_chunked)\t File \"/usr/lib/python3.11/http/client.py\", line 1277, in endheaders\t self._send_output(message_body, encode_chunked=encode_chunked)\t File \"/usr/lib/python3.11/http/client.py\", line 1037, in _send_output\t self.send(msg)\t File \"/usr/lib/python3.11/http/client.py\", line 998, in send\t self.sock.sendall(data)\tBrokenPipeError: [Errno 32] Broken pipe\t", "insertId": "fw412us1tsys1vfh", "resource": { "type": "k8s_container", "labels": { "location": "us-central1-c", "namespace_name": "default", "pod_name": "ofg-pr-177-dg-cxszc", "cluster_name": "llm-experiment", "container_name": "experiment", "project_id": "oss-fuzz" } }, "timestamp": "2024-03-25T06:45:55.502596773Z", "severity": "ERROR", "labels": { "k8s-pod/batch_kubernetes_io/controller-uid": "6d6e5719-618b-4e3d-a161-065c116ccfc0", "k8s-pod/job-name": "ofg-pr-177-dg", "compute.googleapis.com/resource_name": "gke-llm-experiment-ofg-pr-exp-2621ef4f-sms5", "k8s-pod/batch_kubernetes_io/job-name": "ofg-pr-177-dg", "k8s-pod/controller-uid": "6d6e5719-618b-4e3d-a161-065c116ccfc0" }, "logName": "projects/oss-fuzz/logs/stderr", "receiveTimestamp": "2024-03-25T06:45:57.681367653Z" } ``` --- infra/build/functions/build_lib.py | 49 +++++++++++------------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index 0e5b90939..581b6b7ce 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -23,16 +23,13 @@ import six.moves.urllib.parse as urlparse import sys import time import subprocess -import tarfile import tempfile import json -import uuid from googleapiclient.discovery import build as cloud_build import googleapiclient.discovery import google.api_core.client_options import google.auth -from google.cloud import storage from oauth2client import service_account as service_account_lib import requests import yaml @@ -550,15 +547,6 @@ def get_build_body(steps, return build_body -def _tgz_local_build(oss_fuzz_project, temp_tgz_path): - """Prepare a .tgz containing the files required to build `oss_fuzz_project`.""" - # Just the projects/ dir should be sufficient. - project_rel_path = os.path.join('projects', oss_fuzz_project) - with tarfile.open(temp_tgz_path, 'w:gz') as tar: - tar.add(os.path.join(OSS_FUZZ_ROOT, project_rel_path), - arcname=project_rel_path) - - def run_build( # pylint: disable=too-many-arguments oss_fuzz_project, steps, @@ -578,24 +566,23 @@ def run_build( # pylint: disable=too-many-arguments use_build_pool=use_build_pool, experiment=experiment) if experiment: - with tempfile.NamedTemporaryFile(suffix='source.tgz') as tgz_file: - # Archive the necessary files for the build. - _tgz_local_build(oss_fuzz_project, tgz_file.name) - gcs_client = storage.Client() - # This is the automatically created Cloud Build bucket for Cloud Build. - bucket_name = gcs_client.project + '_cloudbuild' - bucket = gcs_client.bucket(bucket_name) - blob_name = f'source/{str(uuid.uuid4())}.tgz' - blob = bucket.blob(blob_name) - logging.info(f'Uploading project to {bucket_name}/{blob_name}') - blob.upload_from_filename(tgz_file.name) - - build_body['source'] = { - 'storageSource': { - 'bucket': bucket_name, - 'object': blob_name, - } - } + with tempfile.NamedTemporaryFile(suffix='build.json') as config_file: + config_file.write(bytes(json.dumps(build_body), 'utf-8')) + config_file.seek(0) + result = subprocess.run([ + 'gcloud', + 'builds', + 'submit', + '--project=oss-fuzz', + f'--config={config_file.name}', + '--async', + '--format=get(id)', + ], + stdout=subprocess.PIPE, + cwd=OSS_FUZZ_ROOT, + encoding='utf-8', + check=True) + return result.stdout.strip() cloudbuild = cloud_build('cloudbuild', 'v1', @@ -643,4 +630,4 @@ def cancel_build(build_id, credentials, cloud_project): cache_discovery=False, client_options=US_CENTRAL_CLIENT_OPTIONS) cloudbuild.projects().builds().cancel(projectId=cloud_project, - id=build_id).execute() + id=build_id).execute() \ No newline at end of file