diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py
index 561837770..b14d1d317 100644
--- a/infra/cifuzz/config_utils.py
+++ b/infra/cifuzz/config_utils.py
@@ -209,7 +209,6 @@ class BaseConfig:
     self.git_store_branch = os.environ.get('GIT_STORE_BRANCH')
     self.git_store_branch_coverage = os.environ.get('GIT_STORE_BRANCH_COVERAGE',
                                                     self.git_store_branch)
-    self.upload_build = environment.get_bool('UPLOAD_BUILD', False)
 
     # TODO(metzman): Fix tests to create valid configurations and get rid of
     # CIFUZZ_TEST here and in presubmit.py.
@@ -344,3 +343,7 @@ class BuildFuzzersConfig(BaseConfig):
         # Not from a commit or PR.
         (not self.base_ref and not self.base_commit) or
         environment.get_bool('KEEP_UNAFFECTED_FUZZERS'))
+    self.upload_build = environment.get_bool('UPLOAD_BUILD', False)
+    if self.upload_build:
+      logging.info('Keeping all fuzzers because we are uploading build.')
+      self.keep_unaffected_fuzz_targets = True