mirror of https://github.com/BOINC/boinc.git
[CI] Remove unused aws s3 custom deploy scripts
Signed-off-by: Vitalii Koshura <lestat.de.lionkur@gmail.com>
This commit is contained in:
Vitalii Koshura
parent
a657d5def1
commit
d86472a47e
|
|
@ -1,194 +0,0 @@
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import zipfile
|
|
||||||
import s3
|
|
||||||
|
|
||||||
def get_files(dir):
|
|
||||||
file_names = []
|
|
||||||
for path, _, files in os.walk(dir):
|
|
||||||
for name in files:
|
|
||||||
file_name = os.path.join(path, name)
|
|
||||||
file_names.append(file_name)
|
|
||||||
return file_names
|
|
||||||
|
|
||||||
def read_control(control):
|
|
||||||
package = ''
|
|
||||||
version = '0'
|
|
||||||
port_version = '0'
|
|
||||||
architecture = ''
|
|
||||||
lines = control.split('\n')
|
|
||||||
for line in lines:
|
|
||||||
if (line != ''):
|
|
||||||
pair = line.split(': ')
|
|
||||||
if (pair[0] == 'Package'):
|
|
||||||
package = pair[1]
|
|
||||||
elif (pair[0] == 'Version'):
|
|
||||||
version = pair[1]
|
|
||||||
elif (pair[0] == 'Port-Version'):
|
|
||||||
port_version = pair[1]
|
|
||||||
elif (pair[0] == 'Architecture'):
|
|
||||||
architecture = pair[1]
|
|
||||||
return package, version + '-' + port_version, architecture
|
|
||||||
|
|
||||||
def get_packages(archives):
|
|
||||||
packages = {}
|
|
||||||
for archive in archives:
|
|
||||||
zip_file = zipfile.ZipFile(archive, 'r')
|
|
||||||
control = zip_file.read('CONTROL')
|
|
||||||
package, version, architecture = read_control(control.decode('utf-8'))
|
|
||||||
if (architecture not in packages.keys()):
|
|
||||||
packages[architecture] = {}
|
|
||||||
if (package not in packages[architecture].keys()):
|
|
||||||
packages[architecture][package] = {}
|
|
||||||
if (version not in packages[architecture][package].keys()):
|
|
||||||
packages[architecture][package][version] = []
|
|
||||||
if (archive not in packages[architecture][package][version]):
|
|
||||||
packages[architecture][package][version].append(archive)
|
|
||||||
return packages
|
|
||||||
|
|
||||||
def print_packages(packages):
|
|
||||||
for architecture in packages:
|
|
||||||
print(architecture)
|
|
||||||
for package in packages[architecture]:
|
|
||||||
print('\t', package)
|
|
||||||
for version in packages[architecture][package]:
|
|
||||||
print('\t\t', version)
|
|
||||||
for archive in packages[architecture][package][version]:
|
|
||||||
print('\t\t\t', archive)
|
|
||||||
|
|
||||||
def get_hash_from_name(name):
|
|
||||||
parts = name.split('-')
|
|
||||||
return parts[2].split('.')[0]
|
|
||||||
|
|
||||||
def read_vcpkg_abi_info_content(content, packages):
|
|
||||||
dependencies = []
|
|
||||||
lines = content.split('\n')
|
|
||||||
for line in lines:
|
|
||||||
if line:
|
|
||||||
pair = line.split(' ')
|
|
||||||
if (pair[0] in packages):
|
|
||||||
dependencies.append(pair[1])
|
|
||||||
return dependencies
|
|
||||||
|
|
||||||
def read_vcpkg_abi_info(archive, package, packages):
|
|
||||||
zip_file = zipfile.ZipFile(archive, 'r')
|
|
||||||
if (package == 'gtest'):
|
|
||||||
package = 'GTest'
|
|
||||||
file_name = 'share/'+package+'/vcpkg_abi_info.txt'
|
|
||||||
try:
|
|
||||||
info_file = zip_file.read(file_name)
|
|
||||||
return read_vcpkg_abi_info_content(info_file.decode('utf-8'), packages)
|
|
||||||
except Exception as ex:
|
|
||||||
print('Failed to read the file', file_name, 'from', archive, ':', ex)
|
|
||||||
return ''
|
|
||||||
|
|
||||||
def mark_outdated_packages(packages, modification_dates):
|
|
||||||
outdated = []
|
|
||||||
for architecture in packages:
|
|
||||||
for package in packages[architecture]:
|
|
||||||
archives_with_same_version = {}
|
|
||||||
max_version = sorted(packages[architecture][package].keys(), reverse=True)[0]
|
|
||||||
for version in packages[architecture][package]:
|
|
||||||
if (version != max_version):
|
|
||||||
for archive in packages[architecture][package][version]:
|
|
||||||
outdated.append(archive)
|
|
||||||
else:
|
|
||||||
if (len(packages[architecture][package][version]) == 1):
|
|
||||||
continue
|
|
||||||
for archive in packages[architecture][package][version]:
|
|
||||||
if (modification_dates[archive] not in archives_with_same_version and archive in modification_dates):
|
|
||||||
archives_with_same_version[modification_dates[archive]] = archive
|
|
||||||
max_date = sorted(archives_with_same_version.keys(), reverse=True)[0]
|
|
||||||
for archive in packages[architecture][package][version]:
|
|
||||||
if (archive != archives_with_same_version[max_date]):
|
|
||||||
outdated.append(archive)
|
|
||||||
return outdated
|
|
||||||
|
|
||||||
def get_hash_list(packages):
|
|
||||||
hash_list = []
|
|
||||||
for architecture in packages:
|
|
||||||
for package in packages[architecture]:
|
|
||||||
for version in packages[architecture][package]:
|
|
||||||
for archive in packages[architecture][package][version]:
|
|
||||||
hash_list.append(get_hash_from_name(os.path.basename(archive)))
|
|
||||||
return hash_list
|
|
||||||
|
|
||||||
def remove_outdated_from_hash_list(hash_list, outdated):
|
|
||||||
for package in outdated:
|
|
||||||
package_hash = get_hash_from_name(os.path.basename(package))
|
|
||||||
if (package_hash in hash_list):
|
|
||||||
hash_list.remove(package_hash)
|
|
||||||
|
|
||||||
def add_package_to_outdated_by_hash(packages, outdated, package_hash, architecture):
|
|
||||||
for package in packages[architecture]:
|
|
||||||
for version in packages[architecture][package]:
|
|
||||||
for archive in packages[architecture][package][version]:
|
|
||||||
if (get_hash_from_name(os.path.basename(archive)) == package_hash and archive not in outdated):
|
|
||||||
outdated.append(archive)
|
|
||||||
return
|
|
||||||
|
|
||||||
def process_package_dependencies(package_hash, dependencies, packages, outdated, hash_list, architecture):
|
|
||||||
is_valid = True
|
|
||||||
if (package_hash not in hash_list):
|
|
||||||
add_package_to_outdated_by_hash(packages, outdated, package_hash, architecture)
|
|
||||||
is_valid = False
|
|
||||||
if (package_hash not in dependencies):
|
|
||||||
return is_valid
|
|
||||||
package_dependencies = dependencies[package_hash]
|
|
||||||
for dependency_hash in package_dependencies:
|
|
||||||
is_valid = is_valid and process_package_dependencies(dependency_hash, dependencies, packages, outdated, hash_list, architecture)
|
|
||||||
if (not is_valid):
|
|
||||||
add_package_to_outdated_by_hash(packages, outdated, package_hash, architecture)
|
|
||||||
if (package_hash in hash_list):
|
|
||||||
hash_list.remove(package_hash)
|
|
||||||
return False
|
|
||||||
return is_valid
|
|
||||||
|
|
||||||
def process_dependencies_list(dependencies, packages, outdated, hash_list, architecture):
|
|
||||||
for package_hash in dependencies:
|
|
||||||
process_package_dependencies(package_hash, dependencies, packages, outdated, hash_list, architecture)
|
|
||||||
|
|
||||||
def mark_duplicate_packages(packages, outdated):
|
|
||||||
hash_list = get_hash_list(packages)
|
|
||||||
remove_outdated_from_hash_list(hash_list, outdated)
|
|
||||||
|
|
||||||
for architecture in packages:
|
|
||||||
dependencies_list = {}
|
|
||||||
for package in packages[architecture]:
|
|
||||||
for version in packages[architecture][package]:
|
|
||||||
for archive in packages[architecture][package][version]:
|
|
||||||
dependencies = read_vcpkg_abi_info(archive, package, packages[architecture].keys())
|
|
||||||
if (len(dependencies) != 0):
|
|
||||||
dependencies_list[get_hash_from_name(os.path.basename(archive))] = dependencies
|
|
||||||
process_dependencies_list(dependencies_list, packages, outdated, hash_list, architecture)
|
|
||||||
|
|
||||||
def print_outdated(outdated, packages):
|
|
||||||
for architecture in packages:
|
|
||||||
for package in packages[architecture]:
|
|
||||||
for version in packages[architecture][package]:
|
|
||||||
for archive in packages[architecture][package][version]:
|
|
||||||
if (archive in outdated):
|
|
||||||
print(architecture, package, version, archive, sep=' -> ')
|
|
||||||
|
|
||||||
def help():
|
|
||||||
print('Usage:')
|
|
||||||
print('python cleanup_vcpkg_archive_cache.py <dir> <bucket> <access_key> <secret_key>')
|
|
||||||
|
|
||||||
if (len(sys.argv) != 5):
|
|
||||||
help()
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
dir_name = sys.argv[1]
|
|
||||||
bucket_name = sys.argv[2]
|
|
||||||
access_key = sys.argv[3]
|
|
||||||
secret_key = sys.argv[4]
|
|
||||||
|
|
||||||
modification_dates = s3.download_all(dir_name, bucket_name)
|
|
||||||
packages = get_packages(get_files(dir_name))
|
|
||||||
print_packages(packages)
|
|
||||||
outdated = mark_outdated_packages(packages, modification_dates)
|
|
||||||
|
|
||||||
mark_duplicate_packages(packages, outdated)
|
|
||||||
print('Outdated packages:')
|
|
||||||
print_outdated(outdated, packages)
|
|
||||||
s3.remove_files(outdated, bucket_name, access_key, secret_key)
|
|
||||||
|
|
@ -1,27 +0,0 @@
|
||||||
import sys
|
|
||||||
import s3
|
|
||||||
|
|
||||||
def help():
|
|
||||||
print('Usage:')
|
|
||||||
print('python manage_vcpkg_archive_cache.py <action> <dir> <os> <bucket> <access_key> <secret_key>')
|
|
||||||
|
|
||||||
if (len(sys.argv) != 7 and len(sys.argv) != 5):
|
|
||||||
help()
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
action_name = sys.argv[1]
|
|
||||||
dir_name = sys.argv[2]
|
|
||||||
os_name = sys.argv[3]
|
|
||||||
bucket_name = sys.argv[4]
|
|
||||||
|
|
||||||
if (action_name == 'upload' and len(sys.argv) == 7):
|
|
||||||
access_key = sys.argv[5]
|
|
||||||
secret_key = sys.argv[6]
|
|
||||||
s3.upload(os_name, dir_name, bucket_name, access_key, secret_key)
|
|
||||||
elif (action_name == 'download'):
|
|
||||||
s3.download(os_name, dir_name, bucket_name)
|
|
||||||
else:
|
|
||||||
help()
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
sys.exit(0)
|
|
||||||
114
deploy/s3.py
114
deploy/s3.py
|
|
@ -1,114 +0,0 @@
|
||||||
import boto3
|
|
||||||
import os
|
|
||||||
from botocore import UNSIGNED
|
|
||||||
from botocore.client import Config
|
|
||||||
from botocore.exceptions import NoCredentialsError
|
|
||||||
|
|
||||||
def s3_upload(local_file, s3_file, bucket, access_key, secret_key):
|
|
||||||
print('Uploading', local_file, '->', s3_file)
|
|
||||||
|
|
||||||
s3 = boto3.client('s3', aws_access_key_id=access_key,
|
|
||||||
aws_secret_access_key=secret_key)
|
|
||||||
|
|
||||||
try:
|
|
||||||
s3.upload_file(local_file, bucket, s3_file)
|
|
||||||
print("Upload Successful")
|
|
||||||
except FileNotFoundError:
|
|
||||||
print("The file was not found")
|
|
||||||
except NoCredentialsError:
|
|
||||||
print("Credentials not available")
|
|
||||||
except Exception:
|
|
||||||
print("Upload failed")
|
|
||||||
|
|
||||||
def s3_download(local_file, s3_file, bucket):
|
|
||||||
print('Downloading', s3_file, '->', local_file)
|
|
||||||
s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))
|
|
||||||
|
|
||||||
try:
|
|
||||||
s3.download_file(bucket, s3_file, local_file)
|
|
||||||
print("Download Successful")
|
|
||||||
except NoCredentialsError:
|
|
||||||
print("Credentials not available")
|
|
||||||
except Exception:
|
|
||||||
print("Download failed")
|
|
||||||
|
|
||||||
def s3_list(bucket):
|
|
||||||
s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))
|
|
||||||
|
|
||||||
try:
|
|
||||||
return s3.list_objects(Bucket=bucket)['Contents']
|
|
||||||
except Exception:
|
|
||||||
print("Failed to retrieve list of files in bucket")
|
|
||||||
return None
|
|
||||||
|
|
||||||
def upload(os_name, dir, bucket, access_key, secret_key):
|
|
||||||
objects = s3_list(bucket)
|
|
||||||
for path, _, files in os.walk(dir):
|
|
||||||
for name in files:
|
|
||||||
found = False
|
|
||||||
file_path = os.path.join(path, name)
|
|
||||||
dir_name = os.path.basename(path)
|
|
||||||
file_name = os_name + '-' + dir_name + '-' + name
|
|
||||||
if objects:
|
|
||||||
for object in objects:
|
|
||||||
key = object['Key']
|
|
||||||
if (key == file_name):
|
|
||||||
found = True
|
|
||||||
break
|
|
||||||
if (not found):
|
|
||||||
s3_upload(file_path, file_name, bucket, access_key, secret_key)
|
|
||||||
|
|
||||||
def cleanup(dir):
|
|
||||||
if not os.path.isdir(dir):
|
|
||||||
return
|
|
||||||
for filename in os.listdir(dir):
|
|
||||||
file_path = os.path.join(dir, filename)
|
|
||||||
try:
|
|
||||||
if os.path.isfile(file_path):
|
|
||||||
os.unlink(file_path)
|
|
||||||
elif os.path.isdir(file_path):
|
|
||||||
cleanup(file_path)
|
|
||||||
except Exception as e:
|
|
||||||
print('Failed to delete %s. Reason: %s' % (file_path, e))
|
|
||||||
|
|
||||||
def download(os_name, dir, bucket):
|
|
||||||
cleanup(dir)
|
|
||||||
objects = s3_list(bucket)
|
|
||||||
if objects:
|
|
||||||
for object in objects:
|
|
||||||
key = object['Key']
|
|
||||||
args = key.split('-')
|
|
||||||
if (len(args) == 3 and args[0] == os_name):
|
|
||||||
os.makedirs(os.path.join(dir, args[1]), exist_ok=True)
|
|
||||||
local_file = os.path.join(dir, args[1], args[2])
|
|
||||||
if (os.path.isfile(local_file)):
|
|
||||||
print('Found local file', local_file)
|
|
||||||
continue
|
|
||||||
s3_download(local_file, key, bucket)
|
|
||||||
|
|
||||||
def download_all(dir, bucket):
|
|
||||||
files = {}
|
|
||||||
cleanup(dir)
|
|
||||||
os.makedirs(dir, exist_ok=True)
|
|
||||||
objects = s3_list(bucket)
|
|
||||||
if objects:
|
|
||||||
for object in objects:
|
|
||||||
key = object['Key']
|
|
||||||
local_file = os.path.join(dir, key)
|
|
||||||
files[local_file] = object['LastModified']
|
|
||||||
s3_download(local_file, key, bucket)
|
|
||||||
return files
|
|
||||||
|
|
||||||
def s3_remove(s3_file, bucket, access_key, secret_key):
|
|
||||||
print('Removing', s3_file)
|
|
||||||
s3 = boto3.client('s3', aws_access_key_id=access_key,
|
|
||||||
aws_secret_access_key=secret_key)
|
|
||||||
try:
|
|
||||||
s3.delete_object(Bucket=bucket, Key=s3_file)
|
|
||||||
print("Remove successful")
|
|
||||||
except Exception as ex:
|
|
||||||
print("Remove failed: ", ex)
|
|
||||||
|
|
||||||
def remove_files(files, bucket, access_key, secret_key):
|
|
||||||
for file in files:
|
|
||||||
s3_remove(os.path.basename(file), bucket, access_key, secret_key)
|
|
||||||
Loading…
Reference in New Issue