2020-10-07 09:15:18 +00:00
|
|
|
|
from __future__ import annotations
|
|
|
|
|
from typing import Dict, Generator, List, Optional, Tuple
|
2020-04-24 23:36:52 +00:00
|
|
|
|
import argparse
|
2020-10-07 09:15:18 +00:00
|
|
|
|
import asyncio
|
|
|
|
|
import contextlib
|
2020-04-24 23:36:52 +00:00
|
|
|
|
import json
|
|
|
|
|
import os
|
2020-10-07 09:15:18 +00:00
|
|
|
|
import re
|
2020-04-24 23:36:52 +00:00
|
|
|
|
import subprocess
|
|
|
|
|
import sys
|
2020-10-07 09:15:18 +00:00
|
|
|
|
import tempfile
|
2020-04-24 23:36:52 +00:00
|
|
|
|
|
2020-10-07 09:15:18 +00:00
|
|
|
|
class CalledProcessError(Exception):
|
|
|
|
|
process: asyncio.subprocess.Process
|
2020-04-24 23:36:52 +00:00
|
|
|
|
|
2022-09-22 12:36:57 +00:00
|
|
|
|
class UpdateFailedException(Exception):
|
|
|
|
|
pass
|
|
|
|
|
|
2020-04-24 23:36:52 +00:00
|
|
|
|
def eprint(*args, **kwargs):
|
|
|
|
|
print(*args, file=sys.stderr, **kwargs)
|
|
|
|
|
|
2020-10-07 09:15:18 +00:00
|
|
|
|
async def check_subprocess(*args, **kwargs):
|
|
|
|
|
"""
|
|
|
|
|
Emulate check argument of subprocess.run function.
|
|
|
|
|
"""
|
|
|
|
|
process = await asyncio.create_subprocess_exec(*args, **kwargs)
|
|
|
|
|
returncode = await process.wait()
|
|
|
|
|
|
|
|
|
|
if returncode != 0:
|
|
|
|
|
error = CalledProcessError()
|
|
|
|
|
error.process = process
|
|
|
|
|
|
|
|
|
|
raise error
|
|
|
|
|
|
|
|
|
|
return process
|
|
|
|
|
|
|
|
|
|
async def run_update_script(nixpkgs_root: str, merge_lock: asyncio.Lock, temp_dir: Optional[Tuple[str, str]], package: Dict, keep_going: bool):
|
|
|
|
|
worktree: Optional[str] = None
|
|
|
|
|
|
|
|
|
|
update_script_command = package['updateScript']
|
|
|
|
|
|
|
|
|
|
if temp_dir is not None:
|
|
|
|
|
worktree, _branch = temp_dir
|
|
|
|
|
|
2021-05-20 23:08:51 +00:00
|
|
|
|
# Ensure the worktree is clean before update.
|
|
|
|
|
await check_subprocess('git', 'reset', '--hard', '--quiet', 'HEAD', cwd=worktree)
|
|
|
|
|
|
2020-10-07 09:15:18 +00:00
|
|
|
|
# Update scripts can use $(dirname $0) to get their location but we want to run
|
|
|
|
|
# their clones in the git worktree, not in the main nixpkgs repo.
|
|
|
|
|
update_script_command = map(lambda arg: re.sub(r'^{0}'.format(re.escape(nixpkgs_root)), worktree, arg), update_script_command)
|
|
|
|
|
|
2020-04-24 23:36:52 +00:00
|
|
|
|
eprint(f" - {package['name']}: UPDATING ...")
|
|
|
|
|
|
2020-10-07 09:15:18 +00:00
|
|
|
|
try:
|
2022-09-30 11:47:45 +00:00
|
|
|
|
update_process = await check_subprocess(
|
|
|
|
|
'env',
|
|
|
|
|
f"UPDATE_NIX_NAME={package['name']}",
|
|
|
|
|
f"UPDATE_NIX_PNAME={package['pname']}",
|
|
|
|
|
f"UPDATE_NIX_OLD_VERSION={package['oldVersion']}",
|
|
|
|
|
f"UPDATE_NIX_ATTR_PATH={package['attrPath']}",
|
|
|
|
|
*update_script_command,
|
|
|
|
|
stdout=asyncio.subprocess.PIPE,
|
|
|
|
|
stderr=asyncio.subprocess.PIPE,
|
|
|
|
|
cwd=worktree,
|
|
|
|
|
)
|
2020-10-07 09:15:18 +00:00
|
|
|
|
update_info = await update_process.stdout.read()
|
|
|
|
|
|
|
|
|
|
await merge_changes(merge_lock, package, update_info, temp_dir)
|
|
|
|
|
except KeyboardInterrupt as e:
|
|
|
|
|
eprint('Cancelling…')
|
|
|
|
|
raise asyncio.exceptions.CancelledError()
|
|
|
|
|
except CalledProcessError as e:
|
|
|
|
|
eprint(f" - {package['name']}: ERROR")
|
|
|
|
|
eprint()
|
|
|
|
|
eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
|
|
|
|
|
eprint()
|
|
|
|
|
stderr = await e.process.stderr.read()
|
|
|
|
|
eprint(stderr.decode('utf-8'))
|
|
|
|
|
with open(f"{package['pname']}.log", 'wb') as logfile:
|
|
|
|
|
logfile.write(stderr)
|
|
|
|
|
eprint()
|
|
|
|
|
eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
|
|
|
|
|
|
|
|
|
|
if not keep_going:
|
2022-09-22 12:36:57 +00:00
|
|
|
|
raise UpdateFailedException(f"The update script for {package['name']} failed with exit code {e.process.returncode}")
|
2020-10-07 09:15:18 +00:00
|
|
|
|
|
|
|
|
|
@contextlib.contextmanager
|
|
|
|
|
def make_worktree() -> Generator[Tuple[str, str], None, None]:
|
|
|
|
|
with tempfile.TemporaryDirectory() as wt:
|
|
|
|
|
branch_name = f'update-{os.path.basename(wt)}'
|
|
|
|
|
target_directory = f'{wt}/nixpkgs'
|
|
|
|
|
|
|
|
|
|
subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory])
|
|
|
|
|
yield (target_directory, branch_name)
|
|
|
|
|
subprocess.run(['git', 'worktree', 'remove', '--force', target_directory])
|
|
|
|
|
subprocess.run(['git', 'branch', '-D', branch_name])
|
|
|
|
|
|
|
|
|
|
async def commit_changes(name: str, merge_lock: asyncio.Lock, worktree: str, branch: str, changes: List[Dict]) -> None:
|
|
|
|
|
for change in changes:
|
|
|
|
|
# Git can only handle a single index operation at a time
|
|
|
|
|
async with merge_lock:
|
|
|
|
|
await check_subprocess('git', 'add', *change['files'], cwd=worktree)
|
2023-04-12 12:48:02 +00:00
|
|
|
|
commit_message = '{attrPath}: {oldVersion} -> {newVersion}'.format(**change)
|
2022-03-05 16:20:37 +00:00
|
|
|
|
if 'commitMessage' in change:
|
|
|
|
|
commit_message = change['commitMessage']
|
|
|
|
|
elif 'commitBody' in change:
|
|
|
|
|
commit_message = commit_message + '\n\n' + change['commitBody']
|
2020-10-07 09:15:18 +00:00
|
|
|
|
await check_subprocess('git', 'commit', '--quiet', '-m', commit_message, cwd=worktree)
|
|
|
|
|
await check_subprocess('git', 'cherry-pick', branch)
|
|
|
|
|
|
|
|
|
|
async def check_changes(package: Dict, worktree: str, update_info: str):
|
|
|
|
|
if 'commit' in package['supportedFeatures']:
|
|
|
|
|
changes = json.loads(update_info)
|
|
|
|
|
else:
|
|
|
|
|
changes = [{}]
|
|
|
|
|
|
|
|
|
|
# Try to fill in missing attributes when there is just a single change.
|
|
|
|
|
if len(changes) == 1:
|
|
|
|
|
# Dynamic data from updater take precedence over static data from passthru.updateScript.
|
|
|
|
|
if 'attrPath' not in changes[0]:
|
|
|
|
|
# update.nix is always passing attrPath
|
|
|
|
|
changes[0]['attrPath'] = package['attrPath']
|
|
|
|
|
|
|
|
|
|
if 'oldVersion' not in changes[0]:
|
|
|
|
|
# update.nix is always passing oldVersion
|
|
|
|
|
changes[0]['oldVersion'] = package['oldVersion']
|
|
|
|
|
|
|
|
|
|
if 'newVersion' not in changes[0]:
|
|
|
|
|
attr_path = changes[0]['attrPath']
|
|
|
|
|
obtain_new_version_process = await check_subprocess('nix-instantiate', '--expr', f'with import ./. {{}}; lib.getVersion {attr_path}', '--eval', '--strict', '--json', stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=worktree)
|
|
|
|
|
changes[0]['newVersion'] = json.loads((await obtain_new_version_process.stdout.read()).decode('utf-8'))
|
|
|
|
|
|
|
|
|
|
if 'files' not in changes[0]:
|
2021-12-06 16:07:01 +00:00
|
|
|
|
changed_files_process = await check_subprocess('git', 'diff', '--name-only', 'HEAD', stdout=asyncio.subprocess.PIPE, cwd=worktree)
|
2020-10-07 09:15:18 +00:00
|
|
|
|
changed_files = (await changed_files_process.stdout.read()).splitlines()
|
|
|
|
|
changes[0]['files'] = changed_files
|
|
|
|
|
|
|
|
|
|
if len(changed_files) == 0:
|
|
|
|
|
return []
|
|
|
|
|
|
|
|
|
|
return changes
|
|
|
|
|
|
|
|
|
|
async def merge_changes(merge_lock: asyncio.Lock, package: Dict, update_info: str, temp_dir: Optional[Tuple[str, str]]) -> None:
|
|
|
|
|
if temp_dir is not None:
|
|
|
|
|
worktree, branch = temp_dir
|
|
|
|
|
changes = await check_changes(package, worktree, update_info)
|
|
|
|
|
|
|
|
|
|
if len(changes) > 0:
|
|
|
|
|
await commit_changes(package['name'], merge_lock, worktree, branch, changes)
|
|
|
|
|
else:
|
|
|
|
|
eprint(f" - {package['name']}: DONE, no changes.")
|
|
|
|
|
else:
|
|
|
|
|
eprint(f" - {package['name']}: DONE.")
|
2020-04-24 23:36:52 +00:00
|
|
|
|
|
2020-10-07 09:15:18 +00:00
|
|
|
|
async def updater(nixpkgs_root: str, temp_dir: Optional[Tuple[str, str]], merge_lock: asyncio.Lock, packages_to_update: asyncio.Queue[Optional[Dict]], keep_going: bool, commit: bool):
|
|
|
|
|
while True:
|
|
|
|
|
package = await packages_to_update.get()
|
|
|
|
|
if package is None:
|
|
|
|
|
# A sentinel received, we are done.
|
|
|
|
|
return
|
2020-04-24 23:36:52 +00:00
|
|
|
|
|
2020-10-07 09:15:18 +00:00
|
|
|
|
if not ('commit' in package['supportedFeatures'] or 'attrPath' in package):
|
|
|
|
|
temp_dir = None
|
|
|
|
|
|
|
|
|
|
await run_update_script(nixpkgs_root, merge_lock, temp_dir, package, keep_going)
|
|
|
|
|
|
|
|
|
|
async def start_updates(max_workers: int, keep_going: bool, commit: bool, packages: List[Dict]):
|
|
|
|
|
merge_lock = asyncio.Lock()
|
|
|
|
|
packages_to_update: asyncio.Queue[Optional[Dict]] = asyncio.Queue()
|
|
|
|
|
|
|
|
|
|
with contextlib.ExitStack() as stack:
|
|
|
|
|
temp_dirs: List[Optional[Tuple[str, str]]] = []
|
|
|
|
|
|
|
|
|
|
# Do not create more workers than there are packages.
|
|
|
|
|
num_workers = min(max_workers, len(packages))
|
|
|
|
|
|
|
|
|
|
nixpkgs_root_process = await check_subprocess('git', 'rev-parse', '--show-toplevel', stdout=asyncio.subprocess.PIPE)
|
|
|
|
|
nixpkgs_root = (await nixpkgs_root_process.stdout.read()).decode('utf-8').strip()
|
|
|
|
|
|
|
|
|
|
# Set up temporary directories when using auto-commit.
|
|
|
|
|
for i in range(num_workers):
|
|
|
|
|
temp_dir = stack.enter_context(make_worktree()) if commit else None
|
|
|
|
|
temp_dirs.append(temp_dir)
|
|
|
|
|
|
|
|
|
|
# Fill up an update queue,
|
|
|
|
|
for package in packages:
|
|
|
|
|
await packages_to_update.put(package)
|
|
|
|
|
|
|
|
|
|
# Add sentinels, one for each worker.
|
|
|
|
|
# A workers will terminate when it gets sentinel from the queue.
|
|
|
|
|
for i in range(num_workers):
|
|
|
|
|
await packages_to_update.put(None)
|
|
|
|
|
|
|
|
|
|
# Prepare updater workers for each temp_dir directory.
|
|
|
|
|
# At most `num_workers` instances of `run_update_script` will be running at one time.
|
|
|
|
|
updaters = asyncio.gather(*[updater(nixpkgs_root, temp_dir, merge_lock, packages_to_update, keep_going, commit) for temp_dir in temp_dirs])
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
# Start updater workers.
|
|
|
|
|
await updaters
|
2022-09-22 12:36:57 +00:00
|
|
|
|
except asyncio.exceptions.CancelledError:
|
2020-10-07 09:15:18 +00:00
|
|
|
|
# When one worker is cancelled, cancel the others too.
|
|
|
|
|
updaters.cancel()
|
2022-09-22 12:36:57 +00:00
|
|
|
|
except UpdateFailedException as e:
|
|
|
|
|
# When one worker fails, cancel the others, as this exception is only thrown when keep_going is false.
|
|
|
|
|
updaters.cancel()
|
|
|
|
|
eprint(e)
|
|
|
|
|
sys.exit(1)
|
2020-10-07 09:15:18 +00:00
|
|
|
|
|
|
|
|
|
def main(max_workers: int, keep_going: bool, commit: bool, packages_path: str) -> None:
|
|
|
|
|
with open(packages_path) as f:
|
2020-04-24 23:36:52 +00:00
|
|
|
|
packages = json.load(f)
|
|
|
|
|
|
|
|
|
|
eprint()
|
|
|
|
|
eprint('Going to be running update for following packages:')
|
|
|
|
|
for package in packages:
|
|
|
|
|
eprint(f" - {package['name']}")
|
|
|
|
|
eprint()
|
|
|
|
|
|
|
|
|
|
confirm = input('Press Enter key to continue...')
|
|
|
|
|
if confirm == '':
|
|
|
|
|
eprint()
|
|
|
|
|
eprint('Running update for:')
|
|
|
|
|
|
2020-10-07 09:15:18 +00:00
|
|
|
|
asyncio.run(start_updates(max_workers, keep_going, commit, packages))
|
2020-04-24 23:36:52 +00:00
|
|
|
|
|
|
|
|
|
eprint()
|
|
|
|
|
eprint('Packages updated!')
|
|
|
|
|
sys.exit()
|
|
|
|
|
else:
|
|
|
|
|
eprint('Aborting!')
|
|
|
|
|
sys.exit(130)
|
|
|
|
|
|
|
|
|
|
parser = argparse.ArgumentParser(description='Update packages')
|
|
|
|
|
parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
|
|
|
|
|
parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
|
2020-10-07 09:15:18 +00:00
|
|
|
|
parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes')
|
2020-04-24 23:36:52 +00:00
|
|
|
|
parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
|
|
|
|
try:
|
2020-10-07 09:15:18 +00:00
|
|
|
|
main(args.max_workers, args.keep_going, args.commit, args.packages)
|
|
|
|
|
except KeyboardInterrupt as e:
|
|
|
|
|
# Let’s cancel outside of the main loop too.
|
|
|
|
|
sys.exit(130)
|