86 lines
4.0 KiB
Python
86 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Transfer an explicit source delta to this task's dedicated Ubuntu guest."""
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import tarfile
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--name', required=True)
|
|
parser.add_argument('files', nargs='+')
|
|
args = parser.parse_args()
|
|
if not re.fullmatch(r'[a-z0-9][a-z0-9-]{0,63}', args.name):
|
|
parser.error('Use a short lowercase name for a new delta')
|
|
work = Path(os.environ.get('DOCVIEW_VM_WORK', ROOT / 'build-ubuntu-vm')).resolve()
|
|
output = work / 'source-deltas' / args.name
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
rows, payload = [], {}
|
|
for name in sorted(set(args.files)):
|
|
relative = Path(name)
|
|
if relative.is_absolute() or '..' in relative.parts or str(relative) != name:
|
|
raise SystemExit('Only canonical repository-relative source paths are allowed')
|
|
if relative.parts[0] not in ('src', 'qml', 'resources', 'cmake', 'tools', 'tests', 'CMakeLists.txt', 'README.md'):
|
|
raise SystemExit('Path is not in the allowed source set')
|
|
path = ROOT / relative
|
|
if path.is_symlink() or not path.resolve().is_relative_to(ROOT) or not path.is_file():
|
|
raise SystemExit('Only regular local source files are allowed')
|
|
data = path.read_bytes()
|
|
if len(data) > 8 * 1024 * 1024 or len(payload) >= 256:
|
|
raise SystemExit('Source delta exceeds finite limits')
|
|
payload[name] = data
|
|
rows.append({'path': name, 'size': len(data), 'sha256': hashlib.sha256(data).hexdigest()})
|
|
archive = output / 'patch.tar.gz'
|
|
with tarfile.open(archive, 'w:gz') as stream:
|
|
for name, data in payload.items():
|
|
info = tarfile.TarInfo(name)
|
|
info.size, info.mode = len(data), 0o644
|
|
stream.addfile(info, io.BytesIO(data))
|
|
record = {'name': args.name, 'files': rows,
|
|
'archiveSha256': hashlib.sha256(archive.read_bytes()).hexdigest()}
|
|
manifest = output / 'patch.json'
|
|
manifest.write_text(json.dumps(record, indent=2) + '\n')
|
|
apply = output / 'apply.py'
|
|
apply.write_text('''import hashlib,json,shutil,tarfile
|
|
from pathlib import Path
|
|
here=Path(__file__).resolve().parent
|
|
record=json.loads((here/'patch.json').read_text())
|
|
assert hashlib.sha256((here/'patch.tar.gz').read_bytes()).hexdigest()==record['archiveSha256']
|
|
source=Path.home()/'docview-source'
|
|
history=Path.home()/'validation/source-deltas'/record['name']
|
|
history.mkdir(parents=True,exist_ok=False)
|
|
changes=[]
|
|
with tarfile.open(here/'patch.tar.gz') as archive:
|
|
for row in record['files']:
|
|
rel=Path(row['path'])
|
|
assert not rel.is_absolute() and '..' not in rel.parts
|
|
data=archive.extractfile(row['path']).read()
|
|
assert len(data)==row['size'] and hashlib.sha256(data).hexdigest()==row['sha256']
|
|
target=source/rel
|
|
assert target.resolve().is_relative_to(source) and not target.is_symlink()
|
|
before=None
|
|
if target.exists():
|
|
before=hashlib.sha256(target.read_bytes()).hexdigest()
|
|
original=history/'originals'/rel
|
|
original.parent.mkdir(parents=True,exist_ok=True)
|
|
shutil.copyfile(target,original)
|
|
target.parent.mkdir(parents=True,exist_ok=True)
|
|
target.write_bytes(data)
|
|
changes.append({**row,'previousSha256':before,'changed':before!=row['sha256']})
|
|
(history/'record.json').write_text(json.dumps({**record,'files':changes},indent=2)+'\\n')
|
|
print(json.dumps({'sourceDelta':record['name'],'files':len(changes),'changed':sum(x['changed'] for x in changes)}))
|
|
''')
|
|
base = ['-i', str(work / 'private/id_ed25519'), '-o', 'IdentitiesOnly=yes', '-o', 'BatchMode=yes',
|
|
'-o', 'StrictHostKeyChecking=yes', '-o', 'UserKnownHostsFile=' + str(work / 'private/known_hosts')]
|
|
remote = 'incoming/delta-' + args.name
|
|
ssh = ['ssh', *base, '-p', '22224', '[email protected]']
|
|
subprocess.run([*ssh, 'mkdir -p ' + remote], check=True)
|
|
subprocess.run(['scp', *base, '-P', '22224', str(archive), str(manifest), str(apply),
|
|
'[email protected]:' + remote + '/'], check=True)
|
|
subprocess.run([*ssh, 'python3 ' + remote + '/apply.py'], check=True)
|