qubes-apply
python script to automate qubes saltstack
git clone https://9o.is/git/qubes-apply.git
commit 5586dd48baa46aa1c5defe0041faf500dd47d185 Author: Jul <jul@9o.is> Date: Fri, 27 Feb 2026 11:51:39 +0800 init Diffstat:
| A | Makefile | | | 18 | ++++++++++++++++++ |
| A | qubes-apply | | | 444 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | qubes-apply.fish | | | 5 | +++++ |
3 files changed, 467 insertions(+), 0 deletions(-)
diff --git a/Makefile b/Makefile @@ -0,0 +1,18 @@ +.POSIX: +.SUFFIXES: + +PREFIX=/usr/local +PREFIX_FISH=/usr/share/fish + +install: + install -D -m 0755 qubes-apply $(DESTDIR)$(PREFIX)/bin/qubes-apply + +install-fish-comp: + mkdir -p $(DESTDIR)$(PREFIX_FISH)/vendor_completions.d + install -D -m 0644 -o user -g user qubes-apply.fish $(DESTDIR)$(PREFIX_FISH)/vendor_completions.d/qubes-apply.fish + +uninstall: + rm -f $(DESTDIR)$(PREFIX)/bin/qubes-apply + rm -f $(DESTDIR)$(PREFIX_FISH)/vendor_completions.d/qubes-apply.fish + +.PHONY: install install-fish-comp uninstall diff --git a/qubes-apply b/qubes-apply @@ -0,0 +1,444 @@ +#!/usr/bin/python3 + +from itertools import chain +from pathlib import Path +from qubesadmin import Qubes +from functools import cache +import argparse, hashlib, json, os, sys, yaml, subprocess, fnmatch, re + +parser = argparse.ArgumentParser(description='Apply user salt qubes') +parser.add_argument('-a', '--all', action='store_true', help='apply all defined qvms') +parser.add_argument('-n', '--dry-run', action='store_true', help='run without changes') +parser.add_argument('-i', '--init', action='store_true', help='initialize diff state') +parser.add_argument('--base-shared', help='path to shared salts iterated for dependency gathering') +parser.add_argument('--force-update', action='store_true', help='force state update without running') +parser.add_argument('--force-color', action='store_true', help='force colored output') +parser.add_argument('--_completion', action='store_true', help=argparse.SUPPRESS) +parser.add_argument('qvms', metavar='qvm', nargs='*', default=[], help='a qvm to apply (leave empty to apply diff)') +parser.set_defaults(force_color=True, base_shared=r'base\..+') +args = parser.parse_args() + + +def log(msg): + print(f'{msg}', flush=True) + +def warn(msg): + print(f'warn: {msg}', flush=True) + +def error(msg): + print(f'error: {msg}', flush=True, file=sys.stderr) + exit(1) + +state_apply_output = ['-l', 'quiet', '--state-verbose=false', '--state-output=changes'] + +def apply_local(states): + subprocess.run(['qubesctl', '--show-output', 'state.apply', + 'saltenv=user', ','.join(states)] + state_apply_output) + +def apply(vm): + command = ['qubesctl', '--skip-dom0', '--show-output', + '--targets', vm, 'state.apply'] + state_apply_output + if args.force_color: + command.insert(1, '--force-color') + subprocess.run(command) + +def open_yaml(file): + with open(str(file)) as stream: + try: return yaml.safe_load(stream) + except yaml.YAMLError as exc: error(f'yaml: {exc}') + +def assert_subset(xs1, xs2, msg): + if not set(xs1).issubset(set(xs2)): + error(f'{msg}: {", ".join(map(str, set(xs1).difference(set(xs2))))}') + +def dedup(lst): + return list(dict.fromkeys(lst)) + +@cache +def hash_file(path): + hasher = hashlib.sha1() + if not (Path(path).is_file() or Path(path).is_symlink()): + error(f'hash_file: {path} is not a file') + with open(str(path), 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + hasher.update(chunk) + return hasher.hexdigest() + +@cache +def hash_directory(path): + hasher = hashlib.sha1() + if not Path(path).is_dir(): + error(f'hash_directory: {path} is not a directory') + for root, dirs, files in os.walk(path, followlinks=True): + dirs.sort() + files.sort() + hasher.update(hashlib.sha1(root.encode()).hexdigest().encode()) + for file in files: + if re.search(r'^.*\.top$', file): + continue + file_path = os.path.join(root, file) + file_hash = hash_file(file_path) + hasher.update(file_hash.encode()) + return hasher.hexdigest() + +class QubesVMs(list): + def __init__(self): + super().__init__([vm for vm in Qubes().domains if vm.klass != 'DispVM']) + + def templates(self): + return filter(lambda vm: vm.klass == 'TemplateVM', self) + + def apps(self): + return filter(lambda vm: vm.klass == 'AppVM', self) + + def dom0(self): + return list(filter(lambda vm: vm.klass == 'AdminVM', self))[0] + +class State(str): + def __new__(cls, value, file_roots): + inst = super().__new__(cls, value) + inst.file_root = State.get_file_root(file_roots, value) + inst.path = State.get_path(inst.file_root, value) + inst.states = State.get_states(value, inst.path, file_roots) + return inst + + def dependencies(self): + return dedup(chain.from_iterable(s.dependencies() for s in self.states)) + [self] + + def is_init(self): + return self.path.name == 'init.sls' + + def key(self): + return str(self.path if not self.is_init() else self.path.parent) + + def hash(self): + return hash_directory(self.key()) if self.is_init() else hash_file(self.key()) + + def get_states(value, path, file_roots): + if not re.fullmatch(args.base_shared, value): return [] + return [State(value, file_roots) for value in State.load(path)] + + def get_file_root(file_roots, value): + for file_root in file_roots: + if State.get_path(file_root, value).exists(): + return file_root + error(f'State file does not exist: {value}') + + def get_path(file_root, value): + base = Path(file_root) / value.replace(".", "/") + p1 = base.with_suffix('.sls') + p2 = base / 'init.sls' + return p1 if p1.is_file() else p2 + + @cache + def load(path): + return open_yaml(path).get('include', []) + +def pillar(vm, tgt): + pillars = { + 'qubes:tags:(.*)': lambda vm, tag: tag in vm.tags, + 'qubes:type:app': lambda vm: vm.klass == 'AppVM', + 'qubes:type:template': lambda vm: vm.klass == 'TemplateVM', + } + for pillar, f in pillars.items(): + match = re.fullmatch(pillar, tgt) + if match: return f(vm, *match.groups()) + warn(f'unsupported pillar {tgt}') + return [] + +class Target: + filters = { + 'glob': lambda vm, tgt: fnmatch.fnmatch(vm.name, tgt), + 'pcre': lambda vm, tgt: re.search(tgt, vm.name), + 'pillar': pillar, + } + + def __init__(self, name, values, file_roots): + if not values: error(f'No state files defined for target {target}') + self.name = name + self.states = [State(value, file_roots) for value in values if type(value) is str] + self.matcher = Target.get_matcher(name, values) + + def match(self, vm): + if self.matcher not in Target.filters: return False + else: return Target.filters[self.matcher](vm, self.name) + + def dependencies(self): + return dedup(chain.from_iterable(state.dependencies() for state in self.states)) + + def get_matcher(name, values): + matcher = values[0].get('match') if values and type(values[0]) is dict else 'glob' + if matcher not in Target.filters: warn(f'found unsupported matcher {matcher} in {name}') + return matcher + +class Top: + def __init__(self, path, file_roots): + self.path = path + self.targets = [Target(name, values, file_roots) + for name, values in open_yaml(path).get('user', {}).items()] + + def enable(self, source): + fn = self.path.name if self.path.name != 'init.top' else self.path.parent.name + '.top' + (source / fn).symlink_to(self.path) + +class FileRoot: + def __init__(self, path, enabled_path, file_roots=[]): + self.path = path + self.enabled_path = Path(enabled_path) + self.tops = FileRoot.get_tops(path, [path, *file_roots]) + + def targets(self, vm = None): + return [target for top in self.tops for target in top.targets if not vm or target.match(vm)] + + def states(self, vm = None): + return dedup(chain.from_iterable(target.states for target in self.targets(vm))) + + def dependencies(self, vm = None): + return dedup(chain.from_iterable(target.dependencies() for target in self.targets(vm))) + + def configured(self, vms): + return [*filter(lambda vm: any(target.match(vm) for target in self.targets()), vms)] + + def get_tops(path, file_roots): + tops = [] + for root, dirs, files in os.walk(path): + dirs.sort() + files.sort() + tops.extend([Top(Path(root) / f, file_roots) + for f in files if f.endswith('.top')]) + return tops + + def clean_tops(self): + try: + for fn in os.listdir(self.enabled_path): + os.unlink(self.enabled_path / fn) + except Exception as e: + error(f'Failed to clean tops for {self}. Reason: {e}') + + def enable_tops(self): + try: + for top in self.tops: + top.enable(self.enabled_path) + except Exception as e: + error(f'Failed to enable tops for {self}. Reason: {e}') + +class SaltRoot(FileRoot): + def __init__(self, path, enabled_path, formulas_path): + self.formulas = SaltRoot.get_formulas(Path(formulas_path), enabled_path) + super().__init__(path, enabled_path, [formula.path for formula in self.formulas]) + + def states(self, vm = None): + return dedup(self.flatmap(lambda root: root.states(vm))) + + def dependencies(self, vm = None): + return dedup(self.flatmap(lambda root: root.dependencies(vm))) + + def configured(self, vms): + return dedup(self.flatmap(lambda root: root.configured(vms))) + + def flatmap(self, func): + return chain.from_iterable(map(func, [super(), *self.formulas])) + + def enable_tops(self): + super().enable_tops() + for formula in self.formulas: formula.enable_tops() + + def enable_formulas(self): + data = {'file_roots': {'user': [formula.path for formula in self.formulas]}} + try: + with open('/etc/salt/minion.d/qubes-apply.conf', 'w') as yaml_file: + yaml.dump(data, yaml_file, default_flow_style=False) + except Exception as e: + error(f'Failed to enable formulas for {self}. Reason: {e}') + + def get_formulas(path, enabled_path): + if not path.is_dir(): return [] + return [FileRoot(str(file), enabled_path) for file in path.iterdir() if file.is_dir()] + +class LastRun: + file = Path('/usr/local/var/qubes-apply/last-run.json') + + def __init__(self, salt, pillar): + self.salt = salt + self.pillar = pillar + self.prev = {} + self.curr = {} + + def initialized(self): + return LastRun.file.exists() + + def load_current(self, vms): + if 'vm' not in self.curr: + self.curr['vm'] = {} + if 'state' not in self.curr: + self.curr['state'] = {} + for state in salt.dependencies() + pillar.dependencies(): + self.curr['state'][state.key()] = state.hash() + for vm in vms: + if vm.name in self.curr['vm']: continue + states = self.salt.dependencies(vm) + self.pillar.dependencies(vm) + self.curr['vm'][vm.name] = {} + for state in states: + self.curr['vm'][vm.name][state.key()] = state.hash() + + def load_previous(self): + if 'state' in self.prev: + return + try: + with open(LastRun.file, 'r') as file: + self.prev = json.load(file) + except FileNotFoundError: + error(f'diff state has not been initialized yet') + except Exception as exc: + error(f'failed loading diff state: {exc}') + + def updated(self, vms): + self.load_current(vms) + self.load_previous() + prev_state = self.prev['state'] + curr_state = self.curr['state'] + updated = {} + for vm in vms: + prev_vm_states = self.prev['vm'][vm.name] if vm.name in self.prev['vm'] else {} + curr_vm_states = self.curr['vm'][vm.name] if vm.name in self.curr['vm'] else {} + reasons = [] + removed = {*prev_vm_states}.difference({*curr_vm_states}) + added = {*curr_vm_states}.difference({*prev_vm_states}) + modified = [x for x in curr_vm_states if x in prev_vm_states and prev_vm_states[x] != curr_vm_states[x]] + if not prev_vm_states: reasons.append({'type': 'initialized', 'file': ''}) + else: + for file in added: reasons.append({'type': 'added', 'file': file}) + for file in removed: reasons.append({'type': 'removed', 'file': file}) + for file in modified: reasons.append({'type': 'modified', 'file': file}) + if not (added or modified or removed or prev_vm_states == curr_vm_states): + reasons.append({'type': 'reordered states', 'file': ''}) + if reasons: updated[vm] = reasons + return updated + + def save(self, vms): + self.load_current(vms) + if self.initialized(): + self.load_previous() + else: + LastRun.file.parent.mkdir(parents=True, exist_ok=True) + LastRun.file.touch() + self.prev = { 'state': {}, 'vm': {} } + self.prev['state'] = {**self.curr['state']} + for vm in vms: + self.prev['vm'][vm.name] = {**self.curr['vm'][vm.name]} + with open(LastRun.file, 'w') as file: + json.dump(self.prev, file, indent = 2) + +color = { + 'initialized': '\033[32m', + 'added': '\033[32m', + 'removed': '\033[31m', + 'modified': '\033[33m', + 'reordered states': '\033[33m', + 'reset': '\033[0m' +} + +salt = SaltRoot('/srv/user_salt', '/srv/salt/_tops/user', '/srv/user_formulas') +pillar = FileRoot('/srv/user_pillar', '/srv/pillar/_tops/user') +lastrun = LastRun(salt, pillar) +vms = QubesVMs() +diff = not len(args.qvms) +applied_dom0 = False + +if args._completion: + log(' '.join(map(str, salt.configured(vms)))) + exit(0) + +if args.init: + if lastrun.initialized(): + log('skipping. diff state already initialized') + else: + lastrun.save([]) + log('diff state initialized') + exit(0) + +if args.force_update: + if not lastrun.initialized(): + log('skipping. diff state not initialized') + else: + lastrun.save(salt.configured(vms)) + log('diff state updated forcefully') + exit(0) + +salt.clean_tops() +pillar.clean_tops() + +salt.enable_tops() +salt.enable_formulas() +pillar.enable_tops() + +if diff: + vms_updated = lastrun.updated(salt.configured(vms)) + if 'dom0' in vms_updated: + log('Changes in dom0:') + for reason in vms_updated['dom0']: + log(f' {color[reason["type"]]}{reason["type"]}{color["reset"]} {reason["file"]}') + if any([x['type'] != 'removed' for x in vms_updated['dom0']]): + args.qvms = ['dom0'] + else: + log(' (Skipping update. Only states were removed)') + lastrun.save([vms.dom0()]) + log('') + applied_dom0 = True + +if args.all or 'dom0' in args.qvms: + if 'dom0' in args.qvms: args.qvms.remove('dom0') + log('Applying dom0') + if not args.dry_run: + dom0 = vms.dom0() + apply_local(salt.states(dom0)) + lastrun.save([dom0]) + +vms = QubesVMs() +vms_configured = [vm.name for vm in salt.configured(vms)] + +if args.all: + args.qvms = vms_configured + +assert_subset(args.qvms, vms_configured, 'Target is not in user salt configuration') + +if diff: + vms_updated = lastrun.updated(salt.configured(vms)) + if not vms_updated: + if not applied_dom0: log('no changes to apply') + exit(0) + if 'dom0' in vms_updated: del vms_updated['dom0'] + args.qvms = [] + if vms_updated: + log('Changes in vms:') + for vm in vms_updated: + log(f' {vm}') + for reason in vms_updated[vm]: + log(f' {color[reason["type"]]}{reason["type"]}{color["reset"]} {reason["file"]}') + if any([x['type'] != 'removed' for x in vms_updated[vm]]): + args.qvms.append(vm) + else: + log(' (Skipping update. Only states were removed)') + lastrun.save([vms[vms.index(vm)]]) + log('') + +if args.dry_run: + exit(0) + +count = 1 +total = len(args.qvms) + +for vm in vms.templates(): + if vm in args.qvms: + log(f'Applying {vm} [{count}/{total}]') + apply(vm.name) + lastrun.save([vm]) + count += 1 + +for vm in vms.apps(): + if vm in args.qvms: + log(f'Applying {vm} [{count}/{total}]') + apply(vm.name) + lastrun.save([vm]) + count += 1 + diff --git a/qubes-apply.fish b/qubes-apply.fish @@ -0,0 +1,5 @@ +complete --no-files --command qubes-apply --short-option a --long-option all --description 'apply all defined qvms' +complete --no-files --command qubes-apply --short-option n --long-option dry-run --description 'run without changes' +complete --no-files --command qubes-apply --long-option force-color --description 'force color output' +complete --no-files --command qubes-apply --short-option i --long-option init --description 'initialize diff state' +complete --no-files --command qubes-apply --arguments (qubes-apply --_completion (commandline -cp))