qubes-apply
python script to automate qubes saltstack
git clone https://9o.is/git/qubes-apply.git
qubes-apply
(15725B)
1 #!/usr/bin/python3
2
3 from itertools import chain
4 from pathlib import Path
5 from qubesadmin import Qubes
6 from functools import cache
7 import argparse, hashlib, json, os, sys, yaml, subprocess, fnmatch, re
8
9 parser = argparse.ArgumentParser(description='Apply user salt qubes')
10 parser.add_argument('-a', '--all', action='store_true', help='apply all defined qvms')
11 parser.add_argument('-n', '--dry-run', action='store_true', help='run without changes')
12 parser.add_argument('-i', '--init', action='store_true', help='initialize diff state')
13 parser.add_argument('--base-shared', help='path to shared salts iterated for dependency gathering')
14 parser.add_argument('--force-update', action='store_true', help='force state update without running')
15 parser.add_argument('--force-color', action='store_true', help='force colored output')
16 parser.add_argument('--_completion', action='store_true', help=argparse.SUPPRESS)
17 parser.add_argument('qvms', metavar='qvm', nargs='*', default=[], help='a qvm to apply (leave empty to apply diff)')
18 parser.set_defaults(force_color=True, base_shared=r'base\..+')
19 args = parser.parse_args()
20
21
22 def log(msg):
23 print(f'{msg}', flush=True)
24
25 def warn(msg):
26 print(f'warn: {msg}', flush=True)
27
28 def error(msg):
29 print(f'error: {msg}', flush=True, file=sys.stderr)
30 exit(1)
31
32 state_apply_output = ['-l', 'quiet', '--state-verbose=false', '--state-output=changes']
33
34 def apply_local(states):
35 subprocess.run(['qubesctl', '--show-output', 'state.apply',
36 'saltenv=user', ','.join(states)] + state_apply_output)
37
38 def apply(vm):
39 command = ['qubesctl', '--skip-dom0', '--show-output',
40 '--targets', vm, 'state.apply'] + state_apply_output
41 if args.force_color:
42 command.insert(1, '--force-color')
43 subprocess.run(command)
44
45 def open_yaml(file):
46 with open(str(file)) as stream:
47 try: return yaml.safe_load(stream)
48 except yaml.YAMLError as exc: error(f'yaml: {exc}')
49
50 def assert_subset(xs1, xs2, msg):
51 if not set(xs1).issubset(set(xs2)):
52 error(f'{msg}: {", ".join(map(str, set(xs1).difference(set(xs2))))}')
53
54 def dedup(lst):
55 return list(dict.fromkeys(lst))
56
57 @cache
58 def hash_file(path):
59 hasher = hashlib.sha1()
60 if not (Path(path).is_file() or Path(path).is_symlink()):
61 error(f'hash_file: {path} is not a file')
62 with open(str(path), 'rb') as f:
63 for chunk in iter(lambda: f.read(4096), b""):
64 hasher.update(chunk)
65 return hasher.hexdigest()
66
67 @cache
68 def hash_directory(path):
69 hasher = hashlib.sha1()
70 if not Path(path).is_dir():
71 error(f'hash_directory: {path} is not a directory')
72 for root, dirs, files in os.walk(path, followlinks=True):
73 dirs.sort()
74 files.sort()
75 hasher.update(hashlib.sha1(root.encode()).hexdigest().encode())
76 for file in files:
77 if re.search(r'^.*\.top$', file):
78 continue
79 file_path = os.path.join(root, file)
80 file_hash = hash_file(file_path)
81 hasher.update(file_hash.encode())
82 return hasher.hexdigest()
83
84 class QubesVMs(list):
85 def __init__(self):
86 super().__init__([vm for vm in Qubes().domains if vm.klass != 'DispVM'])
87
88 def templates(self):
89 return filter(lambda vm: vm.klass == 'TemplateVM', self)
90
91 def apps(self):
92 return filter(lambda vm: vm.klass == 'AppVM', self)
93
94 def dom0(self):
95 return list(filter(lambda vm: vm.klass == 'AdminVM', self))[0]
96
97 class State(str):
98 def __new__(cls, value, file_roots):
99 inst = super().__new__(cls, value)
100 inst.file_root = State.get_file_root(file_roots, value)
101 inst.path = State.get_path(inst.file_root, value)
102 inst.states = State.get_states(value, inst.path, file_roots)
103 return inst
104
105 def dependencies(self):
106 return dedup(chain.from_iterable(s.dependencies() for s in self.states)) + [self]
107
108 def is_init(self):
109 return self.path.name == 'init.sls'
110
111 def key(self):
112 return str(self.path if not self.is_init() else self.path.parent)
113
114 def hash(self):
115 return hash_directory(self.key()) if self.is_init() else hash_file(self.key())
116
117 def get_states(value, path, file_roots):
118 if not re.fullmatch(args.base_shared, value): return []
119 return [State(value, file_roots) for value in State.load(path)]
120
121 def get_file_root(file_roots, value):
122 for file_root in file_roots:
123 if State.get_path(file_root, value).exists():
124 return file_root
125 error(f'State file does not exist: {value}')
126
127 def get_path(file_root, value):
128 base = Path(file_root) / value.replace(".", "/")
129 p1 = base.with_suffix('.sls')
130 p2 = base / 'init.sls'
131 return p1 if p1.is_file() else p2
132
133 @cache
134 def load(path):
135 return open_yaml(path).get('include', [])
136
137 def pillar(vm, tgt):
138 pillars = {
139 'qubes:tags:(.*)': lambda vm, tag: tag in vm.tags,
140 'qubes:type:app': lambda vm: vm.klass == 'AppVM',
141 'qubes:type:template': lambda vm: vm.klass == 'TemplateVM',
142 }
143 for pillar, f in pillars.items():
144 match = re.fullmatch(pillar, tgt)
145 if match: return f(vm, *match.groups())
146 warn(f'unsupported pillar {tgt}')
147 return []
148
149 class Target:
150 filters = {
151 'glob': lambda vm, tgt: fnmatch.fnmatch(vm.name, tgt),
152 'pcre': lambda vm, tgt: re.search(tgt, vm.name),
153 'pillar': pillar,
154 }
155
156 def __init__(self, name, values, file_roots):
157 if not values: error(f'No state files defined for target {target}')
158 self.name = name
159 self.states = [State(value, file_roots) for value in values if type(value) is str]
160 self.matcher = Target.get_matcher(name, values)
161
162 def match(self, vm):
163 if self.matcher not in Target.filters: return False
164 else: return Target.filters[self.matcher](vm, self.name)
165
166 def dependencies(self):
167 return dedup(chain.from_iterable(state.dependencies() for state in self.states))
168
169 def get_matcher(name, values):
170 matcher = values[0].get('match') if values and type(values[0]) is dict else 'glob'
171 if matcher not in Target.filters: warn(f'found unsupported matcher {matcher} in {name}')
172 return matcher
173
174 class Top:
175 def __init__(self, path, file_roots):
176 self.path = path
177 self.targets = [Target(name, values, file_roots)
178 for name, values in open_yaml(path).get('user', {}).items()]
179
180 def enable(self, source):
181 fn = self.path.name if self.path.name != 'init.top' else self.path.parent.name + '.top'
182 (source / fn).symlink_to(self.path)
183
184 class FileRoot:
185 def __init__(self, path, enabled_path, file_roots=[]):
186 self.path = path
187 self.enabled_path = Path(enabled_path)
188 self.tops = FileRoot.get_tops(path, [path, *file_roots])
189
190 def targets(self, vm = None):
191 return [target for top in self.tops for target in top.targets if not vm or target.match(vm)]
192
193 def states(self, vm = None):
194 return dedup(chain.from_iterable(target.states for target in self.targets(vm)))
195
196 def dependencies(self, vm = None):
197 return dedup(chain.from_iterable(target.dependencies() for target in self.targets(vm)))
198
199 def configured(self, vms):
200 return [*filter(lambda vm: any(target.match(vm) for target in self.targets()), vms)]
201
202 def get_tops(path, file_roots):
203 tops = []
204 for root, dirs, files in os.walk(path):
205 dirs.sort()
206 files.sort()
207 tops.extend([Top(Path(root) / f, file_roots)
208 for f in files if f.endswith('.top')])
209 return tops
210
211 def clean_tops(self):
212 try:
213 for fn in os.listdir(self.enabled_path):
214 os.unlink(self.enabled_path / fn)
215 except Exception as e:
216 error(f'Failed to clean tops for {self}. Reason: {e}')
217
218 def enable_tops(self):
219 try:
220 for top in self.tops:
221 top.enable(self.enabled_path)
222 except Exception as e:
223 error(f'Failed to enable tops for {self}. Reason: {e}')
224
225 class SaltRoot(FileRoot):
226 def __init__(self, path, enabled_path, formulas_path):
227 self.formulas = SaltRoot.get_formulas(Path(formulas_path), enabled_path)
228 super().__init__(path, enabled_path, [formula.path for formula in self.formulas])
229
230 def states(self, vm = None):
231 return dedup(self.flatmap(lambda root: root.states(vm)))
232
233 def dependencies(self, vm = None):
234 return dedup(self.flatmap(lambda root: root.dependencies(vm)))
235
236 def configured(self, vms):
237 return dedup(self.flatmap(lambda root: root.configured(vms)))
238
239 def flatmap(self, func):
240 return chain.from_iterable(map(func, [super(), *self.formulas]))
241
242 def enable_tops(self):
243 super().enable_tops()
244 for formula in self.formulas: formula.enable_tops()
245
246 def enable_formulas(self):
247 data = {'file_roots': {'user': [formula.path for formula in self.formulas]}}
248 try:
249 with open('/etc/salt/minion.d/qubes-apply.conf', 'w') as yaml_file:
250 yaml.dump(data, yaml_file, default_flow_style=False)
251 except Exception as e:
252 error(f'Failed to enable formulas for {self}. Reason: {e}')
253
254 def get_formulas(path, enabled_path):
255 if not path.is_dir(): return []
256 return [FileRoot(str(file), enabled_path) for file in path.iterdir() if file.is_dir()]
257
258 class LastRun:
259 file = Path('/usr/local/var/qubes-apply/last-run.json')
260
261 def __init__(self, salt, pillar):
262 self.salt = salt
263 self.pillar = pillar
264 self.prev = {}
265 self.curr = {}
266
267 def initialized(self):
268 return LastRun.file.exists()
269
270 def load_current(self, vms):
271 if 'vm' not in self.curr:
272 self.curr['vm'] = {}
273 if 'state' not in self.curr:
274 self.curr['state'] = {}
275 for state in salt.dependencies() + pillar.dependencies():
276 self.curr['state'][state.key()] = state.hash()
277 for vm in vms:
278 if vm.name in self.curr['vm']: continue
279 states = self.salt.dependencies(vm) + self.pillar.dependencies(vm)
280 self.curr['vm'][vm.name] = {}
281 for state in states:
282 self.curr['vm'][vm.name][state.key()] = state.hash()
283
284 def load_previous(self):
285 if 'state' in self.prev:
286 return
287 try:
288 with open(LastRun.file, 'r') as file:
289 self.prev = json.load(file)
290 except FileNotFoundError:
291 error(f'diff state has not been initialized yet')
292 except Exception as exc:
293 error(f'failed loading diff state: {exc}')
294
295 def updated(self, vms):
296 self.load_current(vms)
297 self.load_previous()
298 prev_state = self.prev['state']
299 curr_state = self.curr['state']
300 updated = {}
301 for vm in vms:
302 prev_vm_states = self.prev['vm'][vm.name] if vm.name in self.prev['vm'] else {}
303 curr_vm_states = self.curr['vm'][vm.name] if vm.name in self.curr['vm'] else {}
304 reasons = []
305 removed = {*prev_vm_states}.difference({*curr_vm_states})
306 added = {*curr_vm_states}.difference({*prev_vm_states})
307 modified = [x for x in curr_vm_states if x in prev_vm_states and prev_vm_states[x] != curr_vm_states[x]]
308 if not prev_vm_states: reasons.append({'type': 'initialized', 'file': ''})
309 else:
310 for file in added: reasons.append({'type': 'added', 'file': file})
311 for file in removed: reasons.append({'type': 'removed', 'file': file})
312 for file in modified: reasons.append({'type': 'modified', 'file': file})
313 if not (added or modified or removed or prev_vm_states == curr_vm_states):
314 reasons.append({'type': 'reordered states', 'file': ''})
315 if reasons: updated[vm] = reasons
316 return updated
317
318 def save(self, vms):
319 self.load_current(vms)
320 if self.initialized():
321 self.load_previous()
322 else:
323 LastRun.file.parent.mkdir(parents=True, exist_ok=True)
324 LastRun.file.touch()
325 self.prev = { 'state': {}, 'vm': {} }
326 self.prev['state'] = {**self.curr['state']}
327 for vm in vms:
328 self.prev['vm'][vm.name] = {**self.curr['vm'][vm.name]}
329 with open(LastRun.file, 'w') as file:
330 json.dump(self.prev, file, indent = 2)
331
332 color = {
333 'initialized': '\033[32m',
334 'added': '\033[32m',
335 'removed': '\033[31m',
336 'modified': '\033[33m',
337 'reordered states': '\033[33m',
338 'reset': '\033[0m'
339 }
340
341 salt = SaltRoot('/srv/user_salt', '/srv/salt/_tops/user', '/srv/user_formulas')
342 pillar = FileRoot('/srv/user_pillar', '/srv/pillar/_tops/user')
343 lastrun = LastRun(salt, pillar)
344 vms = QubesVMs()
345 diff = not len(args.qvms)
346 applied_dom0 = False
347
348 if args._completion:
349 log(' '.join(map(str, salt.configured(vms))))
350 exit(0)
351
352 if args.init:
353 if lastrun.initialized():
354 log('skipping. diff state already initialized')
355 else:
356 lastrun.save([])
357 log('diff state initialized')
358 exit(0)
359
360 if args.force_update:
361 if not lastrun.initialized():
362 log('skipping. diff state not initialized')
363 else:
364 lastrun.save(salt.configured(vms))
365 log('diff state updated forcefully')
366 exit(0)
367
368 salt.clean_tops()
369 pillar.clean_tops()
370
371 salt.enable_tops()
372 salt.enable_formulas()
373 pillar.enable_tops()
374
375 if diff:
376 vms_updated = lastrun.updated(salt.configured(vms))
377 if 'dom0' in vms_updated:
378 log('Changes in dom0:')
379 for reason in vms_updated['dom0']:
380 log(f' {color[reason["type"]]}{reason["type"]}{color["reset"]} {reason["file"]}')
381 if any([x['type'] != 'removed' for x in vms_updated['dom0']]):
382 args.qvms = ['dom0']
383 else:
384 log(' (Skipping update. Only states were removed)')
385 lastrun.save([vms.dom0()])
386 log('')
387 applied_dom0 = True
388
389 if args.all or 'dom0' in args.qvms:
390 if 'dom0' in args.qvms: args.qvms.remove('dom0')
391 log('Applying dom0')
392 if not args.dry_run:
393 dom0 = vms.dom0()
394 apply_local(salt.states(dom0))
395 lastrun.save([dom0])
396
397 vms = QubesVMs()
398 vms_configured = [vm.name for vm in salt.configured(vms)]
399
400 if args.all:
401 args.qvms = vms_configured
402
403 assert_subset(args.qvms, vms_configured, 'Target is not in user salt configuration')
404
405 if diff:
406 vms_updated = lastrun.updated(salt.configured(vms))
407 if not vms_updated:
408 if not applied_dom0: log('no changes to apply')
409 exit(0)
410 if 'dom0' in vms_updated: del vms_updated['dom0']
411 args.qvms = []
412 if vms_updated:
413 log('Changes in vms:')
414 for vm in vms_updated:
415 log(f' {vm}')
416 for reason in vms_updated[vm]:
417 log(f' {color[reason["type"]]}{reason["type"]}{color["reset"]} {reason["file"]}')
418 if any([x['type'] != 'removed' for x in vms_updated[vm]]):
419 args.qvms.append(vm)
420 else:
421 log(' (Skipping update. Only states were removed)')
422 lastrun.save([vms[vms.index(vm)]])
423 log('')
424
425 if args.dry_run:
426 exit(0)
427
428 count = 1
429 total = len(args.qvms)
430
431 for vm in vms.templates():
432 if vm in args.qvms:
433 log(f'Applying {vm} [{count}/{total}]')
434 apply(vm.name)
435 lastrun.save([vm])
436 count += 1
437
438 for vm in vms.apps():
439 if vm in args.qvms:
440 log(f'Applying {vm} [{count}/{total}]')
441 apply(vm.name)
442 lastrun.save([vm])
443 count += 1
444