runtime: publish SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1
This commit is contained in:
homelab-runtime-publisher
File diff suppressed because one or more lines are too long
@@ -0,0 +1,622 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1
|
||||
STATUS=FAIL
|
||||
RC=3
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1115-root-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=a475f4109c57c07002ec72ba49450725574b1c08d698b9878c58257e7cc704c8
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=3
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=17278f059b28a5b338724f4e22fad8aa2db50e23143eeee6635c075bf93a9691
|
||||
SANITIZED_OUTPUT_SHA256=17278f059b28a5b338724f4e22fad8aa2db50e23143eeee6635c075bf93a9691
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_BACKUP_1115_ROOT_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1115-root-rca
|
||||
MUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_FAILURE_AND_CANONICAL_SKLADCHIK_EVIDENCE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE;NO_REMOTE_MUTATION
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
REFERENCE_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
|
||||
AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
|
||||
SCHEDULER_RC=0
|
||||
SELFTEST_REPLACEMENT486=PASS
|
||||
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
|
||||
US_NETBIRD_STATE_SAFE_BEGIN=1
|
||||
US_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26
|
||||
{"schema":1,"logical_id":"us-netbird","result":"FAIL","generation":"v2-us-netbird-20260922T055316Z-d5c92acdd2e8","updated_at_utc":"2026-09-22T05:53:31Z","last_success_epoch":1789704670,"last_attempt_epoch":1790056411,"retry_after_epoch":1790078011,"consecutive_failures":14,"replica_count":null}
|
||||
{"path":"consecutive_failures","value":14}
|
||||
{"path":"failure_backoff_seconds","value":21600}
|
||||
{"path":"last_attempt_epoch","value":1790056411}
|
||||
{"path":"last_success_epoch","value":1789704670}
|
||||
{"path":"last_success_generation","value":"v2-us-netbird-20260918T041032Z-d7648be4c16b"}
|
||||
{"path":"last_success_manifest_sha256","value":"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5"}
|
||||
{"path":"last_success_payload_sha256","value":"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf"}
|
||||
{"path":"reason","value":"PIPE_FAILED"}
|
||||
{"path":"result","value":"FAIL"}
|
||||
{"path":"retry_after_epoch","value":1790078011}
|
||||
US_NETBIRD_STATE_SAFE_END=1
|
||||
US_NETBIRD_SOURCE_HITS_BEGIN=1
|
||||
SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7
|
||||
155- for node in ('pve01','pve02','pve03'):
|
||||
156- out.append({'logical_id':'node-'+node,'scope':'node','type':'pve_node_bundle','node':node,'due_seconds':7*d,'enabled':True,'green1':node=='pve01'})
|
||||
157- out += [
|
||||
158- {'logical_id':'md-relay','scope':'vps','type':'vps_md','host':'[PRIVATE_IP]','port':22,'user':'ops','hostkey_fingerprint':'SHA256:DXJGwunC5tEVvnC3nRxyayTN8FXmyJcVgf+Oa6udQsY','due_seconds':d,'enabled':True,'green1':True},
|
||||
159- {'logical_id':'spb-ingress','scope':'vps','type':'vps_spb','host':'[PRIVATE_IP]','port':22,'user':'edgeadmin','hostkey_fingerprint':'SHA256:Ef+HsZb8lIXoadvYBXuGKIL3H7NyJMmf9EYCoJRoSv8','due_seconds':d,'enabled':True,'green1':True},
|
||||
160: {'logical_id':'us-netbird','scope':'vps','type':'vps_us','host':'[PRIVATE_IP]','port':22,'user':'root','hostkey_fingerprint':'SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE','due_seconds':d,'enabled':True,'green1':True},
|
||||
161- {'logical_id':'router-netcraze','scope':'router','type':'router','due_seconds':d,'enabled':True,'green1':True},
|
||||
162- ]
|
||||
163- sites=[
|
||||
164- ('codevipe',160,'pve02','/var/www/forums/codevipe/public','xf_codevipe','single_transaction'),
|
||||
165- ('dsmods',160,'pve02','/var/www/forums/dsmods/public','xf_dsmods','single_transaction'),
|
||||
--
|
||||
216- except subprocess.TimeoutExpired:
|
||||
217- for q in (src,z):
|
||||
218- try:q.kill()
|
||||
219- except Exception:pass
|
||||
220- raise BackupError('PIPE_TIMEOUT')
|
||||
221: if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}')
|
||||
222-
|
||||
223-def ensure_dir(p,mode=0o700):
|
||||
224- p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p
|
||||
225-
|
||||
226-def atomic_write(path,data,mode=0o600):
|
||||
--
|
||||
248- try:
|
||||
249- j=json.loads(p.read_text()); d[j['logical_id']]=j
|
||||
250- except Exception: continue
|
||||
251- return d
|
||||
252-
|
||||
253:def write_state(logical_id,result,**kw):
|
||||
254- if result not in RESULT_CLASSES: raise BackupError('RESULT_CLASS_INVALID')
|
||||
255- j={'schema':1,'logical_id':logical_id,'result':result,'updated_at_utc':utcnow(),**kw}
|
||||
256- if result=='SUCCESS': j['last_success_epoch']=epoch()
|
||||
257- atomic_write(STATE_ROOT/'state'/(logical_id+'.json'),canonical_json(j)+b'\n',0o600)
|
||||
258- return j
|
||||
--
|
||||
384- if not names: raise BackupError('TAR_ZST_EMPTY')
|
||||
385- if required_member and required_member not in names: raise BackupError('REQUIRED_MEMBER_MISSING')
|
||||
386-
|
||||
387-def validate_payload(t,path):
|
||||
388- typ=t['type']
|
||||
389: if typ in ('pve_node_bundle','vps_md','vps_spb','vps_us','router'):
|
||||
390- validate_tar_zst(path)
|
||||
391- elif typ=='pve_guest':
|
||||
392- if t['guest_type']=='lxc': validate_tar_zst(path)
|
||||
393- else:
|
||||
394- # Green set is intentionally CT-only. Scheduled QEMU backups still get dual remote byte readback.
|
||||
--
|
||||
405-def cluster_inventory():
|
||||
406- p=run(['pvesh','get','/cluster/resources','--type','vm','--output-format','json'],timeout=60)
|
||||
407- try:return json.loads(p.stdout)
|
||||
408- except Exception: raise BackupError('PVE_INVENTORY_JSON')
|
||||
409-
|
||||
410:def ssh_remote(node,argv,input=None,timeout=1800,check=True):
|
||||
411- cmd='PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec '+shlex.join([str(x) for x in argv])
|
||||
412- if node=='pve01': return run([str(x) for x in argv],input=input,timeout=timeout,check=check)
|
||||
413- return run([SSH,'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node,cmd],input=input,timeout=timeout,check=check)
|
||||
414-
|
||||
415-def _pve_dumpdir_spec(guest_type):
|
||||
--
|
||||
471- if i+1>=len(argv) or argv[i+1]!=str(scratch): raise BackupError('PROVEN_DUMP_ARGV_TMPDIR_BINDING')
|
||||
472- elif '--tmpdir='+str(scratch) not in argv: raise BackupError('PROVEN_DUMP_ARGV_NO_TMPDIR')
|
||||
473- return argv,job
|
||||
474-
|
||||
475-def _default_unprivileged_lxc(node,vmid):
|
||||
476: q=ssh_remote(node,['pct','config',str(vmid)],timeout=30)
|
||||
477- lines=[x.strip() for x in q.stdout.decode('utf-8','replace').splitlines()]
|
||||
478- if 'unprivileged: 1' not in lines: raise BackupError('LXC_NOT_UNPRIVILEGED')
|
||||
479- if any(x.startswith('lxc.idmap:') or x.startswith('lxc.idmap =') for x in lines): raise BackupError('LXC_CUSTOM_IDMAP_UNSUPPORTED')
|
||||
480-
|
||||
481-def mapped_scratch_probe(node,scratch):
|
||||
482- tool=shutil.which('lxc-usernsexec') or '/usr/bin/lxc-usernsexec'
|
||||
483- pos=[tool,'-m','u:0:100000:65536','-m','g:0:100000:65536','--','/usr/bin/test','-x',str(scratch)]
|
||||
484: q=ssh_remote(node,pos,timeout=30,check=False)
|
||||
485- if q.returncode!=0: raise BackupError('MAPPED_UID_ACCESS_PROBE_FAILED')
|
||||
486-
|
||||
487-def _stream_argv_to_local(node,argv,dest,timeout=7200):
|
||||
488- dest=pathlib.Path(dest)
|
||||
489- if node=='pve01': cmd=argv
|
||||
--
|
||||
509- if typ!=t['guest_type']: raise BackupError('PVE_GUEST_TYPE_DRIFT')
|
||||
510- if typ=='lxc':
|
||||
511- _require_exact_source(P270,P270_SHA,'P270_IDENTITY');_require_exact_source(P282,P282_SHA,'P282_IDENTITY')
|
||||
512- _default_unprivileged_lxc(node,vmid)
|
||||
513- scratch=f'/var/tmp/homelab-vzdump-v2-{gen}'
|
||||
514: ssh_remote(node,['install','-d','-m','710','-o','0','-g','100000',scratch],timeout=30)
|
||||
515: q=ssh_remote(node,['stat','-c','%a:%u:%g',scratch],timeout=30)
|
||||
516- if q.stdout.decode().strip()!='710:0:100000': raise BackupError('PVE_MAPPED_SCRATCH_ACCESS_CONTRACT')
|
||||
517- try:
|
||||
518- mapped_scratch_probe(node,scratch)
|
||||
519- argv,job=proven_lxc_dump_argv(t,row,node,scratch)
|
||||
520- local=work/'payload.bin'
|
||||
521- _stream_argv_to_local(node,argv,local,timeout=7200)
|
||||
522- if local.stat().st_size>int(job['archive_cap_bytes']): raise BackupError('VZDUMP_ARCHIVE_CAP_EXCEEDED')
|
||||
523- except Exception:
|
||||
524: ssh_remote(node,['rm','-rf','--',scratch],timeout=60,check=False);raise
|
||||
525- cleanup={'kind':'pve_source_dir','node':node,'path':scratch}
|
||||
526- return local,{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status'),'producer_contract':'proven-270-282-stream'},cleanup
|
||||
527- # QEMU path remains unchanged: bounded dedicated dumpdir with later cloud readback/reconstruction.
|
||||
528- srcdir=f'/var/tmp/homelab-vzdump-v2-{gen}'
|
||||
529: ssh_remote(node,['install','-d','-m','700','-o','0','-g','0',srcdir],timeout=30)
|
||||
530- try:
|
||||
531: ssh_remote(node,['vzdump',str(vmid),'--mode','snapshot','--compress','zstd','--dumpdir',srcdir],timeout=7200)
|
||||
532: p=ssh_remote(node,['find',srcdir,'-maxdepth','1','-type','f','-printf','%p\n'],timeout=30)
|
||||
533- files=[x for x in p.stdout.decode().splitlines() if x.strip()]
|
||||
534- if len(files)!=1: raise BackupError('VZDUMP_OUTPUT_NOT_UNIQUE')
|
||||
535- src=files[0];local=work/'payload.bin'
|
||||
536- if node=='pve01': local=pathlib.Path(src)
|
||||
537- else: run([SCP,'-q','-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node+':'+src,str(local)],timeout=7200)
|
||||
538- if not pathlib.Path(local).is_file() or pathlib.Path(local).stat().st_size<=0: raise BackupError('VZDUMP_PAYLOAD_MISSING')
|
||||
539- except Exception:
|
||||
540: ssh_remote(node,['rm','-rf','--',srcdir],timeout=60,check=False);raise
|
||||
541- return pathlib.Path(local),{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status')},{'kind':'pve_source_dir','node':node,'path':srcdir}
|
||||
542-
|
||||
543-def operator_key_fingerprint():
|
||||
544- if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING')
|
||||
545- p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30)
|
||||
--
|
||||
563- return good[0]
|
||||
564-
|
||||
565-def ensure_vps_known_hosts(targets):
|
||||
566- lines=[]
|
||||
567- for t in targets:
|
||||
568: if t['type'] not in ('vps_md','vps_spb','vps_us'): continue
|
||||
569- lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint']))
|
||||
570- atomic_write(VPS_KNOWN_HOSTS,('\n'.join(lines)+'\n').encode(),0o600)
|
||||
571-
|
||||
572-def vps_ssh_argv(t, *, use_agent=False):
|
||||
573- a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15']
|
||||
--
|
||||
598- tag=p.stdout.decode().strip()
|
||||
599- if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT')
|
||||
600- vps_run(t,['true'],timeout=30,use_agent=False)
|
||||
601- results[t['logical_id']]=tag
|
||||
602- # US was already operator-key based; prove it agentless too.
|
||||
603: us=[x for x in targets if x['type']=='vps_us'][0]
|
||||
604- vps_run(us,['true'],timeout=30,use_agent=False)
|
||||
605- results[us['logical_id']]='ALREADY_US_OPERATOR_KEY'
|
||||
606- return results
|
||||
607-
|
||||
608-def remote_hashset(t,command):
|
||||
609- p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False)
|
||||
610- return sha256_bytes(p.stdout)
|
||||
611-
|
||||
612:def capture_vps_md(t,gen,work):
|
||||
613- files=['etc/netbird/install.conf','var/lib/netbird/state.json','var/lib/netbird/service.json','var/lib/netbird/active_profile.json','var/lib/netbird/default.json']
|
||||
614- cmd="sudo -n sha256sum -- "+' '.join(shlex.quote('/'+x) for x in files)
|
||||
615- pre=remote_hashset(t,cmd)
|
||||
616- remote=shlex.join(['sudo','-n','tar','--numeric-owner','-C','/','-cf','-']+files)
|
||||
617- argv=vps_ssh_argv(t,use_agent=False)+[remote]
|
||||
618- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=600)
|
||||
619- post=remote_hashset(t,cmd)
|
||||
620- if pre!=post: raise BackupError('MD_SOURCE_CHANGED_DURING_CAPTURE')
|
||||
621- return payload,{'generation':gen,'source_class':'vps_md','consistency':'FILESET_PRE_POST_HASH_STABILITY','hashset_sha256':pre},None
|
||||
622-
|
||||
623:def capture_vps_spb(t,gen,work):
|
||||
624- hcmd="find /etc/nginx -xdev -type f -print0 | sort -z | xargs -0 sha256sum"
|
||||
625- pre=remote_hashset(t,hcmd)
|
||||
626- remote=shlex.join(['tar','--numeric-owner','-C','/','-cf','-','etc/nginx'])
|
||||
627- argv=vps_ssh_argv(t,use_agent=False)+[remote]
|
||||
628- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=900)
|
||||
--
|
||||
657- if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33)
|
||||
658- b=dst.serialize(); dst.close(); src.close()
|
||||
659- ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b))
|
||||
660-'''
|
||||
661-
|
||||
662:def capture_vps_us(t,gen,work):
|
||||
663- remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT])
|
||||
664- argv=vps_ssh_argv(t,use_agent=False)+[remote]
|
||||
665- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200)
|
||||
666: return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None
|
||||
667-
|
||||
668-def _simple_shell_assignments(text):
|
||||
669- vals={}
|
||||
670- for line in text.splitlines():
|
||||
671- m=re.match(r'^\s*(?:export\s+)?([A-Z_][A-Z0-9_]*)=(.*)$',line)
|
||||
--
|
||||
801-
|
||||
802-def qga_exec(node,vmid,guest_argv,input_bytes=None,timeout=600):
|
||||
803- args=['qm','guest','exec',str(vmid),'--synchronous','1','--timeout',str(int(timeout))]
|
||||
804- if input_bytes is not None: args += ['--pass-stdin','1']
|
||||
805- args += ['--']+[str(x) for x in guest_argv]
|
||||
806: p=ssh_remote(node,args,input=input_bytes,timeout=timeout+60)
|
||||
807- try:j=json.loads(p.stdout)
|
||||
808- except Exception: raise BackupError('QGA_JSON')
|
||||
809- rc=j.get('exitcode',j.get('exit-code'))
|
||||
810- if rc is None: rc=0 if j.get('exited') else None
|
||||
811- if rc not in (0,None): raise BackupError('QGA_GUEST_COMMAND_FAILED',str(rc))
|
||||
--
|
||||
854-
|
||||
855-def capture_node_bundle(t,gen,work):
|
||||
856- node=t['node']; raw=node_bundle_paths(node); paths=[]
|
||||
857- for rel in raw:
|
||||
858- if node=='pve01': ok=(pathlib.Path('/')/rel).exists()
|
||||
859: else: ok=(ssh_remote(node,['test','-e','/'+rel],timeout=20,check=False).returncode==0)
|
||||
860- if ok: paths.append(rel)
|
||||
861- required={'etc/network/interfaces','etc/hosts','etc/hostname','etc/pve/nodes/'+node}
|
||||
862- if not required.issubset(set(paths)): raise BackupError('NODE_BUNDLE_REQUIRED_PATH_MISSING')
|
||||
863- payload=work/'payload.tar.zst'
|
||||
864- argv=['tar','--numeric-owner','--xattrs','--acls','-C','/','-cf','-']+paths
|
||||
--
|
||||
870-
|
||||
871-def capture_target(t,gen,work):
|
||||
872- typ=t['type']
|
||||
873- if typ=='pve_guest': return capture_pve_guest(t,gen,work)
|
||||
874- if typ=='pve_node_bundle': return capture_node_bundle(t,gen,work)
|
||||
875: if typ=='vps_md': return capture_vps_md(t,gen,work)
|
||||
876: if typ=='vps_spb': return capture_vps_spb(t,gen,work)
|
||||
877: if typ=='vps_us': return capture_vps_us(t,gen,work)
|
||||
878- if typ=='router': return capture_router(t,gen,work)
|
||||
879- if typ=='xenforo': return capture_xenforo(t,gen,work)
|
||||
880- raise BackupError('CAPTURE_TYPE_UNKNOWN')
|
||||
881-
|
||||
882-def cleanup_source(cleanup):
|
||||
883- if not cleanup: return
|
||||
884- if cleanup.get('kind')=='pve_source_dir':
|
||||
885: ssh_remote(cleanup['node'],['rm','-rf','--',cleanup['path']],timeout=60)
|
||||
886-
|
||||
887-def control_verify(t):
|
||||
888- engine=pathlib.Path('/usr/local/sbin/homelab-control-backup')
|
||||
889- if sha256_file(engine)!='0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57': raise BackupError('CONTROL_ENGINE_IDENTITY')
|
||||
890- root=pathlib.Path('/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2')
|
||||
--
|
||||
895- if j.get('status')=='SEALED' and j.get('generation'): rows.append((p.stat().st_mtime,j['generation']))
|
||||
896- except Exception: continue
|
||||
897- if not rows: raise BackupError('CONTROL_SEALED_GENERATION_MISSING')
|
||||
898- gen=sorted(rows)[-1][1]
|
||||
899- p=run([str(engine),'verify',gen],timeout=1800)
|
||||
900: write_state(t['logical_id'],'SUCCESS',generation=gen,verification='existing_control_engine_verify',last_success_epoch=epoch())
|
||||
901- return {'logical_id':t['logical_id'],'generation':gen,'result':'SUCCESS','payload_bytes':0,'replicas':2}
|
||||
902-
|
||||
903:def backup_target(t,green_verify=False):
|
||||
904- if t['type']=='control': return control_verify(t)
|
||||
905- gen=generation_id(t['logical_id']); work=ensure_dir(WORK_ROOT/gen,0o700)
|
||||
906- payload=None; cleanup=None; sealed=False
|
||||
907- try:
|
||||
908- payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen
|
||||
--
|
||||
916- ha=reconstruct_replica(manifest,manifest['replicas'][0],a); validate_payload(t,a)
|
||||
SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py SHA256=2b5eb4fc0f6f466a0d3e5a007af987092b6c17baa3ff9241a6afc2c2e97df1d5
|
||||
2-from __future__ import annotations
|
||||
3-import errno,fcntl,importlib.util,os,sys
|
||||
4-from pathlib import Path
|
||||
5-ROOT=Path(__file__).resolve().parent
|
||||
6-CT_ID='guest-pve01-lxc-112'
|
||||
7:LEGACY_IDS=('node-pve01','md-relay','spb-ingress','us-netbird','router-netcraze','xf-codevipe','xf-zakrutim','control-bootstrap-v2')
|
||||
8-LOCK=Path('/run/homelab-backup-heavy.lock')
|
||||
9-
|
||||
10-def _load(name,path):
|
||||
11- spec=importlib.util.spec_from_file_location(name,str(path))
|
||||
12- if spec is None or spec.loader is None: raise RuntimeError('MODULE_SPEC')
|
||||
--
|
||||
14-
|
||||
15-legacy=_load('_backupv2_legacy486',ROOT/'legacy_engine.py')
|
||||
16-ct112=_load('_backupv2_ct112_486',ROOT/'ct112_handler.py')
|
||||
17-driver=_load('_backupv2_driver454_479',ROOT/'representative_driver.py')
|
||||
18-_legacy_load_targets=legacy.load_targets
|
||||
19:_legacy_backup_target=legacy.backup_target
|
||||
20-
|
||||
21-def canonical_targets():
|
||||
22- rows=_legacy_load_targets()
|
||||
23- if not isinstance(rows,list) or len(rows)!=28: raise legacy.BackupError('TARGET_REGISTRY_COUNT')
|
||||
24- out=[dict(x) for x in rows]
|
||||
--
|
||||
35- 'guest_type':'lxc','due_seconds':due,'enabled':True,'green1':True}
|
||||
36-
|
||||
37-def load_targets():
|
||||
38- base=canonical_targets();return base+[_ct_descriptor(base)]
|
||||
39-
|
||||
40:def backup_target(t,green_verify=False):
|
||||
41- if not isinstance(t,dict) or t.get('logical_id')!=CT_ID:
|
||||
42: return _legacy_backup_target(t,green_verify)
|
||||
43- try:
|
||||
44: r=ct112.run_target(t,state_writer=legacy.write_state)
|
||||
45- return {'logical_id':CT_ID,'generation':r['generation'],'result':'SUCCESS','payload_bytes':r['payload_bytes'],
|
||||
46- 'replicas':2,'manifest_sha256':r['manifest_sha256'],'payload_sha256':r['payload_sha256']}
|
||||
47- except BaseException:
|
||||
48: try: legacy.write_state(CT_ID,'FAIL',reason='CT112_PIPELINE_FAILED')
|
||||
49- except BaseException: pass
|
||||
50- raise legacy.BackupError('CT112_PIPELINE_FAILED') from None
|
||||
51-
|
||||
52-legacy.load_targets=load_targets
|
||||
53:legacy.backup_target=backup_target
|
||||
54-
|
||||
55-def _legacy_reused_boundary_selftest():
|
||||
56- if legacy.PART_SIZE!=536870912: raise RuntimeError('LEGACY_PART_SIZE')
|
||||
57- if len(legacy.REMOTES)!=13 or len({p for _,p in legacy.REMOTES})!=13: raise RuntimeError('LEGACY_REMOTES')
|
||||
58- d=legacy.default_targets()
|
||||
SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py SHA256=ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793
|
||||
57- if gp==legacy_path:
|
||||
58- found[id(g)]=g
|
||||
59- if len(found)!=1:
|
||||
60- raise RuntimeError('LEGACY_PROVENANCE_CARDINALITY')
|
||||
61- g=next(iter(found.values()))
|
||||
62: required=('capture_pve_guest','backup_target','load_targets','cluster_inventory','ssh_remote',
|
||||
63- '_default_unprivileged_lxc','mapped_scratch_probe','_stream_argv_to_local','run',
|
||||
64: 'choose_due_target','write_state','epoch')
|
||||
65- if any(not callable(g.get(n)) for n in required):
|
||||
66- raise RuntimeError('LEGACY_PROVENANCE_API')
|
||||
67- original=g['capture_pve_guest']
|
||||
68- proxy=_GlobalsProxy(g)
|
||||
69- if not isinstance(g.get('SCP'),str) or not g['SCP'].startswith('/'):
|
||||
--
|
||||
73- raise RuntimeError('QEMU_ADAPTER_BIND_RESULT')
|
||||
74- patched=lxc_adapter.bind(proxy,qemu_patched)
|
||||
75- if not callable(patched):
|
||||
76- raise RuntimeError('LXC_ADAPTER_BIND_RESULT')
|
||||
77- g['capture_pve_guest']=patched
|
||||
78: scheduler_wired=scheduler_adapter.bind(proxy,g['choose_due_target'],g['write_state'])
|
||||
79: if g.get('choose_due_target') is not scheduler_wired['choose_due_target'] or g.get('write_state') is not scheduler_wired['write_state']:
|
||||
80- raise RuntimeError('SCHEDULER_ADAPTER_BIND_RESULT')
|
||||
81- return {'legacy_globals':g,'legacy':proxy,'original_capture':original,'qemu_patched_capture':qemu_patched,'patched_capture':patched,'scheduler_wired':scheduler_wired}
|
||||
82-
|
||||
83-def _load():
|
||||
84- _verify_base()
|
||||
--
|
||||
105- wrapper,lxc_adapter,qemu_adapter,scheduler_adapter,wired=_load()
|
||||
106- if wired['legacy_globals'].get('capture_pve_guest') is not wired['patched_capture']:
|
||||
107- raise RuntimeError('PATCH_BINDING')
|
||||
108- if wired['legacy_globals'].get('choose_due_target') is not wired['scheduler_wired']['choose_due_target']:
|
||||
109- raise RuntimeError('SCHEDULER_CHOOSE_BINDING')
|
||||
110: if wired['legacy_globals'].get('write_state') is not wired['scheduler_wired']['write_state']:
|
||||
111- raise RuntimeError('SCHEDULER_WRITE_BINDING')
|
||||
112- if [scheduler_adapter.backoff_seconds(n) for n in range(1,7)]!=[1800,3600,7200,14400,21600,21600]:
|
||||
113- raise RuntimeError('SCHEDULER_BACKOFF_CONTRACT')
|
||||
114: if not callable(wrapper.backup_target) or not callable(wrapper.main) or not callable(wrapper.load_targets):
|
||||
115- raise RuntimeError('WRAPPER_PUBLIC_API')
|
||||
116- print('SELFTEST_OVERLAY504=PASS',flush=True)
|
||||
117- return 0
|
||||
118-
|
||||
119-def main(argv=None):
|
||||
--
|
||||
131- try:
|
||||
132- with tempfile.TemporaryDirectory(prefix='homelab-scheduler-proof-',dir='/run') as td:
|
||||
133- g['STATE_ROOT']=pathlib.Path(td); (g['STATE_ROOT']/'state').mkdir(mode=0o700)
|
||||
134- g['epoch']=lambda: now[0]
|
||||
135- common={'payload_sha256':'a'*64,'manifest_sha256':'b'*64,'replicas':['proof-r1','proof-r2']}
|
||||
136: legacy.write_state('proof-a','SUCCESS',generation='proof-a-good',**common)
|
||||
137: legacy.write_state('proof-b','SUCCESS',generation='proof-b-good',**common)
|
||||
138- now[0]=100000
|
||||
139: legacy.write_state('proof-a','FAIL',generation='proof-a-bad',reason='PROOF_FAILURE',sealed=False)
|
||||
140- state=legacy.load_state_all(); a=state['proof-a']
|
||||
141- targets=[{'logical_id':'proof-a','enabled':True,'due_seconds':100},{'logical_id':'proof-b','enabled':True,'due_seconds':100}]
|
||||
142- first=legacy.choose_due_target(targets,state,now[0])
|
||||
143- if not isinstance(first,dict) or first.get('logical_id')!='proof-b': raise RuntimeError('SCHEDULER_PROOF_PEER_NOT_SELECTED')
|
||||
144: retry=int(a.get('retry_after_epoch',0)); preserved=int(a.get('last_success_epoch',0))
|
||||
145: if preserved!=1000 or a.get('last_success_generation')!='proof-a-good' or int(a.get('consecutive_failures',0))!=1 or retry!=101800:
|
||||
146- raise RuntimeError('SCHEDULER_PROOF_FAILURE_STATE')
|
||||
147- now[0]=retry+1
|
||||
148- second=legacy.choose_due_target(targets,legacy.load_state_all(),now[0])
|
||||
149- if not isinstance(second,dict) or second.get('logical_id')!='proof-a': raise RuntimeError('SCHEDULER_PROOF_RETRY_NOT_REELIGIBLE')
|
||||
150: legacy.write_state('proof-a','SUCCESS',generation='proof-a-recovered',**common)
|
||||
151- recovered=legacy.load_state_all()['proof-a']
|
||||
152: if int(recovered.get('consecutive_failures',-1))!=0 or int(recovered.get('retry_after_epoch',-1))!=0:
|
||||
153- raise RuntimeError('SCHEDULER_PROOF_SUCCESS_RESET')
|
||||
154: proof={'failed_target':'proof-a','peer_selected_during_backoff':first['logical_id'],'retry_after_epoch':retry,'last_success_preserved':preserved==1000,'retry_target_after_backoff':second['logical_id'],'success_resets_failures':True,'production_state_root_used':False}
|
||||
155- print('SCHEDULER_FAILURE_ISOLATION_PROOF='+json.dumps(proof,separators=(',',':'),sort_keys=True),flush=True)
|
||||
156- return 0
|
||||
157- finally:
|
||||
158- g['STATE_ROOT']=old_root; g['epoch']=old_epoch
|
||||
159- if argv==['scheduler-select-live']:
|
||||
--
|
||||
174- if len(argv)==2 and argv[0]=='verify-target':
|
||||
175- t=_target(wrapper,argv[1])
|
||||
176- if t.get('logical_id')=='guest-pve01-lxc-112':
|
||||
177- raise RuntimeError('VERIFY_SYNTHETIC_CT112_FORBIDDEN')
|
||||
178- try:
|
||||
179: out=wrapper.backup_target(t,True)
|
||||
180- except legacy.BackupError as e:
|
||||
181- print('VERIFY_TARGET_RESULT=FAIL',flush=True)
|
||||
182- print('VERIFY_TARGET_REASON='+str(e.code),flush=True)
|
||||
183- return 2
|
||||
184- print('VERIFY_TARGET_RESULT='+json.dumps(out,separators=(',',':'),sort_keys=True),flush=True)
|
||||
SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/scheduler_adapter.py SHA256=a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96
|
||||
14- n=int(v)
|
||||
15- except Exception:
|
||||
16- return default
|
||||
17- return n if n>=0 else default
|
||||
18-
|
||||
19:def backoff_seconds(consecutive_failures:int)->int:
|
||||
20: n=max(1,_as_nonneg_int(consecutive_failures,1))
|
||||
21- return min(MAX_BACKOFF,MIN_BACKOFF*(2**min(n-1,4)))
|
||||
22-
|
||||
23-def _read_prior(legacy,logical_id):
|
||||
24- root=pathlib.Path(legacy.STATE_ROOT)/'state'
|
||||
25- p=root/(str(logical_id)+'.json')
|
||||
--
|
||||
60- if not isinstance(t,dict) or t.get('enabled') is not True: continue
|
||||
61- lid=t.get('logical_id'); seconds=_as_nonneg_int(t.get('due_seconds'),0)
|
||||
62- _need(isinstance(lid,str) and lid and seconds>0,'SCHEDULER_TARGET_SCHEMA')
|
||||
63- s=state.get(lid,{})
|
||||
64- if not isinstance(s,dict): s={}
|
||||
65: retry_after=_as_nonneg_int(s.get('retry_after_epoch'),0)
|
||||
66: if retry_after>now: continue
|
||||
67- last_success=_as_nonneg_int(s.get('last_success_epoch'),0)
|
||||
68- age=(now-last_success) if last_success and now>=last_success else (10**12 if last_success==0 else 0)
|
||||
69- if age<seconds: continue
|
||||
70- overdue=age-seconds
|
||||
71- due.append((overdue,-order,t))
|
||||
72- if not due: return None
|
||||
73- return max(due,key=lambda x:(x[0],x[1]))[2]
|
||||
74-
|
||||
75: def write_state(logical_id,result,**kw):
|
||||
76- now=_as_nonneg_int(legacy.epoch(),0)
|
||||
77- prior=_read_prior(legacy,logical_id)
|
||||
78- if result=='SUCCESS':
|
||||
79- clean=dict(kw)
|
||||
80: clean.update({'last_attempt_epoch':now,'consecutive_failures':0,'retry_after_epoch':0,'failure_backoff_seconds':0})
|
||||
81- return original_write(logical_id,result,**clean)
|
||||
82: failures=_as_nonneg_int(prior.get('consecutive_failures'),0)+1
|
||||
83- backoff=backoff_seconds(failures)
|
||||
84- clean=dict(kw)
|
||||
85- clean.update(_success_snapshot(prior))
|
||||
86: clean.update({'last_attempt_epoch':now,'consecutive_failures':failures,
|
||||
87: 'retry_after_epoch':now+backoff,'failure_backoff_seconds':backoff})
|
||||
88- return original_write(logical_id,result,**clean)
|
||||
89-
|
||||
90- legacy.choose_due_target=choose_due_target
|
||||
91: legacy.write_state=write_state
|
||||
92: return {'choose_due_target':choose_due_target,'write_state':write_state,
|
||||
93- 'original_choose':original_choose,'original_write':original_write}
|
||||
US_NETBIRD_SOURCE_HITS_END=1
|
||||
US_NETBIRD_RECENT_JOURNAL_BEGIN=1
|
||||
2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_TARGET=xf-hkmods
|
||||
2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED
|
||||
2026-09-21T20:30:51+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.
|
||||
2026-09-21T20:30:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_TARGET=md-relay
|
||||
2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_TARGET=spb-ingress
|
||||
2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_TARGET=router-netcraze
|
||||
2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_TARGET=xf-codevipe
|
||||
2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_TARGET=xf-zakrutim
|
||||
2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:27:32+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_TARGET=ct110
|
||||
2026-09-21T23:27:32+03:00 pve01 homelab-backup-v2[1495880]: LXC_EXECUTION_DIAG={"archive_bytes":546400504,"exception_class":null,"exception_message_sha256":null,"executor":"local-proven-pve270","helper_rc":0,"node":"pve01","producer_rc":0,"stderr_bytes_observed":1405,"stderr_bytes_retained":1405,"stderr_stream_sha256":"35adf6e0c931a6c547164a0716f388ca307d8c4aac70af8300f821f47bd0f01c","stderr_truncated":false}
|
||||
2026-09-21T23:28:28+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:30:13+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_TARGET=ct111
|
||||
2026-09-21T23:30:13+03:00 pve01 homelab-backup-v2[1511723]: LXC_EXECUTION_DIAG={"executor":"remote-bounded-ssh-cwd-root","node":"pve02","producer_rc":0,"stderr_bytes_observed":1035,"stderr_bytes_retained":1035,"stderr_stream_sha256":"ad5f87b168930991fbbf7aff71f42683f2bd8c3a7a0e678d12a09b665f4dd5af","stderr_truncated":false}
|
||||
2026-09-21T23:30:55+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T00:04:14+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_TARGET=vm210
|
||||
2026-09-22T00:05:17+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T02:02:23+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_TARGET=vm170
|
||||
2026-09-22T02:12:47+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED
|
||||
2026-09-22T02:37:13+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.
|
||||
2026-09-22T02:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_TARGET=control-bootstrap-v2
|
||||
2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T04:29:13+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_TARGET=vm9130
|
||||
2026-09-22T04:44:11+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T06:31:51+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_TARGET=vm211
|
||||
2026-09-22T06:32:52+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_TARGET=node-pve01
|
||||
2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_TARGET=us-netbird
|
||||
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_RESULT=FAIL
|
||||
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_REASON=PIPE_FAILED
|
||||
2026-09-22T08:53:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.
|
||||
2026-09-22T08:53:31+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
US_NETBIRD_RECENT_JOURNAL_END=1
|
||||
BACKUP_V2_TIMER_STATE_BEGIN=1
|
||||
MainPID=0
|
||||
Result=exit-code
|
||||
ExecMainStartTimestamp=Tue 2026-09-22 08:53:16 MSK
|
||||
ExecMainExitTimestamp=Tue 2026-09-22 08:53:31 MSK
|
||||
ExecMainStatus=2
|
||||
ActiveState=failed
|
||||
SubState=failed
|
||||
Unit=homelab-backup-v2.service
|
||||
NextElapseUSecRealtime=
|
||||
LastTriggerUSec=Tue 2026-09-22 08:53:16 MSK
|
||||
LoadState=loaded
|
||||
ActiveState=active
|
||||
SubState=waiting
|
||||
UnitFileState=enabled
|
||||
BACKUP_V2_TIMER_STATE_END=1
|
||||
CANONICAL_SKLADCHIK_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json
|
||||
CANONICAL_SKLADCHIK_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc
|
||||
CANONICAL_SKLADCHIK_STATE_BEGIN=1
|
||||
{"schema":"homelab-cloud-quorum-payload-v1","status":"SEALED","logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","archive_size_bytes":178950,"archive_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","part_count":1,"parts_manifest_sha256":"664ef74ec0f30cd3e7fef848794a8813e390f55295c5c6874c1f30867708f740","cloud_quorum":2,"cloud_a_verified":true,"cloud_b_verified":true,"part_hashes_verified":true,"seal_written":true,"cloud_a_seal_verified":true,"cloud_b_seal_verified":true,"local_payload_present":false,"local_cleanup":"OK","sealed_at_utc":"2026-09-21T17:50:41Z","updated_at_utc":"2026-09-21T17:50:43Z","remote_a_base":"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z","remote_b_base":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z"}
|
||||
CANONICAL_SKLADCHIK_STATE_END=1
|
||||
CANONICAL_SKLADCHIK_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json
|
||||
CANONICAL_SKLADCHIK_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e
|
||||
CANONICAL_SKLADCHIK_RESTORE_BEGIN=1
|
||||
{"result":"OK","logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","restore_source":"Cloud A","transport_state":"SEALED","payload_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","payload_sha256_verified":true,"payload_decompression_verified":null,"archive_recovery_readback_verified":null,"parts_manifest_verified":null,"bounded_staging":null,"full_local_payload_written":null,"validation_level":null,"validated_at_utc":"2026-09-21T17:50:46Z"}
|
||||
CANONICAL_SKLADCHIK_RESTORE_END=1
|
||||
CANONICAL_SKLADCHIK_STRICT_STATE=true
|
||||
CANONICAL_SKLADCHIK_STRICT_RESTORE=false
|
||||
CANONICAL_CLOUDB_MANIFEST_READ_RC=0
|
||||
CANONICAL_CLOUDB_MANIFEST_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
CANONICAL_CLOUDB_MANIFEST_SHA256=664ef74ec0f30cd3e7fef848794a8813e390f55295c5c6874c1f30867708f740
|
||||
CANONICAL_CLOUDB_MANIFEST_ROWS=1
|
||||
CANONICAL_CLOUDB_MANIFEST_TOTAL=178950
|
||||
CANONICAL_CLOUDB_MANIFEST_MATCH=true
|
||||
AUDIT_LEGACY_CLOUDB_CONTRACT_BEGIN=1
|
||||
except Exception: pass
|
||||
if lrc!=0: issues.append({"severity":"CRITICAL","code":"CLOUD_UNREADABLE","logical_id":"cloud-"+label,"root_cause":"RCLONE_LSF_FAILED"})
|
||||
cloud[label]=rec
|
||||
|
||||
try: dr=json.loads(DRSEAL.read_text(encoding="utf-8"))
|
||||
except Exception: dr=None
|
||||
if not isinstance(dr,dict) or dr.get("status")!="SEALED":
|
||||
issues.append({"severity":"CRITICAL","code":"DR_SEAL_MISSING","logical_id":"-","root_cause":"SITE_DR_FINAL_SEAL_NOT_SEALED"})
|
||||
|
||||
cloudb_ok=False
|
||||
if CLOUDB_HEALTH.is_file():
|
||||
txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace")
|
||||
kv={}
|
||||
for line in txt.splitlines():
|
||||
if "=" in line:
|
||||
k,v=line.split("=",1); kv[k]=v
|
||||
cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes")
|
||||
if not cloudb_ok:
|
||||
issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"})
|
||||
|
||||
source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
|
||||
hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
|
||||
if hrc==0: source["local_head"]=ho.strip()
|
||||
frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10)
|
||||
prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10)
|
||||
source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED")
|
||||
if frc==0 and fo.strip():
|
||||
lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
|
||||
if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
|
||||
if source["remote_head"]!=source["local_head"]:
|
||||
issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
|
||||
|
||||
status="GREEN" if not issues else "RED"
|
||||
for t in targets:
|
||||
print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None))
|
||||
for i in issues:
|
||||
AUDIT_LEGACY_CLOUDB_CONTRACT_END=1
|
||||
SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614
|
||||
SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b
|
||||
SOURCE_REMOTE_BRANCH_HEAD=ABSENT
|
||||
SOURCE_REMOTE_HEAD_RC=0
|
||||
SOURCE_REMOTE_BRANCH_RC=0
|
||||
SOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT
|
||||
US_NETBIRD_FAILURE_CLASS=STATE_FAILURE_DETAIL_PRESENT
|
||||
DECISION=HOLD_BACKUP_1116_ROOT_CAUSE_INCOMPLETE
|
||||
NEXT_GATE=RECONCILE_ONLY_BACKUP_1116_FAILURE
|
||||
TASK_RESULT=HOLD_HOMELAB_BACKUP_1115_ROOT_RCA
|
||||
CHANGES_MADE_BY_1116=false
|
||||
PRODUCT_MUTATION_BY_1116=false
|
||||
VM_MUTATION_BY_1116=false
|
||||
CLOUD_MUTATION_BY_1116=false
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_1115_ROOT_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+8
-8
File diff suppressed because one or more lines are too long
+556
-261
@@ -1,15 +1,15 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1
|
||||
STATUS=FAIL
|
||||
RC=3
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1114-due-restore-rca
|
||||
COMPONENT=homelab-backup-1115-root-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=723d1d786602a2ec8eea15aa8bcad0fba171dfbbd94f08d92cc2f8fa575695b2
|
||||
COMMAND_SHA256=a475f4109c57c07002ec72ba49450725574b1c08d698b9878c58257e7cc704c8
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
@@ -24,14 +24,14 @@ MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a
|
||||
SANITIZED_OUTPUT_SHA256=6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a
|
||||
RAW_EVIDENCE_SHA256=17278f059b28a5b338724f4e22fad8aa2db50e23143eeee6635c075bf93a9691
|
||||
SANITIZED_OUTPUT_SHA256=17278f059b28a5b338724f4e22fad8aa2db50e23143eeee6635c075bf93a9691
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1
|
||||
WORKERS2_BACKUP_1115_ROOT_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1114-due-restore-rca
|
||||
MUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_SCHEDULER_CLOUDB_REMOTE_AND_SOURCE_OFFHOST_CLASSIFICATION;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE
|
||||
COMPONENT=homelab-backup-1115-root-rca
|
||||
MUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_FAILURE_AND_CANONICAL_SKLADCHIK_EVIDENCE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE;NO_REMOTE_MUTATION
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
REFERENCE_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
@@ -39,289 +39,584 @@ ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4
|
||||
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
|
||||
AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
|
||||
RESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23
|
||||
BACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3
|
||||
SCHEDULER_RC=0
|
||||
SELFTEST_REPLACEMENT486=PASS
|
||||
SCHEDULER_LIVE_SELECTION={"due_seconds":86400,"enabled":true,"logical_id":"us-netbird"}
|
||||
SCHEDULER_CLASS=US_NETBIRD_DUE
|
||||
US_NETBIRD_STATE_BEGIN=1
|
||||
US_NETBIRD_STATE_SHA256=3fd55f07f513d33d8190aea4eaa952413758b1956cb1ed3677fbb656c20bc38a
|
||||
{"schema":1,"logical_id":"us-netbird","result":"FAIL","generation":"v2-us-netbird-20260921T233658Z-709cf01e1256","updated_at_utc":"2026-09-21T23:37:13Z","last_success_epoch":1789704670,"last_attempt_epoch":1790033833,"retry_after_epoch":1790055433,"consecutive_failures":13,"replica_count":null}
|
||||
US_NETBIRD_STATE_END=1
|
||||
BACKUP_V2_UNIT_STATE_BEGIN=1
|
||||
MainPID=0
|
||||
Result=success
|
||||
ExecMainStartTimestamp=Tue 2026-09-22 08:14:07 MSK
|
||||
ExecMainExitTimestamp=Tue 2026-09-22 08:14:07 MSK
|
||||
ExecMainStatus=0
|
||||
ActiveState=inactive
|
||||
SubState=dead
|
||||
BACKUP_V2_TIMER_ENABLED=enabled
|
||||
BACKUP_V2_TIMER_ACTIVE=active
|
||||
Unit=homelab-backup-v2.service
|
||||
NextElapseUSecRealtime=
|
||||
LastTriggerUSec=Tue 2026-09-22 08:14:07 MSK
|
||||
Result=success
|
||||
BACKUP_V2_UNIT_STATE_END=1
|
||||
BACKUP_V2_JOURNAL_US_NETBIRD_BEGIN=1
|
||||
2026-09-21T14:42:42+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T15:13:12+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T15:13:19+03:00 pve01 homelab-backup-v2[4054724]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T15:13:19+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T15:13:19+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T15:22:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
|
||||
US_NETBIRD_STATE_SAFE_BEGIN=1
|
||||
US_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26
|
||||
{"schema":1,"logical_id":"us-netbird","result":"FAIL","generation":"v2-us-netbird-20260922T055316Z-d5c92acdd2e8","updated_at_utc":"2026-09-22T05:53:31Z","last_success_epoch":1789704670,"last_attempt_epoch":1790056411,"retry_after_epoch":1790078011,"consecutive_failures":14,"replica_count":null}
|
||||
{"path":"consecutive_failures","value":14}
|
||||
{"path":"failure_backoff_seconds","value":21600}
|
||||
{"path":"last_attempt_epoch","value":1790056411}
|
||||
{"path":"last_success_epoch","value":1789704670}
|
||||
{"path":"last_success_generation","value":"v2-us-netbird-20260918T041032Z-d7648be4c16b"}
|
||||
{"path":"last_success_manifest_sha256","value":"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5"}
|
||||
{"path":"last_success_payload_sha256","value":"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf"}
|
||||
{"path":"reason","value":"PIPE_FAILED"}
|
||||
{"path":"result","value":"FAIL"}
|
||||
{"path":"retry_after_epoch","value":1790078011}
|
||||
US_NETBIRD_STATE_SAFE_END=1
|
||||
US_NETBIRD_SOURCE_HITS_BEGIN=1
|
||||
SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7
|
||||
155- for node in ('pve01','pve02','pve03'):
|
||||
156- out.append({'logical_id':'node-'+node,'scope':'node','type':'pve_node_bundle','node':node,'due_seconds':7*d,'enabled':True,'green1':node=='pve01'})
|
||||
157- out += [
|
||||
158- {'logical_id':'md-relay','scope':'vps','type':'vps_md','host':'[PRIVATE_IP]','port':22,'user':'ops','hostkey_fingerprint':'SHA256:DXJGwunC5tEVvnC3nRxyayTN8FXmyJcVgf+Oa6udQsY','due_seconds':d,'enabled':True,'green1':True},
|
||||
159- {'logical_id':'spb-ingress','scope':'vps','type':'vps_spb','host':'[PRIVATE_IP]','port':22,'user':'edgeadmin','hostkey_fingerprint':'SHA256:Ef+HsZb8lIXoadvYBXuGKIL3H7NyJMmf9EYCoJRoSv8','due_seconds':d,'enabled':True,'green1':True},
|
||||
160: {'logical_id':'us-netbird','scope':'vps','type':'vps_us','host':'[PRIVATE_IP]','port':22,'user':'root','hostkey_fingerprint':'SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE','due_seconds':d,'enabled':True,'green1':True},
|
||||
161- {'logical_id':'router-netcraze','scope':'router','type':'router','due_seconds':d,'enabled':True,'green1':True},
|
||||
162- ]
|
||||
163- sites=[
|
||||
164- ('codevipe',160,'pve02','/var/www/forums/codevipe/public','xf_codevipe','single_transaction'),
|
||||
165- ('dsmods',160,'pve02','/var/www/forums/dsmods/public','xf_dsmods','single_transaction'),
|
||||
--
|
||||
216- except subprocess.TimeoutExpired:
|
||||
217- for q in (src,z):
|
||||
218- try:q.kill()
|
||||
219- except Exception:pass
|
||||
220- raise BackupError('PIPE_TIMEOUT')
|
||||
221: if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}')
|
||||
222-
|
||||
223-def ensure_dir(p,mode=0o700):
|
||||
224- p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p
|
||||
225-
|
||||
226-def atomic_write(path,data,mode=0o600):
|
||||
--
|
||||
248- try:
|
||||
249- j=json.loads(p.read_text()); d[j['logical_id']]=j
|
||||
250- except Exception: continue
|
||||
251- return d
|
||||
252-
|
||||
253:def write_state(logical_id,result,**kw):
|
||||
254- if result not in RESULT_CLASSES: raise BackupError('RESULT_CLASS_INVALID')
|
||||
255- j={'schema':1,'logical_id':logical_id,'result':result,'updated_at_utc':utcnow(),**kw}
|
||||
256- if result=='SUCCESS': j['last_success_epoch']=epoch()
|
||||
257- atomic_write(STATE_ROOT/'state'/(logical_id+'.json'),canonical_json(j)+b'\n',0o600)
|
||||
258- return j
|
||||
--
|
||||
384- if not names: raise BackupError('TAR_ZST_EMPTY')
|
||||
385- if required_member and required_member not in names: raise BackupError('REQUIRED_MEMBER_MISSING')
|
||||
386-
|
||||
387-def validate_payload(t,path):
|
||||
388- typ=t['type']
|
||||
389: if typ in ('pve_node_bundle','vps_md','vps_spb','vps_us','router'):
|
||||
390- validate_tar_zst(path)
|
||||
391- elif typ=='pve_guest':
|
||||
392- if t['guest_type']=='lxc': validate_tar_zst(path)
|
||||
393- else:
|
||||
394- # Green set is intentionally CT-only. Scheduled QEMU backups still get dual remote byte readback.
|
||||
--
|
||||
405-def cluster_inventory():
|
||||
406- p=run(['pvesh','get','/cluster/resources','--type','vm','--output-format','json'],timeout=60)
|
||||
407- try:return json.loads(p.stdout)
|
||||
408- except Exception: raise BackupError('PVE_INVENTORY_JSON')
|
||||
409-
|
||||
410:def ssh_remote(node,argv,input=None,timeout=1800,check=True):
|
||||
411- cmd='PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec '+shlex.join([str(x) for x in argv])
|
||||
412- if node=='pve01': return run([str(x) for x in argv],input=input,timeout=timeout,check=check)
|
||||
413- return run([SSH,'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node,cmd],input=input,timeout=timeout,check=check)
|
||||
414-
|
||||
415-def _pve_dumpdir_spec(guest_type):
|
||||
--
|
||||
471- if i+1>=len(argv) or argv[i+1]!=str(scratch): raise BackupError('PROVEN_DUMP_ARGV_TMPDIR_BINDING')
|
||||
472- elif '--tmpdir='+str(scratch) not in argv: raise BackupError('PROVEN_DUMP_ARGV_NO_TMPDIR')
|
||||
473- return argv,job
|
||||
474-
|
||||
475-def _default_unprivileged_lxc(node,vmid):
|
||||
476: q=ssh_remote(node,['pct','config',str(vmid)],timeout=30)
|
||||
477- lines=[x.strip() for x in q.stdout.decode('utf-8','replace').splitlines()]
|
||||
478- if 'unprivileged: 1' not in lines: raise BackupError('LXC_NOT_UNPRIVILEGED')
|
||||
479- if any(x.startswith('lxc.idmap:') or x.startswith('lxc.idmap =') for x in lines): raise BackupError('LXC_CUSTOM_IDMAP_UNSUPPORTED')
|
||||
480-
|
||||
481-def mapped_scratch_probe(node,scratch):
|
||||
482- tool=shutil.which('lxc-usernsexec') or '/usr/bin/lxc-usernsexec'
|
||||
483- pos=[tool,'-m','u:0:100000:65536','-m','g:0:100000:65536','--','/usr/bin/test','-x',str(scratch)]
|
||||
484: q=ssh_remote(node,pos,timeout=30,check=False)
|
||||
485- if q.returncode!=0: raise BackupError('MAPPED_UID_ACCESS_PROBE_FAILED')
|
||||
486-
|
||||
487-def _stream_argv_to_local(node,argv,dest,timeout=7200):
|
||||
488- dest=pathlib.Path(dest)
|
||||
489- if node=='pve01': cmd=argv
|
||||
--
|
||||
509- if typ!=t['guest_type']: raise BackupError('PVE_GUEST_TYPE_DRIFT')
|
||||
510- if typ=='lxc':
|
||||
511- _require_exact_source(P270,P270_SHA,'P270_IDENTITY');_require_exact_source(P282,P282_SHA,'P282_IDENTITY')
|
||||
512- _default_unprivileged_lxc(node,vmid)
|
||||
513- scratch=f'/var/tmp/homelab-vzdump-v2-{gen}'
|
||||
514: ssh_remote(node,['install','-d','-m','710','-o','0','-g','100000',scratch],timeout=30)
|
||||
515: q=ssh_remote(node,['stat','-c','%a:%u:%g',scratch],timeout=30)
|
||||
516- if q.stdout.decode().strip()!='710:0:100000': raise BackupError('PVE_MAPPED_SCRATCH_ACCESS_CONTRACT')
|
||||
517- try:
|
||||
518- mapped_scratch_probe(node,scratch)
|
||||
519- argv,job=proven_lxc_dump_argv(t,row,node,scratch)
|
||||
520- local=work/'payload.bin'
|
||||
521- _stream_argv_to_local(node,argv,local,timeout=7200)
|
||||
522- if local.stat().st_size>int(job['archive_cap_bytes']): raise BackupError('VZDUMP_ARCHIVE_CAP_EXCEEDED')
|
||||
523- except Exception:
|
||||
524: ssh_remote(node,['rm','-rf','--',scratch],timeout=60,check=False);raise
|
||||
525- cleanup={'kind':'pve_source_dir','node':node,'path':scratch}
|
||||
526- return local,{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status'),'producer_contract':'proven-270-282-stream'},cleanup
|
||||
527- # QEMU path remains unchanged: bounded dedicated dumpdir with later cloud readback/reconstruction.
|
||||
528- srcdir=f'/var/tmp/homelab-vzdump-v2-{gen}'
|
||||
529: ssh_remote(node,['install','-d','-m','700','-o','0','-g','0',srcdir],timeout=30)
|
||||
530- try:
|
||||
531: ssh_remote(node,['vzdump',str(vmid),'--mode','snapshot','--compress','zstd','--dumpdir',srcdir],timeout=7200)
|
||||
532: p=ssh_remote(node,['find',srcdir,'-maxdepth','1','-type','f','-printf','%p\n'],timeout=30)
|
||||
533- files=[x for x in p.stdout.decode().splitlines() if x.strip()]
|
||||
534- if len(files)!=1: raise BackupError('VZDUMP_OUTPUT_NOT_UNIQUE')
|
||||
535- src=files[0];local=work/'payload.bin'
|
||||
536- if node=='pve01': local=pathlib.Path(src)
|
||||
537- else: run([SCP,'-q','-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node+':'+src,str(local)],timeout=7200)
|
||||
538- if not pathlib.Path(local).is_file() or pathlib.Path(local).stat().st_size<=0: raise BackupError('VZDUMP_PAYLOAD_MISSING')
|
||||
539- except Exception:
|
||||
540: ssh_remote(node,['rm','-rf','--',srcdir],timeout=60,check=False);raise
|
||||
541- return pathlib.Path(local),{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status')},{'kind':'pve_source_dir','node':node,'path':srcdir}
|
||||
542-
|
||||
543-def operator_key_fingerprint():
|
||||
544- if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING')
|
||||
545- p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30)
|
||||
--
|
||||
563- return good[0]
|
||||
564-
|
||||
565-def ensure_vps_known_hosts(targets):
|
||||
566- lines=[]
|
||||
567- for t in targets:
|
||||
568: if t['type'] not in ('vps_md','vps_spb','vps_us'): continue
|
||||
569- lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint']))
|
||||
570- atomic_write(VPS_KNOWN_HOSTS,('\n'.join(lines)+'\n').encode(),0o600)
|
||||
571-
|
||||
572-def vps_ssh_argv(t, *, use_agent=False):
|
||||
573- a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15']
|
||||
--
|
||||
598- tag=p.stdout.decode().strip()
|
||||
599- if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT')
|
||||
600- vps_run(t,['true'],timeout=30,use_agent=False)
|
||||
601- results[t['logical_id']]=tag
|
||||
602- # US was already operator-key based; prove it agentless too.
|
||||
603: us=[x for x in targets if x['type']=='vps_us'][0]
|
||||
604- vps_run(us,['true'],timeout=30,use_agent=False)
|
||||
605- results[us['logical_id']]='ALREADY_US_OPERATOR_KEY'
|
||||
606- return results
|
||||
607-
|
||||
608-def remote_hashset(t,command):
|
||||
609- p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False)
|
||||
610- return sha256_bytes(p.stdout)
|
||||
611-
|
||||
612:def capture_vps_md(t,gen,work):
|
||||
613- files=['etc/netbird/install.conf','var/lib/netbird/state.json','var/lib/netbird/service.json','var/lib/netbird/active_profile.json','var/lib/netbird/default.json']
|
||||
614- cmd="sudo -n sha256sum -- "+' '.join(shlex.quote('/'+x) for x in files)
|
||||
615- pre=remote_hashset(t,cmd)
|
||||
616- remote=shlex.join(['sudo','-n','tar','--numeric-owner','-C','/','-cf','-']+files)
|
||||
617- argv=vps_ssh_argv(t,use_agent=False)+[remote]
|
||||
618- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=600)
|
||||
619- post=remote_hashset(t,cmd)
|
||||
620- if pre!=post: raise BackupError('MD_SOURCE_CHANGED_DURING_CAPTURE')
|
||||
621- return payload,{'generation':gen,'source_class':'vps_md','consistency':'FILESET_PRE_POST_HASH_STABILITY','hashset_sha256':pre},None
|
||||
622-
|
||||
623:def capture_vps_spb(t,gen,work):
|
||||
624- hcmd="find /etc/nginx -xdev -type f -print0 | sort -z | xargs -0 sha256sum"
|
||||
625- pre=remote_hashset(t,hcmd)
|
||||
626- remote=shlex.join(['tar','--numeric-owner','-C','/','-cf','-','etc/nginx'])
|
||||
627- argv=vps_ssh_argv(t,use_agent=False)+[remote]
|
||||
628- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=900)
|
||||
--
|
||||
657- if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33)
|
||||
658- b=dst.serialize(); dst.close(); src.close()
|
||||
659- ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b))
|
||||
660-'''
|
||||
661-
|
||||
662:def capture_vps_us(t,gen,work):
|
||||
663- remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT])
|
||||
664- argv=vps_ssh_argv(t,use_agent=False)+[remote]
|
||||
665- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200)
|
||||
666: return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None
|
||||
667-
|
||||
668-def _simple_shell_assignments(text):
|
||||
669- vals={}
|
||||
670- for line in text.splitlines():
|
||||
671- m=re.match(r'^\s*(?:export\s+)?([A-Z_][A-Z0-9_]*)=(.*)$',line)
|
||||
--
|
||||
801-
|
||||
802-def qga_exec(node,vmid,guest_argv,input_bytes=None,timeout=600):
|
||||
803- args=['qm','guest','exec',str(vmid),'--synchronous','1','--timeout',str(int(timeout))]
|
||||
804- if input_bytes is not None: args += ['--pass-stdin','1']
|
||||
805- args += ['--']+[str(x) for x in guest_argv]
|
||||
806: p=ssh_remote(node,args,input=input_bytes,timeout=timeout+60)
|
||||
807- try:j=json.loads(p.stdout)
|
||||
808- except Exception: raise BackupError('QGA_JSON')
|
||||
809- rc=j.get('exitcode',j.get('exit-code'))
|
||||
810- if rc is None: rc=0 if j.get('exited') else None
|
||||
811- if rc not in (0,None): raise BackupError('QGA_GUEST_COMMAND_FAILED',str(rc))
|
||||
--
|
||||
854-
|
||||
855-def capture_node_bundle(t,gen,work):
|
||||
856- node=t['node']; raw=node_bundle_paths(node); paths=[]
|
||||
857- for rel in raw:
|
||||
858- if node=='pve01': ok=(pathlib.Path('/')/rel).exists()
|
||||
859: else: ok=(ssh_remote(node,['test','-e','/'+rel],timeout=20,check=False).returncode==0)
|
||||
860- if ok: paths.append(rel)
|
||||
861- required={'etc/network/interfaces','etc/hosts','etc/hostname','etc/pve/nodes/'+node}
|
||||
862- if not required.issubset(set(paths)): raise BackupError('NODE_BUNDLE_REQUIRED_PATH_MISSING')
|
||||
863- payload=work/'payload.tar.zst'
|
||||
864- argv=['tar','--numeric-owner','--xattrs','--acls','-C','/','-cf','-']+paths
|
||||
--
|
||||
870-
|
||||
871-def capture_target(t,gen,work):
|
||||
872- typ=t['type']
|
||||
873- if typ=='pve_guest': return capture_pve_guest(t,gen,work)
|
||||
874- if typ=='pve_node_bundle': return capture_node_bundle(t,gen,work)
|
||||
875: if typ=='vps_md': return capture_vps_md(t,gen,work)
|
||||
876: if typ=='vps_spb': return capture_vps_spb(t,gen,work)
|
||||
877: if typ=='vps_us': return capture_vps_us(t,gen,work)
|
||||
878- if typ=='router': return capture_router(t,gen,work)
|
||||
879- if typ=='xenforo': return capture_xenforo(t,gen,work)
|
||||
880- raise BackupError('CAPTURE_TYPE_UNKNOWN')
|
||||
881-
|
||||
882-def cleanup_source(cleanup):
|
||||
883- if not cleanup: return
|
||||
884- if cleanup.get('kind')=='pve_source_dir':
|
||||
885: ssh_remote(cleanup['node'],['rm','-rf','--',cleanup['path']],timeout=60)
|
||||
886-
|
||||
887-def control_verify(t):
|
||||
888- engine=pathlib.Path('/usr/local/sbin/homelab-control-backup')
|
||||
889- if sha256_file(engine)!='0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57': raise BackupError('CONTROL_ENGINE_IDENTITY')
|
||||
890- root=pathlib.Path('/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2')
|
||||
--
|
||||
895- if j.get('status')=='SEALED' and j.get('generation'): rows.append((p.stat().st_mtime,j['generation']))
|
||||
896- except Exception: continue
|
||||
897- if not rows: raise BackupError('CONTROL_SEALED_GENERATION_MISSING')
|
||||
898- gen=sorted(rows)[-1][1]
|
||||
899- p=run([str(engine),'verify',gen],timeout=1800)
|
||||
900: write_state(t['logical_id'],'SUCCESS',generation=gen,verification='existing_control_engine_verify',last_success_epoch=epoch())
|
||||
901- return {'logical_id':t['logical_id'],'generation':gen,'result':'SUCCESS','payload_bytes':0,'replicas':2}
|
||||
902-
|
||||
903:def backup_target(t,green_verify=False):
|
||||
904- if t['type']=='control': return control_verify(t)
|
||||
905- gen=generation_id(t['logical_id']); work=ensure_dir(WORK_ROOT/gen,0o700)
|
||||
906- payload=None; cleanup=None; sealed=False
|
||||
907- try:
|
||||
908- payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen
|
||||
--
|
||||
916- ha=reconstruct_replica(manifest,manifest['replicas'][0],a); validate_payload(t,a)
|
||||
SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py SHA256=2b5eb4fc0f6f466a0d3e5a007af987092b6c17baa3ff9241a6afc2c2e97df1d5
|
||||
2-from __future__ import annotations
|
||||
3-import errno,fcntl,importlib.util,os,sys
|
||||
4-from pathlib import Path
|
||||
5-ROOT=Path(__file__).resolve().parent
|
||||
6-CT_ID='guest-pve01-lxc-112'
|
||||
7:LEGACY_IDS=('node-pve01','md-relay','spb-ingress','us-netbird','router-netcraze','xf-codevipe','xf-zakrutim','control-bootstrap-v2')
|
||||
8-LOCK=Path('/run/homelab-backup-heavy.lock')
|
||||
9-
|
||||
10-def _load(name,path):
|
||||
11- spec=importlib.util.spec_from_file_location(name,str(path))
|
||||
12- if spec is None or spec.loader is None: raise RuntimeError('MODULE_SPEC')
|
||||
--
|
||||
14-
|
||||
15-legacy=_load('_backupv2_legacy486',ROOT/'legacy_engine.py')
|
||||
16-ct112=_load('_backupv2_ct112_486',ROOT/'ct112_handler.py')
|
||||
17-driver=_load('_backupv2_driver454_479',ROOT/'representative_driver.py')
|
||||
18-_legacy_load_targets=legacy.load_targets
|
||||
19:_legacy_backup_target=legacy.backup_target
|
||||
20-
|
||||
21-def canonical_targets():
|
||||
22- rows=_legacy_load_targets()
|
||||
23- if not isinstance(rows,list) or len(rows)!=28: raise legacy.BackupError('TARGET_REGISTRY_COUNT')
|
||||
24- out=[dict(x) for x in rows]
|
||||
--
|
||||
35- 'guest_type':'lxc','due_seconds':due,'enabled':True,'green1':True}
|
||||
36-
|
||||
37-def load_targets():
|
||||
38- base=canonical_targets();return base+[_ct_descriptor(base)]
|
||||
39-
|
||||
40:def backup_target(t,green_verify=False):
|
||||
41- if not isinstance(t,dict) or t.get('logical_id')!=CT_ID:
|
||||
42: return _legacy_backup_target(t,green_verify)
|
||||
43- try:
|
||||
44: r=ct112.run_target(t,state_writer=legacy.write_state)
|
||||
45- return {'logical_id':CT_ID,'generation':r['generation'],'result':'SUCCESS','payload_bytes':r['payload_bytes'],
|
||||
46- 'replicas':2,'manifest_sha256':r['manifest_sha256'],'payload_sha256':r['payload_sha256']}
|
||||
47- except BaseException:
|
||||
48: try: legacy.write_state(CT_ID,'FAIL',reason='CT112_PIPELINE_FAILED')
|
||||
49- except BaseException: pass
|
||||
50- raise legacy.BackupError('CT112_PIPELINE_FAILED') from None
|
||||
51-
|
||||
52-legacy.load_targets=load_targets
|
||||
53:legacy.backup_target=backup_target
|
||||
54-
|
||||
55-def _legacy_reused_boundary_selftest():
|
||||
56- if legacy.PART_SIZE!=536870912: raise RuntimeError('LEGACY_PART_SIZE')
|
||||
57- if len(legacy.REMOTES)!=13 or len({p for _,p in legacy.REMOTES})!=13: raise RuntimeError('LEGACY_REMOTES')
|
||||
58- d=legacy.default_targets()
|
||||
SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py SHA256=ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793
|
||||
57- if gp==legacy_path:
|
||||
58- found[id(g)]=g
|
||||
59- if len(found)!=1:
|
||||
60- raise RuntimeError('LEGACY_PROVENANCE_CARDINALITY')
|
||||
61- g=next(iter(found.values()))
|
||||
62: required=('capture_pve_guest','backup_target','load_targets','cluster_inventory','ssh_remote',
|
||||
63- '_default_unprivileged_lxc','mapped_scratch_probe','_stream_argv_to_local','run',
|
||||
64: 'choose_due_target','write_state','epoch')
|
||||
65- if any(not callable(g.get(n)) for n in required):
|
||||
66- raise RuntimeError('LEGACY_PROVENANCE_API')
|
||||
67- original=g['capture_pve_guest']
|
||||
68- proxy=_GlobalsProxy(g)
|
||||
69- if not isinstance(g.get('SCP'),str) or not g['SCP'].startswith('/'):
|
||||
--
|
||||
73- raise RuntimeError('QEMU_ADAPTER_BIND_RESULT')
|
||||
74- patched=lxc_adapter.bind(proxy,qemu_patched)
|
||||
75- if not callable(patched):
|
||||
76- raise RuntimeError('LXC_ADAPTER_BIND_RESULT')
|
||||
77- g['capture_pve_guest']=patched
|
||||
78: scheduler_wired=scheduler_adapter.bind(proxy,g['choose_due_target'],g['write_state'])
|
||||
79: if g.get('choose_due_target') is not scheduler_wired['choose_due_target'] or g.get('write_state') is not scheduler_wired['write_state']:
|
||||
80- raise RuntimeError('SCHEDULER_ADAPTER_BIND_RESULT')
|
||||
81- return {'legacy_globals':g,'legacy':proxy,'original_capture':original,'qemu_patched_capture':qemu_patched,'patched_capture':patched,'scheduler_wired':scheduler_wired}
|
||||
82-
|
||||
83-def _load():
|
||||
84- _verify_base()
|
||||
--
|
||||
105- wrapper,lxc_adapter,qemu_adapter,scheduler_adapter,wired=_load()
|
||||
106- if wired['legacy_globals'].get('capture_pve_guest') is not wired['patched_capture']:
|
||||
107- raise RuntimeError('PATCH_BINDING')
|
||||
108- if wired['legacy_globals'].get('choose_due_target') is not wired['scheduler_wired']['choose_due_target']:
|
||||
109- raise RuntimeError('SCHEDULER_CHOOSE_BINDING')
|
||||
110: if wired['legacy_globals'].get('write_state') is not wired['scheduler_wired']['write_state']:
|
||||
111- raise RuntimeError('SCHEDULER_WRITE_BINDING')
|
||||
112- if [scheduler_adapter.backoff_seconds(n) for n in range(1,7)]!=[1800,3600,7200,14400,21600,21600]:
|
||||
113- raise RuntimeError('SCHEDULER_BACKOFF_CONTRACT')
|
||||
114: if not callable(wrapper.backup_target) or not callable(wrapper.main) or not callable(wrapper.load_targets):
|
||||
115- raise RuntimeError('WRAPPER_PUBLIC_API')
|
||||
116- print('SELFTEST_OVERLAY504=PASS',flush=True)
|
||||
117- return 0
|
||||
118-
|
||||
119-def main(argv=None):
|
||||
--
|
||||
131- try:
|
||||
132- with tempfile.TemporaryDirectory(prefix='homelab-scheduler-proof-',dir='/run') as td:
|
||||
133- g['STATE_ROOT']=pathlib.Path(td); (g['STATE_ROOT']/'state').mkdir(mode=0o700)
|
||||
134- g['epoch']=lambda: now[0]
|
||||
135- common={'payload_sha256':'a'*64,'manifest_sha256':'b'*64,'replicas':['proof-r1','proof-r2']}
|
||||
136: legacy.write_state('proof-a','SUCCESS',generation='proof-a-good',**common)
|
||||
137: legacy.write_state('proof-b','SUCCESS',generation='proof-b-good',**common)
|
||||
138- now[0]=100000
|
||||
139: legacy.write_state('proof-a','FAIL',generation='proof-a-bad',reason='PROOF_FAILURE',sealed=False)
|
||||
140- state=legacy.load_state_all(); a=state['proof-a']
|
||||
141- targets=[{'logical_id':'proof-a','enabled':True,'due_seconds':100},{'logical_id':'proof-b','enabled':True,'due_seconds':100}]
|
||||
142- first=legacy.choose_due_target(targets,state,now[0])
|
||||
143- if not isinstance(first,dict) or first.get('logical_id')!='proof-b': raise RuntimeError('SCHEDULER_PROOF_PEER_NOT_SELECTED')
|
||||
144: retry=int(a.get('retry_after_epoch',0)); preserved=int(a.get('last_success_epoch',0))
|
||||
145: if preserved!=1000 or a.get('last_success_generation')!='proof-a-good' or int(a.get('consecutive_failures',0))!=1 or retry!=101800:
|
||||
146- raise RuntimeError('SCHEDULER_PROOF_FAILURE_STATE')
|
||||
147- now[0]=retry+1
|
||||
148- second=legacy.choose_due_target(targets,legacy.load_state_all(),now[0])
|
||||
149- if not isinstance(second,dict) or second.get('logical_id')!='proof-a': raise RuntimeError('SCHEDULER_PROOF_RETRY_NOT_REELIGIBLE')
|
||||
150: legacy.write_state('proof-a','SUCCESS',generation='proof-a-recovered',**common)
|
||||
151- recovered=legacy.load_state_all()['proof-a']
|
||||
152: if int(recovered.get('consecutive_failures',-1))!=0 or int(recovered.get('retry_after_epoch',-1))!=0:
|
||||
153- raise RuntimeError('SCHEDULER_PROOF_SUCCESS_RESET')
|
||||
154: proof={'failed_target':'proof-a','peer_selected_during_backoff':first['logical_id'],'retry_after_epoch':retry,'last_success_preserved':preserved==1000,'retry_target_after_backoff':second['logical_id'],'success_resets_failures':True,'production_state_root_used':False}
|
||||
155- print('SCHEDULER_FAILURE_ISOLATION_PROOF='+json.dumps(proof,separators=(',',':'),sort_keys=True),flush=True)
|
||||
156- return 0
|
||||
157- finally:
|
||||
158- g['STATE_ROOT']=old_root; g['epoch']=old_epoch
|
||||
159- if argv==['scheduler-select-live']:
|
||||
--
|
||||
174- if len(argv)==2 and argv[0]=='verify-target':
|
||||
175- t=_target(wrapper,argv[1])
|
||||
176- if t.get('logical_id')=='guest-pve01-lxc-112':
|
||||
177- raise RuntimeError('VERIFY_SYNTHETIC_CT112_FORBIDDEN')
|
||||
178- try:
|
||||
179: out=wrapper.backup_target(t,True)
|
||||
180- except legacy.BackupError as e:
|
||||
181- print('VERIFY_TARGET_RESULT=FAIL',flush=True)
|
||||
182- print('VERIFY_TARGET_REASON='+str(e.code),flush=True)
|
||||
183- return 2
|
||||
184- print('VERIFY_TARGET_RESULT='+json.dumps(out,separators=(',',':'),sort_keys=True),flush=True)
|
||||
SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/scheduler_adapter.py SHA256=a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96
|
||||
14- n=int(v)
|
||||
15- except Exception:
|
||||
16- return default
|
||||
17- return n if n>=0 else default
|
||||
18-
|
||||
19:def backoff_seconds(consecutive_failures:int)->int:
|
||||
20: n=max(1,_as_nonneg_int(consecutive_failures,1))
|
||||
21- return min(MAX_BACKOFF,MIN_BACKOFF*(2**min(n-1,4)))
|
||||
22-
|
||||
23-def _read_prior(legacy,logical_id):
|
||||
24- root=pathlib.Path(legacy.STATE_ROOT)/'state'
|
||||
25- p=root/(str(logical_id)+'.json')
|
||||
--
|
||||
60- if not isinstance(t,dict) or t.get('enabled') is not True: continue
|
||||
61- lid=t.get('logical_id'); seconds=_as_nonneg_int(t.get('due_seconds'),0)
|
||||
62- _need(isinstance(lid,str) and lid and seconds>0,'SCHEDULER_TARGET_SCHEMA')
|
||||
63- s=state.get(lid,{})
|
||||
64- if not isinstance(s,dict): s={}
|
||||
65: retry_after=_as_nonneg_int(s.get('retry_after_epoch'),0)
|
||||
66: if retry_after>now: continue
|
||||
67- last_success=_as_nonneg_int(s.get('last_success_epoch'),0)
|
||||
68- age=(now-last_success) if last_success and now>=last_success else (10**12 if last_success==0 else 0)
|
||||
69- if age<seconds: continue
|
||||
70- overdue=age-seconds
|
||||
71- due.append((overdue,-order,t))
|
||||
72- if not due: return None
|
||||
73- return max(due,key=lambda x:(x[0],x[1]))[2]
|
||||
74-
|
||||
75: def write_state(logical_id,result,**kw):
|
||||
76- now=_as_nonneg_int(legacy.epoch(),0)
|
||||
77- prior=_read_prior(legacy,logical_id)
|
||||
78- if result=='SUCCESS':
|
||||
79- clean=dict(kw)
|
||||
80: clean.update({'last_attempt_epoch':now,'consecutive_failures':0,'retry_after_epoch':0,'failure_backoff_seconds':0})
|
||||
81- return original_write(logical_id,result,**clean)
|
||||
82: failures=_as_nonneg_int(prior.get('consecutive_failures'),0)+1
|
||||
83- backoff=backoff_seconds(failures)
|
||||
84- clean=dict(kw)
|
||||
85- clean.update(_success_snapshot(prior))
|
||||
86: clean.update({'last_attempt_epoch':now,'consecutive_failures':failures,
|
||||
87: 'retry_after_epoch':now+backoff,'failure_backoff_seconds':backoff})
|
||||
88- return original_write(logical_id,result,**clean)
|
||||
89-
|
||||
90- legacy.choose_due_target=choose_due_target
|
||||
91: legacy.write_state=write_state
|
||||
92: return {'choose_due_target':choose_due_target,'write_state':write_state,
|
||||
93- 'original_choose':original_choose,'original_write':original_write}
|
||||
US_NETBIRD_SOURCE_HITS_END=1
|
||||
US_NETBIRD_RECENT_JOURNAL_BEGIN=1
|
||||
2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_TARGET=xf-hkmods
|
||||
2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T20:30:35+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T20:30:35+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T20:30:35+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED
|
||||
2026-09-21T20:30:51+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.
|
||||
2026-09-21T20:30:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T21:03:24+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_TARGET=md-relay
|
||||
2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T21:03:57+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T21:03:57+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T21:37:26+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_TARGET=spb-ingress
|
||||
2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T21:37:49+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T21:37:49+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T22:17:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_TARGET=router-netcraze
|
||||
2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T22:17:41+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T22:17:41+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T22:49:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_TARGET=xf-codevipe
|
||||
2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:02:25+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:02:25+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:11:13+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_TARGET=xf-zakrutim
|
||||
2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:25:05+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:25:05+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:25:14+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:27:32+03:00 pve01 vzdump[1496016]: INFO: Backup job finished successfully
|
||||
2026-09-21T23:27:32+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_TARGET=ct110
|
||||
2026-09-21T23:27:32+03:00 pve01 homelab-backup-v2[1495880]: LXC_EXECUTION_DIAG={"archive_bytes":546400504,"exception_class":null,"exception_message_sha256":null,"executor":"local-proven-pve270","helper_rc":0,"node":"pve01","producer_rc":0,"stderr_bytes_observed":1405,"stderr_bytes_retained":1405,"stderr_stream_sha256":"35adf6e0c931a6c547164a0716f388ca307d8c4aac70af8300f821f47bd0f01c","stderr_truncated":false}
|
||||
2026-09-21T23:28:28+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:28:28+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:28:28+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:28:33+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:30:13+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_TARGET=ct111
|
||||
2026-09-21T23:30:13+03:00 pve01 homelab-backup-v2[1511723]: LXC_EXECUTION_DIAG={"executor":"remote-bounded-ssh-cwd-root","node":"pve02","producer_rc":0,"stderr_bytes_observed":1035,"stderr_bytes_retained":1035,"stderr_stream_sha256":"ad5f87b168930991fbbf7aff71f42683f2bd8c3a7a0e678d12a09b665f4dd5af","stderr_truncated":false}
|
||||
2026-09-21T23:30:55+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:30:55+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:30:55+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:31:01+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:31:01+03:00 pve01 homelab-backup-v2[1515875]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-21T23:31:01+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:31:01+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:31:09+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:31:10+03:00 pve01 homelab-backup-v2[1524090]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-21T23:31:10+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:31:10+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:31:18+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:31:18+03:00 pve01 homelab-backup-v2[1526070]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-21T23:31:18+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:31:18+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T00:03:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T00:04:14+03:00 pve01 vzdump[1627438]: INFO: Backup job finished successfully
|
||||
2026-09-22T00:04:14+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_TARGET=vm210
|
||||
2026-09-22T00:05:17+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T00:05:17+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T00:05:17+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T00:38:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T00:38:32+03:00 pve01 homelab-backup-v2[1741601]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T00:38:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T00:38:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T01:38:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T01:38:16+03:00 pve01 homelab-backup-v2[1935141]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T01:38:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T01:38:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T02:00:52+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T02:02:23+03:00 pve01 vzdump[2000008]: INFO: Backup job finished successfully
|
||||
2026-09-22T02:02:23+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_TARGET=vm170
|
||||
2026-09-22T02:12:47+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T02:12:47+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T02:12:47+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T02:36:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED
|
||||
2026-09-22T02:37:13+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.
|
||||
2026-09-22T02:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T03:12:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T03:12:32+03:00 pve01 homelab-backup-v2[2223458]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T03:12:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T03:12:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T03:46:25+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_TARGET=control-bootstrap-v2
|
||||
2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T03:46:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T03:46:31+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T04:21:11+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T04:29:13+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_TARGET=vm9130
|
||||
2026-09-22T04:44:11+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T04:44:11+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T04:44:11+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T04:53:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T04:53:59+03:00 pve01 homelab-backup-v2[2612150]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T04:53:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T04:53:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T05:27:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T05:27:59+03:00 pve01 homelab-backup-v2[2723689]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T05:27:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T05:27:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T05:58:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T05:58:16+03:00 pve01 homelab-backup-v2[2823098]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T05:58:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T05:58:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T06:31:19+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T06:31:51+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_TARGET=vm211
|
||||
2026-09-22T06:32:52+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T06:32:52+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T06:32:52+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T07:09:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_TARGET=node-pve01
|
||||
2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T07:09:46+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T07:09:46+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T07:42:15+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T07:42:15+03:00 pve01 homelab-backup-v2[3148085]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T07:42:15+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T07:42:15+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T08:14:07+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T08:14:07+03:00 pve01 homelab-backup-v2[3251854]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T08:14:07+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T08:14:07+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
BACKUP_V2_JOURNAL_US_NETBIRD_END=1
|
||||
RESTORE_SOURCE_SEGMENT_BEGIN=1
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf
|
||||
[ -r "$CONF" ] && . "$CONF"
|
||||
: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}"
|
||||
: "${CLOUD_1:=pve01-mail-01-crypt:}"
|
||||
: "${CLOUD_2:=pve01-mail-02-crypt:}"
|
||||
: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}"
|
||||
: "${KEEP_LOCAL:=10}"
|
||||
LOGDIR=/var/log/skladchik-reports-monitor-backup
|
||||
HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
|
||||
LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock
|
||||
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
WORK="${WORK_ROOT}/${RUN_ID}"
|
||||
mkdir -p "$WORK" "$LOGDIR" "$(dirname "$HEALTH")"
|
||||
exec 9>"$LOCK"
|
||||
if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi
|
||||
log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; }
|
||||
fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; }
|
||||
trap fail ERR
|
||||
command -v rclone >/dev/null
|
||||
log "START run_id=$RUN_ID"
|
||||
cat_chunked(){
|
||||
local remote_path="$1" out="$2" plist part
|
||||
: > "$out"
|
||||
plist="$(mktemp)"
|
||||
if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then
|
||||
while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort)
|
||||
else
|
||||
rclone copyto "$remote_path" "$out" >/dev/null
|
||||
fi
|
||||
rm -f "$plist"
|
||||
[ -s "$out" ]
|
||||
}
|
||||
check_one_remote(){
|
||||
local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status
|
||||
rbase="${remote}${REMOTE_BASE}/daily"
|
||||
latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)"
|
||||
[ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; }
|
||||
archive_name="skladchik-reports-monitor-${latest}.tar.gz"
|
||||
remote_archive="${rbase}/${latest}/${archive_name}"
|
||||
outdir="${WORK}/${label}-${latest}"
|
||||
mkdir -p "$outdir"
|
||||
archive="${outdir}/${archive_name}"
|
||||
log "${label} DOWNLOAD run=${latest}"
|
||||
cat_chunked "$remote_archive" "$archive"
|
||||
cat_chunked "${remote_archive}.sha256" "${archive}.sha256.remote"
|
||||
expected="$(awk '{print $1; exit}' "${archive}.sha256.remote")"
|
||||
actual="$(sha256sum "$archive" | awk '{print $1}')"
|
||||
[ -n "$expected" ] && [ "$expected" = "$actual" ]
|
||||
list="${outdir}/tar-list.txt"
|
||||
tar -tzf "$archive" > "$list"
|
||||
grep -Eq '(^|/)etc-skladchik-reports-monitor/config\.ini$' "$list"
|
||||
grep -Eq '(^|/)etc-skladchik-reports-monitor/secrets\.env$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/monitor\.py$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/health_alert\.py$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/state\.sqlite$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/alert-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/cookies\.jar$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.timer$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/operational_digest\.py$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-runner\.py$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-core$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/weekly-summary-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/age-alert-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/event-selftest-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.timer$' "$list"
|
||||
extract="${outdir}/extract"
|
||||
mkdir -p "$extract"
|
||||
tar -xzf "$archive" -C "$extract"
|
||||
sqlite_status=skipped
|
||||
if command -v sqlite3 >/dev/null && [ -s "$extract/var-lib-skladchik-reports-monitor/state.sqlite" ]; then
|
||||
[ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ]
|
||||
sqlite_status=ok
|
||||
fi
|
||||
py_status=skipped
|
||||
if command -v python3 >/dev/null; then
|
||||
python3 -m py_compile "$extract/opt-skladchik-reports-monitor/monitor.py" "$extract/opt-skladchik-reports-monitor/health_alert.py" "$extract/opt-skladchik-reports-monitor/operational_digest.py" "$extract/opt-skladchik-reports-monitor/event-selftest-runner.py" "$extract/opt-skladchik-reports-monitor/event-selftest-core"
|
||||
py_status=ok
|
||||
fi
|
||||
printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env"
|
||||
log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}"
|
||||
}
|
||||
: > "$WORK/result.env"
|
||||
check_one_remote CLOUD_1 "$CLOUD_1"
|
||||
check_one_remote CLOUD_2 "$CLOUD_2"
|
||||
. "$WORK/result.env"
|
||||
printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH"
|
||||
find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done
|
||||
log "OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}"
|
||||
RESTORE_SOURCE_SEGMENT_END=1
|
||||
RESTORE_SYSTEMD_BINDINGS_BEGIN=1
|
||||
RESTORE_SYSTEMD_BINDING_COUNT=0
|
||||
RESTORE_SYSTEMD_BINDINGS_END=1
|
||||
CLOUD_1_DAILY_LSF_RC=3
|
||||
CLOUD_1_DAILY_STDERR_SHA256=bb3ce24b6401f914e787d66ae4a3fb5460455a41fab035e6ede56f0af463781f
|
||||
2026/09/22 08:47:57 ERROR : : error listing: directory not found
|
||||
2026/09/22 08:47:57 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found
|
||||
CLOUD_2_DAILY_LSF_RC=3
|
||||
CLOUD_2_DAILY_STDERR_SHA256=a3c018181b7d581d2a06e107cb28685794108bc2066f6acc7abdfbc83545fefa
|
||||
2026/09/22 08:47:58 ERROR : : error listing: directory not found
|
||||
2026/09/22 08:47:58 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found
|
||||
CLOUDB_REMOTE_READ_CLASS=INCOMPLETE_OR_ERROR
|
||||
SOURCE_WORKTREE_VALID=true
|
||||
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_TARGET=us-netbird
|
||||
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_RESULT=FAIL
|
||||
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_REASON=PIPE_FAILED
|
||||
2026-09-22T08:53:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.
|
||||
2026-09-22T08:53:31+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
US_NETBIRD_RECENT_JOURNAL_END=1
|
||||
BACKUP_V2_TIMER_STATE_BEGIN=1
|
||||
MainPID=0
|
||||
Result=exit-code
|
||||
ExecMainStartTimestamp=Tue 2026-09-22 08:53:16 MSK
|
||||
ExecMainExitTimestamp=Tue 2026-09-22 08:53:31 MSK
|
||||
ExecMainStatus=2
|
||||
ActiveState=failed
|
||||
SubState=failed
|
||||
Unit=homelab-backup-v2.service
|
||||
NextElapseUSecRealtime=
|
||||
LastTriggerUSec=Tue 2026-09-22 08:53:16 MSK
|
||||
LoadState=loaded
|
||||
ActiveState=active
|
||||
SubState=waiting
|
||||
UnitFileState=enabled
|
||||
BACKUP_V2_TIMER_STATE_END=1
|
||||
CANONICAL_SKLADCHIK_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json
|
||||
CANONICAL_SKLADCHIK_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc
|
||||
CANONICAL_SKLADCHIK_STATE_BEGIN=1
|
||||
{"schema":"homelab-cloud-quorum-payload-v1","status":"SEALED","logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","archive_size_bytes":178950,"archive_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","part_count":1,"parts_manifest_sha256":"664ef74ec0f30cd3e7fef848794a8813e390f55295c5c6874c1f30867708f740","cloud_quorum":2,"cloud_a_verified":true,"cloud_b_verified":true,"part_hashes_verified":true,"seal_written":true,"cloud_a_seal_verified":true,"cloud_b_seal_verified":true,"local_payload_present":false,"local_cleanup":"OK","sealed_at_utc":"2026-09-21T17:50:41Z","updated_at_utc":"2026-09-21T17:50:43Z","remote_a_base":"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z","remote_b_base":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z"}
|
||||
CANONICAL_SKLADCHIK_STATE_END=1
|
||||
CANONICAL_SKLADCHIK_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json
|
||||
CANONICAL_SKLADCHIK_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e
|
||||
CANONICAL_SKLADCHIK_RESTORE_BEGIN=1
|
||||
{"result":"OK","logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","restore_source":"Cloud A","transport_state":"SEALED","payload_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","payload_sha256_verified":true,"payload_decompression_verified":null,"archive_recovery_readback_verified":null,"parts_manifest_verified":null,"bounded_staging":null,"full_local_payload_written":null,"validation_level":null,"validated_at_utc":"2026-09-21T17:50:46Z"}
|
||||
CANONICAL_SKLADCHIK_RESTORE_END=1
|
||||
CANONICAL_SKLADCHIK_STRICT_STATE=true
|
||||
CANONICAL_SKLADCHIK_STRICT_RESTORE=false
|
||||
CANONICAL_CLOUDB_MANIFEST_READ_RC=0
|
||||
CANONICAL_CLOUDB_MANIFEST_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
CANONICAL_CLOUDB_MANIFEST_SHA256=664ef74ec0f30cd3e7fef848794a8813e390f55295c5c6874c1f30867708f740
|
||||
CANONICAL_CLOUDB_MANIFEST_ROWS=1
|
||||
CANONICAL_CLOUDB_MANIFEST_TOTAL=178950
|
||||
CANONICAL_CLOUDB_MANIFEST_MATCH=true
|
||||
AUDIT_LEGACY_CLOUDB_CONTRACT_BEGIN=1
|
||||
except Exception: pass
|
||||
if lrc!=0: issues.append({"severity":"CRITICAL","code":"CLOUD_UNREADABLE","logical_id":"cloud-"+label,"root_cause":"RCLONE_LSF_FAILED"})
|
||||
cloud[label]=rec
|
||||
|
||||
try: dr=json.loads(DRSEAL.read_text(encoding="utf-8"))
|
||||
except Exception: dr=None
|
||||
if not isinstance(dr,dict) or dr.get("status")!="SEALED":
|
||||
issues.append({"severity":"CRITICAL","code":"DR_SEAL_MISSING","logical_id":"-","root_cause":"SITE_DR_FINAL_SEAL_NOT_SEALED"})
|
||||
|
||||
cloudb_ok=False
|
||||
if CLOUDB_HEALTH.is_file():
|
||||
txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace")
|
||||
kv={}
|
||||
for line in txt.splitlines():
|
||||
if "=" in line:
|
||||
k,v=line.split("=",1); kv[k]=v
|
||||
cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes")
|
||||
if not cloudb_ok:
|
||||
issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"})
|
||||
|
||||
source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
|
||||
hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
|
||||
if hrc==0: source["local_head"]=ho.strip()
|
||||
frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10)
|
||||
prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10)
|
||||
source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED")
|
||||
if frc==0 and fo.strip():
|
||||
lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
|
||||
if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
|
||||
if source["remote_head"]!=source["local_head"]:
|
||||
issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
|
||||
|
||||
status="GREEN" if not issues else "RED"
|
||||
for t in targets:
|
||||
print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None))
|
||||
for i in issues:
|
||||
AUDIT_LEGACY_CLOUDB_CONTRACT_END=1
|
||||
SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614
|
||||
SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_PUSHURL_CLASS=DISABLED
|
||||
SOURCE_REMOTE_HEAD_PROBE_RC=0
|
||||
SOURCE_REMOTE_BRANCH_PROBE_RC=0
|
||||
SOURCE_REMOTE_ALL_PROBE_RC=0
|
||||
SOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b
|
||||
SOURCE_REMOTE_BRANCH_HEAD=ABSENT
|
||||
SOURCE_LOCAL_COMMIT_REMOTE_REF_COUNT=0
|
||||
SOURCE_REMOTE_HEAD_RC=0
|
||||
SOURCE_REMOTE_BRANCH_RC=0
|
||||
SOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT
|
||||
DECISION=HOLD_BACKUP_1115_ONE_OR_MORE_PATHS_UNSAFE
|
||||
NEXT_GATE=RECONCILE_ONLY_BACKUP_1115_FAILURE
|
||||
TASK_RESULT=HOLD_HOMELAB_BACKUP_1114_DUE_RESTORE_RCA
|
||||
CHANGES_MADE_BY_1115=false
|
||||
PRODUCT_MUTATION_BY_1115=false
|
||||
VM_MUTATION_BY_1115=false
|
||||
CLOUD_MUTATION_BY_1115=false
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_END=1
|
||||
US_NETBIRD_FAILURE_CLASS=STATE_FAILURE_DETAIL_PRESENT
|
||||
DECISION=HOLD_BACKUP_1116_ROOT_CAUSE_INCOMPLETE
|
||||
NEXT_GATE=RECONCILE_ONLY_BACKUP_1116_FAILURE
|
||||
TASK_RESULT=HOLD_HOMELAB_BACKUP_1115_ROOT_RCA
|
||||
CHANGES_MADE_BY_1116=false
|
||||
PRODUCT_MUTATION_BY_1116=false
|
||||
VM_MUTATION_BY_1116=false
|
||||
CLOUD_MUTATION_BY_1116=false
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_1115_ROOT_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user