From 46ee9c59c9f2ed3f6fe3b445428fc717e5b23919 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 22 Sep 2026 05:57:36 +0000 Subject: [PATCH] runtime: publish SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1 --- ...2-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1.json | 38 + ...22-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1.txt | 622 +++++++++++++ runtime/latest.json | 16 +- runtime/latest.txt | 817 ++++++++++++------ 4 files changed, 1224 insertions(+), 269 deletions(-) create mode 100644 runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1.json create mode 100644 runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1.txt diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1.json b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1.json new file mode 100644 index 00000000..28668f77 --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1", + "status": "FAIL", + "rc": 3, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-1115-root-rca", + "started_at_utc": "2026-09-22T05:57:35Z", + "finished_at_utc": "2026-09-22T05:57:35Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "a475f4109c57c07002ec72ba49450725574b1c08d698b9878c58257e7cc704c8", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 3, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "17278f059b28a5b338724f4e22fad8aa2db50e23143eeee6635c075bf93a9691", + "sanitized_output_sha256": "17278f059b28a5b338724f4e22fad8aa2db50e23143eeee6635c075bf93a9691", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_1115_ROOT_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1\nMODE=read-only\nCOMPONENT=homelab-backup-1115-root-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_FAILURE_AND_CANONICAL_SKLADCHIK_EVIDENCE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE;NO_REMOTE_MUTATION\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nUS_NETBIRD_STATE_SAFE_BEGIN=1\nUS_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26\n{\"schema\":1,\"logical_id\":\"us-netbird\",\"result\":\"FAIL\",\"generation\":\"v2-us-netbird-20260922T055316Z-d5c92acdd2e8\",\"updated_at_utc\":\"2026-09-22T05:53:31Z\",\"last_success_epoch\":1789704670,\"last_attempt_epoch\":1790056411,\"retry_after_epoch\":1790078011,\"consecutive_failures\":14,\"replica_count\":null}\n{\"path\":\"consecutive_failures\",\"value\":14}\n{\"path\":\"failure_backoff_seconds\",\"value\":21600}\n{\"path\":\"last_attempt_epoch\",\"value\":1790056411}\n{\"path\":\"last_success_epoch\",\"value\":1789704670}\n{\"path\":\"last_success_generation\",\"value\":\"v2-us-netbird-20260918T041032Z-d7648be4c16b\"}\n{\"path\":\"last_success_manifest_sha256\",\"value\":\"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5\"}\n{\"path\":\"last_success_payload_sha256\",\"value\":\"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf\"}\n{\"path\":\"reason\",\"value\":\"PIPE_FAILED\"}\n{\"path\":\"result\",\"value\":\"FAIL\"}\n{\"path\":\"retry_after_epoch\",\"value\":1790078011}\nUS_NETBIRD_STATE_SAFE_END=1\nUS_NETBIRD_SOURCE_HITS_BEGIN=1\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7\n155- for node in ('pve01','pve02','pve03'):\n156- out.append({'logical_id':'node-'+node,'scope':'node','type':'pve_node_bundle','node':node,'due_seconds':7*d,'enabled':True,'green1':node=='pve01'})\n157- out += [\n158- {'logical_id':'md-relay','scope':'vps','type':'vps_md','host':'[PRIVATE_IP]','port':22,'user':'ops','hostkey_fingerprint':'SHA256:DXJGwunC5tEVvnC3nRxyayTN8FXmyJcVgf+Oa6udQsY','due_seconds':d,'enabled':True,'green1':True},\n159- {'logical_id':'spb-ingress','scope':'vps','type':'vps_spb','host':'[PRIVATE_IP]','port':22,'user':'edgeadmin','hostkey_fingerprint':'SHA256:Ef+HsZb8lIXoadvYBXuGKIL3H7NyJMmf9EYCoJRoSv8','due_seconds':d,'enabled':True,'green1':True},\n160: {'logical_id':'us-netbird','scope':'vps','type':'vps_us','host':'[PRIVATE_IP]','port':22,'user':'root','hostkey_fingerprint':'SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE','due_seconds':d,'enabled':True,'green1':True},\n161- {'logical_id':'router-netcraze','scope':'router','type':'router','due_seconds':d,'enabled':True,'green1':True},\n162- ]\n163- sites=[\n164- ('codevipe',160,'pve02','/var/www/forums/codevipe/public','xf_codevipe','single_transaction'),\n165- ('dsmods',160,'pve02','/var/www/forums/dsmods/public','xf_dsmods','single_transaction'),\n--\n216- except subprocess.TimeoutExpired:\n217- for q in (src,z):\n218- try:q.kill()\n219- except Exception:pass\n220- raise BackupError('PIPE_TIMEOUT')\n221: if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}')\n222-\n223-def ensure_dir(p,mode=0o700):\n224- p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p\n225-\n226-def atomic_write(path,data,mode=0o600):\n--\n248- try:\n249- j=json.loads(p.read_text()); d[j['logical_id']]=j\n250- except Exception: continue\n251- return d\n252-\n253:def write_state(logical_id,result,**kw):\n254- if result not in RESULT_CLASSES: raise BackupError('RESULT_CLASS_INVALID')\n255- j={'schema':1,'logical_id':logical_id,'result':result,'updated_at_utc':utcnow(),**kw}\n256- if result=='SUCCESS': j['last_success_epoch']=epoch()\n257- atomic_write(STATE_ROOT/'state'/(logical_id+'.json'),canonical_json(j)+b'\\n',0o600)\n258- return j\n--\n384- if not names: raise BackupError('TAR_ZST_EMPTY')\n385- if required_member and required_member not in names: raise BackupError('REQUIRED_MEMBER_MISSING')\n386-\n387-def validate_payload(t,path):\n388- typ=t['type']\n389: if typ in ('pve_node_bundle','vps_md','vps_spb','vps_us','router'):\n390- validate_tar_zst(path)\n391- elif typ=='pve_guest':\n392- if t['guest_type']=='lxc': validate_tar_zst(path)\n393- else:\n394- # Green set is intentionally CT-only. Scheduled QEMU backups still get dual remote byte readback.\n--\n405-def cluster_inventory():\n406- p=run(['pvesh','get','/cluster/resources','--type','vm','--output-format','json'],timeout=60)\n407- try:return json.loads(p.stdout)\n408- except Exception: raise BackupError('PVE_INVENTORY_JSON')\n409-\n410:def ssh_remote(node,argv,input=None,timeout=1800,check=True):\n411- cmd='PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec '+shlex.join([str(x) for x in argv])\n412- if node=='pve01': return run([str(x) for x in argv],input=input,timeout=timeout,check=check)\n413- return run([SSH,'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node,cmd],input=input,timeout=timeout,check=check)\n414-\n415-def _pve_dumpdir_spec(guest_type):\n--\n471- if i+1>=len(argv) or argv[i+1]!=str(scratch): raise BackupError('PROVEN_DUMP_ARGV_TMPDIR_BINDING')\n472- elif '--tmpdir='+str(scratch) not in argv: raise BackupError('PROVEN_DUMP_ARGV_NO_TMPDIR')\n473- return argv,job\n474-\n475-def _default_unprivileged_lxc(node,vmid):\n476: q=ssh_remote(node,['pct','config',str(vmid)],timeout=30)\n477- lines=[x.strip() for x in q.stdout.decode('utf-8','replace').splitlines()]\n478- if 'unprivileged: 1' not in lines: raise BackupError('LXC_NOT_UNPRIVILEGED')\n479- if any(x.startswith('lxc.idmap:') or x.startswith('lxc.idmap =') for x in lines): raise BackupError('LXC_CUSTOM_IDMAP_UNSUPPORTED')\n480-\n481-def mapped_scratch_probe(node,scratch):\n482- tool=shutil.which('lxc-usernsexec') or '/usr/bin/lxc-usernsexec'\n483- pos=[tool,'-m','u:0:100000:65536','-m','g:0:100000:65536','--','/usr/bin/test','-x',str(scratch)]\n484: q=ssh_remote(node,pos,timeout=30,check=False)\n485- if q.returncode!=0: raise BackupError('MAPPED_UID_ACCESS_PROBE_FAILED')\n486-\n487-def _stream_argv_to_local(node,argv,dest,timeout=7200):\n488- dest=pathlib.Path(dest)\n489- if node=='pve01': cmd=argv\n--\n509- if typ!=t['guest_type']: raise BackupError('PVE_GUEST_TYPE_DRIFT')\n510- if typ=='lxc':\n511- _require_exact_source(P270,P270_SHA,'P270_IDENTITY');_require_exact_source(P282,P282_SHA,'P282_IDENTITY')\n512- _default_unprivileged_lxc(node,vmid)\n513- scratch=f'/var/tmp/homelab-vzdump-v2-{gen}'\n514: ssh_remote(node,['install','-d','-m','710','-o','0','-g','100000',scratch],timeout=30)\n515: q=ssh_remote(node,['stat','-c','%a:%u:%g',scratch],timeout=30)\n516- if q.stdout.decode().strip()!='710:0:100000': raise BackupError('PVE_MAPPED_SCRATCH_ACCESS_CONTRACT')\n517- try:\n518- mapped_scratch_probe(node,scratch)\n519- argv,job=proven_lxc_dump_argv(t,row,node,scratch)\n520- local=work/'payload.bin'\n521- _stream_argv_to_local(node,argv,local,timeout=7200)\n522- if local.stat().st_size>int(job['archive_cap_bytes']): raise BackupError('VZDUMP_ARCHIVE_CAP_EXCEEDED')\n523- except Exception:\n524: ssh_remote(node,['rm','-rf','--',scratch],timeout=60,check=False);raise\n525- cleanup={'kind':'pve_source_dir','node':node,'path':scratch}\n526- return local,{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status'),'producer_contract':'proven-270-282-stream'},cleanup\n527- # QEMU path remains unchanged: bounded dedicated dumpdir with later cloud readback/reconstruction.\n528- srcdir=f'/var/tmp/homelab-vzdump-v2-{gen}'\n529: ssh_remote(node,['install','-d','-m','700','-o','0','-g','0',srcdir],timeout=30)\n530- try:\n531: ssh_remote(node,['vzdump',str(vmid),'--mode','snapshot','--compress','zstd','--dumpdir',srcdir],timeout=7200)\n532: p=ssh_remote(node,['find',srcdir,'-maxdepth','1','-type','f','-printf','%p\\n'],timeout=30)\n533- files=[x for x in p.stdout.decode().splitlines() if x.strip()]\n534- if len(files)!=1: raise BackupError('VZDUMP_OUTPUT_NOT_UNIQUE')\n535- src=files[0];local=work/'payload.bin'\n536- if node=='pve01': local=pathlib.Path(src)\n537- else: run([SCP,'-q','-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node+':'+src,str(local)],timeout=7200)\n538- if not pathlib.Path(local).is_file() or pathlib.Path(local).stat().st_size<=0: raise BackupError('VZDUMP_PAYLOAD_MISSING')\n539- except Exception:\n540: ssh_remote(node,['rm','-rf','--',srcdir],timeout=60,check=False);raise\n541- return pathlib.Path(local),{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status')},{'kind':'pve_source_dir','node':node,'path':srcdir}\n542-\n543-def operator_key_fingerprint():\n544- if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING')\n545- p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30)\n--\n563- return good[0]\n564-\n565-def ensure_vps_known_hosts(targets):\n566- lines=[]\n567- for t in targets:\n568: if t['type'] not in ('vps_md','vps_spb','vps_us'): continue\n569- lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint']))\n570- atomic_write(VPS_KNOWN_HOSTS,('\\n'.join(lines)+'\\n').encode(),0o600)\n571-\n572-def vps_ssh_argv(t, *, use_agent=False):\n573- a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15']\n--\n598- tag=p.stdout.decode().strip()\n599- if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT')\n600- vps_run(t,['true'],timeout=30,use_agent=False)\n601- results[t['logical_id']]=tag\n602- # US was already operator-key based; prove it agentless too.\n603: us=[x for x in targets if x['type']=='vps_us'][0]\n604- vps_run(us,['true'],timeout=30,use_agent=False)\n605- results[us['logical_id']]='ALREADY_US_OPERATOR_KEY'\n606- return results\n607-\n608-def remote_hashset(t,command):\n609- p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False)\n610- return sha256_bytes(p.stdout)\n611-\n612:def capture_vps_md(t,gen,work):\n613- files=['etc/netbird/install.conf','var/lib/netbird/state.json','var/lib/netbird/service.json','var/lib/netbird/active_profile.json','var/lib/netbird/default.json']\n614- cmd=\"sudo -n sha256sum -- \"+' '.join(shlex.quote('/'+x) for x in files)\n615- pre=remote_hashset(t,cmd)\n616- remote=shlex.join(['sudo','-n','tar','--numeric-owner','-C','/','-cf','-']+files)\n617- argv=vps_ssh_argv(t,use_agent=False)+[remote]\n618- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=600)\n619- post=remote_hashset(t,cmd)\n620- if pre!=post: raise BackupError('MD_SOURCE_CHANGED_DURING_CAPTURE')\n621- return payload,{'generation':gen,'source_class':'vps_md','consistency':'FILESET_PRE_POST_HASH_STABILITY','hashset_sha256':pre},None\n622-\n623:def capture_vps_spb(t,gen,work):\n624- hcmd=\"find /etc/nginx -xdev -type f -print0 | sort -z | xargs -0 sha256sum\"\n625- pre=remote_hashset(t,hcmd)\n626- remote=shlex.join(['tar','--numeric-owner','-C','/','-cf','-','etc/nginx'])\n627- argv=vps_ssh_argv(t,use_agent=False)+[remote]\n628- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=900)\n--\n657- if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33)\n658- b=dst.serialize(); dst.close(); src.close()\n659- ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b))\n660-'''\n661-\n662:def capture_vps_us(t,gen,work):\n663- remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT])\n664- argv=vps_ssh_argv(t,use_agent=False)+[remote]\n665- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200)\n666: return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None\n667-\n668-def _simple_shell_assignments(text):\n669- vals={}\n670- for line in text.splitlines():\n671- m=re.match(r'^\\s*(?:export\\s+)?([A-Z_][A-Z0-9_]*)=(.*)$',line)\n--\n801-\n802-def qga_exec(node,vmid,guest_argv,input_bytes=None,timeout=600):\n803- args=['qm','guest','exec',str(vmid),'--synchronous','1','--timeout',str(int(timeout))]\n804- if input_bytes is not None: args += ['--pass-stdin','1']\n805- args += ['--']+[str(x) for x in guest_argv]\n806: p=ssh_remote(node,args,input=input_bytes,timeout=timeout+60)\n807- try:j=json.loads(p.stdout)\n808- except Exception: raise BackupError('QGA_JSON')\n809- rc=j.get('exitcode',j.get('exit-code'))\n810- if rc is None: rc=0 if j.get('exited') else None\n811- if rc not in (0,None): raise BackupError('QGA_GUEST_COMMAND_FAILED',str(rc))\n--\n854-\n855-def capture_node_bundle(t,gen,work):\n856- node=t['node']; raw=node_bundle_paths(node); paths=[]\n857- for rel in raw:\n858- if node=='pve01': ok=(pathlib.Path('/')/rel).exists()\n859: else: ok=(ssh_remote(node,['test','-e','/'+rel],timeout=20,check=False).returncode==0)\n860- if ok: paths.append(rel)\n861- required={'etc/network/interfaces','etc/hosts','etc/hostname','etc/pve/nodes/'+node}\n862- if not required.issubset(set(paths)): raise BackupError('NODE_BUNDLE_REQUIRED_PATH_MISSING')\n863- payload=work/'payload.tar.zst'\n864- argv=['tar','--numeric-owner','--xattrs','--acls','-C','/','-cf','-']+paths\n--\n870-\n871-def capture_target(t,gen,work):\n872- typ=t['type']\n873- if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n874- if typ=='pve_node_bundle': return capture_node_bundle(t,gen,work)\n875: if typ=='vps_md': return capture_vps_md(t,gen,work)\n876: if typ=='vps_spb': return capture_vps_spb(t,gen,work)\n877: if typ=='vps_us': return capture_vps_us(t,gen,work)\n878- if typ=='router': return capture_router(t,gen,work)\n879- if typ=='xenforo': return capture_xenforo(t,gen,work)\n880- raise BackupError('CAPTURE_TYPE_UNKNOWN')\n881-\n882-def cleanup_source(cleanup):\n883- if not cleanup: return\n884- if cleanup.get('kind')=='pve_source_dir':\n885: ssh_remote(cleanup['node'],['rm','-rf','--',cleanup['path']],timeout=60)\n886-\n887-def control_verify(t):\n888- engine=pathlib.Path('/usr/local/sbin/homelab-control-backup')\n889- if sha256_file(engine)!='0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57': raise BackupError('CONTROL_ENGINE_IDENTITY')\n890- root=pathlib.Path('/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2')\n--\n895- if j.get('status')=='SEALED' and j.get('generation'): rows.append((p.stat().st_mtime,j['generation']))\n896- except Exception: continue\n897- if not rows: raise BackupError('CONTROL_SEALED_GENERATION_MISSING')\n898- gen=sorted(rows)[-1][1]\n899- p=run([str(engine),'verify',gen],timeout=1800)\n900: write_state(t['logical_id'],'SUCCESS',generation=gen,verification='existing_control_engine_verify',last_success_epoch=epoch())\n901- return {'logical_id':t['logical_id'],'generation':gen,'result':'SUCCESS','payload_bytes':0,'replicas':2}\n902-\n903:def backup_target(t,green_verify=False):\n904- if t['type']=='control': return control_verify(t)\n905- gen=generation_id(t['logical_id']); work=ensure_dir(WORK_ROOT/gen,0o700)\n906- payload=None; cleanup=None; sealed=False\n907- try:\n908- payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n--\n916- ha=reconstruct_replica(manifest,manifest['replicas'][0],a); validate_payload(t,a)\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py SHA256=2b5eb4fc0f6f466a0d3e5a007af987092b6c17baa3ff9241a6afc2c2e97df1d5\n2-from __future__ import annotations\n3-import errno,fcntl,importlib.util,os,sys\n4-from pathlib import Path\n5-ROOT=Path(__file__).resolve().parent\n6-CT_ID='guest-pve01-lxc-112'\n7:LEGACY_IDS=('node-pve01','md-relay','spb-ingress','us-netbird','router-netcraze','xf-codevipe','xf-zakrutim','control-bootstrap-v2')\n8-LOCK=Path('/run/homelab-backup-heavy.lock')\n9-\n10-def _load(name,path):\n11- spec=importlib.util.spec_from_file_location(name,str(path))\n12- if spec is None or spec.loader is None: raise RuntimeError('MODULE_SPEC')\n--\n14-\n15-legacy=_load('_backupv2_legacy486',ROOT/'legacy_engine.py')\n16-ct112=_load('_backupv2_ct112_486',ROOT/'ct112_handler.py')\n17-driver=_load('_backupv2_driver454_479',ROOT/'representative_driver.py')\n18-_legacy_load_targets=legacy.load_targets\n19:_legacy_backup_target=legacy.backup_target\n20-\n21-def canonical_targets():\n22- rows=_legacy_load_targets()\n23- if not isinstance(rows,list) or len(rows)!=28: raise legacy.BackupError('TARGET_REGISTRY_COUNT')\n24- out=[dict(x) for x in rows]\n--\n35- 'guest_type':'lxc','due_seconds':due,'enabled':True,'green1':True}\n36-\n37-def load_targets():\n38- base=canonical_targets();return base+[_ct_descriptor(base)]\n39-\n40:def backup_target(t,green_verify=False):\n41- if not isinstance(t,dict) or t.get('logical_id')!=CT_ID:\n42: return _legacy_backup_target(t,green_verify)\n43- try:\n44: r=ct112.run_target(t,state_writer=legacy.write_state)\n45- return {'logical_id':CT_ID,'generation':r['generation'],'result':'SUCCESS','payload_bytes':r['payload_bytes'],\n46- 'replicas':2,'manifest_sha256':r['manifest_sha256'],'payload_sha256':r['payload_sha256']}\n47- except BaseException:\n48: try: legacy.write_state(CT_ID,'FAIL',reason='CT112_PIPELINE_FAILED')\n49- except BaseException: pass\n50- raise legacy.BackupError('CT112_PIPELINE_FAILED') from None\n51-\n52-legacy.load_targets=load_targets\n53:legacy.backup_target=backup_target\n54-\n55-def _legacy_reused_boundary_selftest():\n56- if legacy.PART_SIZE!=536870912: raise RuntimeError('LEGACY_PART_SIZE')\n57- if len(legacy.REMOTES)!=13 or len({p for _,p in legacy.REMOTES})!=13: raise RuntimeError('LEGACY_REMOTES')\n58- d=legacy.default_targets()\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py SHA256=ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793\n57- if gp==legacy_path:\n58- found[id(g)]=g\n59- if len(found)!=1:\n60- raise RuntimeError('LEGACY_PROVENANCE_CARDINALITY')\n61- g=next(iter(found.values()))\n62: required=('capture_pve_guest','backup_target','load_targets','cluster_inventory','ssh_remote',\n63- '_default_unprivileged_lxc','mapped_scratch_probe','_stream_argv_to_local','run',\n64: 'choose_due_target','write_state','epoch')\n65- if any(not callable(g.get(n)) for n in required):\n66- raise RuntimeError('LEGACY_PROVENANCE_API')\n67- original=g['capture_pve_guest']\n68- proxy=_GlobalsProxy(g)\n69- if not isinstance(g.get('SCP'),str) or not g['SCP'].startswith('/'):\n--\n73- raise RuntimeError('QEMU_ADAPTER_BIND_RESULT')\n74- patched=lxc_adapter.bind(proxy,qemu_patched)\n75- if not callable(patched):\n76- raise RuntimeError('LXC_ADAPTER_BIND_RESULT')\n77- g['capture_pve_guest']=patched\n78: scheduler_wired=scheduler_adapter.bind(proxy,g['choose_due_target'],g['write_state'])\n79: if g.get('choose_due_target') is not scheduler_wired['choose_due_target'] or g.get('write_state') is not scheduler_wired['write_state']:\n80- raise RuntimeError('SCHEDULER_ADAPTER_BIND_RESULT')\n81- return {'legacy_globals':g,'legacy':proxy,'original_capture':original,'qemu_patched_capture':qemu_patched,'patched_capture':patched,'scheduler_wired':scheduler_wired}\n82-\n83-def _load():\n84- _verify_base()\n--\n105- wrapper,lxc_adapter,qemu_adapter,scheduler_adapter,wired=_load()\n106- if wired['legacy_globals'].get('capture_pve_guest') is not wired['patched_capture']:\n107- raise RuntimeError('PATCH_BINDING')\n108- if wired['legacy_globals'].get('choose_due_target') is not wired['scheduler_wired']['choose_due_target']:\n109- raise RuntimeError('SCHEDULER_CHOOSE_BINDING')\n110: if wired['legacy_globals'].get('write_state') is not wired['scheduler_wired']['write_state']:\n111- raise RuntimeError('SCHEDULER_WRITE_BINDING')\n112- if [scheduler_adapter.backoff_seconds(n) for n in range(1,7)]!=[1800,3600,7200,14400,21600,21600]:\n113- raise RuntimeError('SCHEDULER_BACKOFF_CONTRACT')\n114: if not callable(wrapper.backup_target) or not callable(wrapper.main) or not callable(wrapper.load_targets):\n115- raise RuntimeError('WRAPPER_PUBLIC_API')\n116- print('SELFTEST_OVERLAY504=PASS',flush=True)\n117- return 0\n118-\n119-def main(argv=None):\n--\n131- try:\n132- with tempfile.TemporaryDirectory(prefix='homelab-scheduler-proof-',dir='/run') as td:\n133- g['STATE_ROOT']=pathlib.Path(td); (g['STATE_ROOT']/'state').mkdir(mode=0o700)\n134- g['epoch']=lambda: now[0]\n135- common={'payload_sha256':'a'*64,'manifest_sha256':'b'*64,'replicas':['proof-r1','proof-r2']}\n136: legacy.write_state('proof-a','SUCCESS',generation='proof-a-good',**common)\n137: legacy.write_state('proof-b','SUCCESS',generation='proof-b-good',**common)\n138- now[0]=100000\n139: legacy.write_state('proof-a','FAIL',generation='proof-a-bad',reason='PROOF_FAILURE',sealed=False)\n140- state=legacy.load_state_all(); a=state['proof-a']\n141- targets=[{'logical_id':'proof-a','enabled':True,'due_seconds':100},{'logical_id':'proof-b','enabled':True,'due_seconds':100}]\n142- first=legacy.choose_due_target(targets,state,now[0])\n143- if not isinstance(first,dict) or first.get('logical_id')!='proof-b': raise RuntimeError('SCHEDULER_PROOF_PEER_NOT_SELECTED')\n144: retry=int(a.get('retry_after_epoch',0)); preserved=int(a.get('last_success_epoch',0))\n145: if preserved!=1000 or a.get('last_success_generation')!='proof-a-good' or int(a.get('consecutive_failures',0))!=1 or retry!=101800:\n146- raise RuntimeError('SCHEDULER_PROOF_FAILURE_STATE')\n147- now[0]=retry+1\n148- second=legacy.choose_due_target(targets,legacy.load_state_all(),now[0])\n149- if not isinstance(second,dict) or second.get('logical_id')!='proof-a': raise RuntimeError('SCHEDULER_PROOF_RETRY_NOT_REELIGIBLE')\n150: legacy.write_state('proof-a','SUCCESS',generation='proof-a-recovered',**common)\n151- recovered=legacy.load_state_all()['proof-a']\n152: if int(recovered.get('consecutive_failures',-1))!=0 or int(recovered.get('retry_after_epoch',-1))!=0:\n153- raise RuntimeError('SCHEDULER_PROOF_SUCCESS_RESET')\n154: proof={'failed_target':'proof-a','peer_selected_during_backoff':first['logical_id'],'retry_after_epoch':retry,'last_success_preserved':preserved==1000,'retry_target_after_backoff':second['logical_id'],'success_resets_failures':True,'production_state_root_used':False}\n155- print('SCHEDULER_FAILURE_ISOLATION_PROOF='+json.dumps(proof,separators=(',',':'),sort_keys=True),flush=True)\n156- return 0\n157- finally:\n158- g['STATE_ROOT']=old_root; g['epoch']=old_epoch\n159- if argv==['scheduler-select-live']:\n--\n174- if len(argv)==2 and argv[0]=='verify-target':\n175- t=_target(wrapper,argv[1])\n176- if t.get('logical_id')=='guest-pve01-lxc-112':\n177- raise RuntimeError('VERIFY_SYNTHETIC_CT112_FORBIDDEN')\n178- try:\n179: out=wrapper.backup_target(t,True)\n180- except legacy.BackupError as e:\n181- print('VERIFY_TARGET_RESULT=FAIL',flush=True)\n182- print('VERIFY_TARGET_REASON='+str(e.code),flush=True)\n183- return 2\n184- print('VERIFY_TARGET_RESULT='+json.dumps(out,separators=(',',':'),sort_keys=True),flush=True)\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/scheduler_adapter.py SHA256=a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96\n14- n=int(v)\n15- except Exception:\n16- return default\n17- return n if n>=0 else default\n18-\n19:def backoff_seconds(consecutive_failures:int)->int:\n20: n=max(1,_as_nonneg_int(consecutive_failures,1))\n21- return min(MAX_BACKOFF,MIN_BACKOFF*(2**min(n-1,4)))\n22-\n23-def _read_prior(legacy,logical_id):\n24- root=pathlib.Path(legacy.STATE_ROOT)/'state'\n25- p=root/(str(logical_id)+'.json')\n--\n60- if not isinstance(t,dict) or t.get('enabled') is not True: continue\n61- lid=t.get('logical_id'); seconds=_as_nonneg_int(t.get('due_seconds'),0)\n62- _need(isinstance(lid,str) and lid and seconds>0,'SCHEDULER_TARGET_SCHEMA')\n63- s=state.get(lid,{})\n64- if not isinstance(s,dict): s={}\n65: retry_after=_as_nonneg_int(s.get('retry_after_epoch'),0)\n66: if retry_after>now: continue\n67- last_success=_as_nonneg_int(s.get('last_success_epoch'),0)\n68- age=(now-last_success) if last_success and now>=last_success else (10**12 if last_success==0 else 0)\n69- if age=len(argv) or argv[i+1]!=str(scratch): raise BackupError('PROVEN_DUMP_ARGV_TMPDIR_BINDING') +472- elif '--tmpdir='+str(scratch) not in argv: raise BackupError('PROVEN_DUMP_ARGV_NO_TMPDIR') +473- return argv,job +474- +475-def _default_unprivileged_lxc(node,vmid): +476: q=ssh_remote(node,['pct','config',str(vmid)],timeout=30) +477- lines=[x.strip() for x in q.stdout.decode('utf-8','replace').splitlines()] +478- if 'unprivileged: 1' not in lines: raise BackupError('LXC_NOT_UNPRIVILEGED') +479- if any(x.startswith('lxc.idmap:') or x.startswith('lxc.idmap =') for x in lines): raise BackupError('LXC_CUSTOM_IDMAP_UNSUPPORTED') +480- +481-def mapped_scratch_probe(node,scratch): +482- tool=shutil.which('lxc-usernsexec') or '/usr/bin/lxc-usernsexec' +483- pos=[tool,'-m','u:0:100000:65536','-m','g:0:100000:65536','--','/usr/bin/test','-x',str(scratch)] +484: q=ssh_remote(node,pos,timeout=30,check=False) +485- if q.returncode!=0: raise BackupError('MAPPED_UID_ACCESS_PROBE_FAILED') +486- +487-def _stream_argv_to_local(node,argv,dest,timeout=7200): +488- dest=pathlib.Path(dest) +489- if node=='pve01': cmd=argv +-- +509- if typ!=t['guest_type']: raise BackupError('PVE_GUEST_TYPE_DRIFT') +510- if typ=='lxc': +511- _require_exact_source(P270,P270_SHA,'P270_IDENTITY');_require_exact_source(P282,P282_SHA,'P282_IDENTITY') +512- _default_unprivileged_lxc(node,vmid) +513- scratch=f'/var/tmp/homelab-vzdump-v2-{gen}' +514: ssh_remote(node,['install','-d','-m','710','-o','0','-g','100000',scratch],timeout=30) +515: q=ssh_remote(node,['stat','-c','%a:%u:%g',scratch],timeout=30) +516- if q.stdout.decode().strip()!='710:0:100000': raise BackupError('PVE_MAPPED_SCRATCH_ACCESS_CONTRACT') +517- try: +518- mapped_scratch_probe(node,scratch) +519- argv,job=proven_lxc_dump_argv(t,row,node,scratch) +520- local=work/'payload.bin' +521- _stream_argv_to_local(node,argv,local,timeout=7200) +522- if local.stat().st_size>int(job['archive_cap_bytes']): raise BackupError('VZDUMP_ARCHIVE_CAP_EXCEEDED') +523- except Exception: +524: ssh_remote(node,['rm','-rf','--',scratch],timeout=60,check=False);raise +525- cleanup={'kind':'pve_source_dir','node':node,'path':scratch} +526- return local,{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status'),'producer_contract':'proven-270-282-stream'},cleanup +527- # QEMU path remains unchanged: bounded dedicated dumpdir with later cloud readback/reconstruction. +528- srcdir=f'/var/tmp/homelab-vzdump-v2-{gen}' +529: ssh_remote(node,['install','-d','-m','700','-o','0','-g','0',srcdir],timeout=30) +530- try: +531: ssh_remote(node,['vzdump',str(vmid),'--mode','snapshot','--compress','zstd','--dumpdir',srcdir],timeout=7200) +532: p=ssh_remote(node,['find',srcdir,'-maxdepth','1','-type','f','-printf','%p\n'],timeout=30) +533- files=[x for x in p.stdout.decode().splitlines() if x.strip()] +534- if len(files)!=1: raise BackupError('VZDUMP_OUTPUT_NOT_UNIQUE') +535- src=files[0];local=work/'payload.bin' +536- if node=='pve01': local=pathlib.Path(src) +537- else: run([SCP,'-q','-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node+':'+src,str(local)],timeout=7200) +538- if not pathlib.Path(local).is_file() or pathlib.Path(local).stat().st_size<=0: raise BackupError('VZDUMP_PAYLOAD_MISSING') +539- except Exception: +540: ssh_remote(node,['rm','-rf','--',srcdir],timeout=60,check=False);raise +541- return pathlib.Path(local),{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status')},{'kind':'pve_source_dir','node':node,'path':srcdir} +542- +543-def operator_key_fingerprint(): +544- if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING') +545- p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30) +-- +563- return good[0] +564- +565-def ensure_vps_known_hosts(targets): +566- lines=[] +567- for t in targets: +568: if t['type'] not in ('vps_md','vps_spb','vps_us'): continue +569- lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint'])) +570- atomic_write(VPS_KNOWN_HOSTS,('\n'.join(lines)+'\n').encode(),0o600) +571- +572-def vps_ssh_argv(t, *, use_agent=False): +573- a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15'] +-- +598- tag=p.stdout.decode().strip() +599- if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT') +600- vps_run(t,['true'],timeout=30,use_agent=False) +601- results[t['logical_id']]=tag +602- # US was already operator-key based; prove it agentless too. +603: us=[x for x in targets if x['type']=='vps_us'][0] +604- vps_run(us,['true'],timeout=30,use_agent=False) +605- results[us['logical_id']]='ALREADY_US_OPERATOR_KEY' +606- return results +607- +608-def remote_hashset(t,command): +609- p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False) +610- return sha256_bytes(p.stdout) +611- +612:def capture_vps_md(t,gen,work): +613- files=['etc/netbird/install.conf','var/lib/netbird/state.json','var/lib/netbird/service.json','var/lib/netbird/active_profile.json','var/lib/netbird/default.json'] +614- cmd="sudo -n sha256sum -- "+' '.join(shlex.quote('/'+x) for x in files) +615- pre=remote_hashset(t,cmd) +616- remote=shlex.join(['sudo','-n','tar','--numeric-owner','-C','/','-cf','-']+files) +617- argv=vps_ssh_argv(t,use_agent=False)+[remote] +618- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=600) +619- post=remote_hashset(t,cmd) +620- if pre!=post: raise BackupError('MD_SOURCE_CHANGED_DURING_CAPTURE') +621- return payload,{'generation':gen,'source_class':'vps_md','consistency':'FILESET_PRE_POST_HASH_STABILITY','hashset_sha256':pre},None +622- +623:def capture_vps_spb(t,gen,work): +624- hcmd="find /etc/nginx -xdev -type f -print0 | sort -z | xargs -0 sha256sum" +625- pre=remote_hashset(t,hcmd) +626- remote=shlex.join(['tar','--numeric-owner','-C','/','-cf','-','etc/nginx']) +627- argv=vps_ssh_argv(t,use_agent=False)+[remote] +628- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=900) +-- +657- if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33) +658- b=dst.serialize(); dst.close(); src.close() +659- ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b)) +660-''' +661- +662:def capture_vps_us(t,gen,work): +663- remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT]) +664- argv=vps_ssh_argv(t,use_agent=False)+[remote] +665- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200) +666: return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None +667- +668-def _simple_shell_assignments(text): +669- vals={} +670- for line in text.splitlines(): +671- m=re.match(r'^\s*(?:export\s+)?([A-Z_][A-Z0-9_]*)=(.*)$',line) +-- +801- +802-def qga_exec(node,vmid,guest_argv,input_bytes=None,timeout=600): +803- args=['qm','guest','exec',str(vmid),'--synchronous','1','--timeout',str(int(timeout))] +804- if input_bytes is not None: args += ['--pass-stdin','1'] +805- args += ['--']+[str(x) for x in guest_argv] +806: p=ssh_remote(node,args,input=input_bytes,timeout=timeout+60) +807- try:j=json.loads(p.stdout) +808- except Exception: raise BackupError('QGA_JSON') +809- rc=j.get('exitcode',j.get('exit-code')) +810- if rc is None: rc=0 if j.get('exited') else None +811- if rc not in (0,None): raise BackupError('QGA_GUEST_COMMAND_FAILED',str(rc)) +-- +854- +855-def capture_node_bundle(t,gen,work): +856- node=t['node']; raw=node_bundle_paths(node); paths=[] +857- for rel in raw: +858- if node=='pve01': ok=(pathlib.Path('/')/rel).exists() +859: else: ok=(ssh_remote(node,['test','-e','/'+rel],timeout=20,check=False).returncode==0) +860- if ok: paths.append(rel) +861- required={'etc/network/interfaces','etc/hosts','etc/hostname','etc/pve/nodes/'+node} +862- if not required.issubset(set(paths)): raise BackupError('NODE_BUNDLE_REQUIRED_PATH_MISSING') +863- payload=work/'payload.tar.zst' +864- argv=['tar','--numeric-owner','--xattrs','--acls','-C','/','-cf','-']+paths +-- +870- +871-def capture_target(t,gen,work): +872- typ=t['type'] +873- if typ=='pve_guest': return capture_pve_guest(t,gen,work) +874- if typ=='pve_node_bundle': return capture_node_bundle(t,gen,work) +875: if typ=='vps_md': return capture_vps_md(t,gen,work) +876: if typ=='vps_spb': return capture_vps_spb(t,gen,work) +877: if typ=='vps_us': return capture_vps_us(t,gen,work) +878- if typ=='router': return capture_router(t,gen,work) +879- if typ=='xenforo': return capture_xenforo(t,gen,work) +880- raise BackupError('CAPTURE_TYPE_UNKNOWN') +881- +882-def cleanup_source(cleanup): +883- if not cleanup: return +884- if cleanup.get('kind')=='pve_source_dir': +885: ssh_remote(cleanup['node'],['rm','-rf','--',cleanup['path']],timeout=60) +886- +887-def control_verify(t): +888- engine=pathlib.Path('/usr/local/sbin/homelab-control-backup') +889- if sha256_file(engine)!='0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57': raise BackupError('CONTROL_ENGINE_IDENTITY') +890- root=pathlib.Path('/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2') +-- +895- if j.get('status')=='SEALED' and j.get('generation'): rows.append((p.stat().st_mtime,j['generation'])) +896- except Exception: continue +897- if not rows: raise BackupError('CONTROL_SEALED_GENERATION_MISSING') +898- gen=sorted(rows)[-1][1] +899- p=run([str(engine),'verify',gen],timeout=1800) +900: write_state(t['logical_id'],'SUCCESS',generation=gen,verification='existing_control_engine_verify',last_success_epoch=epoch()) +901- return {'logical_id':t['logical_id'],'generation':gen,'result':'SUCCESS','payload_bytes':0,'replicas':2} +902- +903:def backup_target(t,green_verify=False): +904- if t['type']=='control': return control_verify(t) +905- gen=generation_id(t['logical_id']); work=ensure_dir(WORK_ROOT/gen,0o700) +906- payload=None; cleanup=None; sealed=False +907- try: +908- payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +-- +916- ha=reconstruct_replica(manifest,manifest['replicas'][0],a); validate_payload(t,a) +SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py SHA256=2b5eb4fc0f6f466a0d3e5a007af987092b6c17baa3ff9241a6afc2c2e97df1d5 +2-from __future__ import annotations +3-import errno,fcntl,importlib.util,os,sys +4-from pathlib import Path +5-ROOT=Path(__file__).resolve().parent +6-CT_ID='guest-pve01-lxc-112' +7:LEGACY_IDS=('node-pve01','md-relay','spb-ingress','us-netbird','router-netcraze','xf-codevipe','xf-zakrutim','control-bootstrap-v2') +8-LOCK=Path('/run/homelab-backup-heavy.lock') +9- +10-def _load(name,path): +11- spec=importlib.util.spec_from_file_location(name,str(path)) +12- if spec is None or spec.loader is None: raise RuntimeError('MODULE_SPEC') +-- +14- +15-legacy=_load('_backupv2_legacy486',ROOT/'legacy_engine.py') +16-ct112=_load('_backupv2_ct112_486',ROOT/'ct112_handler.py') +17-driver=_load('_backupv2_driver454_479',ROOT/'representative_driver.py') +18-_legacy_load_targets=legacy.load_targets +19:_legacy_backup_target=legacy.backup_target +20- +21-def canonical_targets(): +22- rows=_legacy_load_targets() +23- if not isinstance(rows,list) or len(rows)!=28: raise legacy.BackupError('TARGET_REGISTRY_COUNT') +24- out=[dict(x) for x in rows] +-- +35- 'guest_type':'lxc','due_seconds':due,'enabled':True,'green1':True} +36- +37-def load_targets(): +38- base=canonical_targets();return base+[_ct_descriptor(base)] +39- +40:def backup_target(t,green_verify=False): +41- if not isinstance(t,dict) or t.get('logical_id')!=CT_ID: +42: return _legacy_backup_target(t,green_verify) +43- try: +44: r=ct112.run_target(t,state_writer=legacy.write_state) +45- return {'logical_id':CT_ID,'generation':r['generation'],'result':'SUCCESS','payload_bytes':r['payload_bytes'], +46- 'replicas':2,'manifest_sha256':r['manifest_sha256'],'payload_sha256':r['payload_sha256']} +47- except BaseException: +48: try: legacy.write_state(CT_ID,'FAIL',reason='CT112_PIPELINE_FAILED') +49- except BaseException: pass +50- raise legacy.BackupError('CT112_PIPELINE_FAILED') from None +51- +52-legacy.load_targets=load_targets +53:legacy.backup_target=backup_target +54- +55-def _legacy_reused_boundary_selftest(): +56- if legacy.PART_SIZE!=536870912: raise RuntimeError('LEGACY_PART_SIZE') +57- if len(legacy.REMOTES)!=13 or len({p for _,p in legacy.REMOTES})!=13: raise RuntimeError('LEGACY_REMOTES') +58- d=legacy.default_targets() +SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py SHA256=ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793 +57- if gp==legacy_path: +58- found[id(g)]=g +59- if len(found)!=1: +60- raise RuntimeError('LEGACY_PROVENANCE_CARDINALITY') +61- g=next(iter(found.values())) +62: required=('capture_pve_guest','backup_target','load_targets','cluster_inventory','ssh_remote', +63- '_default_unprivileged_lxc','mapped_scratch_probe','_stream_argv_to_local','run', +64: 'choose_due_target','write_state','epoch') +65- if any(not callable(g.get(n)) for n in required): +66- raise RuntimeError('LEGACY_PROVENANCE_API') +67- original=g['capture_pve_guest'] +68- proxy=_GlobalsProxy(g) +69- if not isinstance(g.get('SCP'),str) or not g['SCP'].startswith('/'): +-- +73- raise RuntimeError('QEMU_ADAPTER_BIND_RESULT') +74- patched=lxc_adapter.bind(proxy,qemu_patched) +75- if not callable(patched): +76- raise RuntimeError('LXC_ADAPTER_BIND_RESULT') +77- g['capture_pve_guest']=patched +78: scheduler_wired=scheduler_adapter.bind(proxy,g['choose_due_target'],g['write_state']) +79: if g.get('choose_due_target') is not scheduler_wired['choose_due_target'] or g.get('write_state') is not scheduler_wired['write_state']: +80- raise RuntimeError('SCHEDULER_ADAPTER_BIND_RESULT') +81- return {'legacy_globals':g,'legacy':proxy,'original_capture':original,'qemu_patched_capture':qemu_patched,'patched_capture':patched,'scheduler_wired':scheduler_wired} +82- +83-def _load(): +84- _verify_base() +-- +105- wrapper,lxc_adapter,qemu_adapter,scheduler_adapter,wired=_load() +106- if wired['legacy_globals'].get('capture_pve_guest') is not wired['patched_capture']: +107- raise RuntimeError('PATCH_BINDING') +108- if wired['legacy_globals'].get('choose_due_target') is not wired['scheduler_wired']['choose_due_target']: +109- raise RuntimeError('SCHEDULER_CHOOSE_BINDING') +110: if wired['legacy_globals'].get('write_state') is not wired['scheduler_wired']['write_state']: +111- raise RuntimeError('SCHEDULER_WRITE_BINDING') +112- if [scheduler_adapter.backoff_seconds(n) for n in range(1,7)]!=[1800,3600,7200,14400,21600,21600]: +113- raise RuntimeError('SCHEDULER_BACKOFF_CONTRACT') +114: if not callable(wrapper.backup_target) or not callable(wrapper.main) or not callable(wrapper.load_targets): +115- raise RuntimeError('WRAPPER_PUBLIC_API') +116- print('SELFTEST_OVERLAY504=PASS',flush=True) +117- return 0 +118- +119-def main(argv=None): +-- +131- try: +132- with tempfile.TemporaryDirectory(prefix='homelab-scheduler-proof-',dir='/run') as td: +133- g['STATE_ROOT']=pathlib.Path(td); (g['STATE_ROOT']/'state').mkdir(mode=0o700) +134- g['epoch']=lambda: now[0] +135- common={'payload_sha256':'a'*64,'manifest_sha256':'b'*64,'replicas':['proof-r1','proof-r2']} +136: legacy.write_state('proof-a','SUCCESS',generation='proof-a-good',**common) +137: legacy.write_state('proof-b','SUCCESS',generation='proof-b-good',**common) +138- now[0]=100000 +139: legacy.write_state('proof-a','FAIL',generation='proof-a-bad',reason='PROOF_FAILURE',sealed=False) +140- state=legacy.load_state_all(); a=state['proof-a'] +141- targets=[{'logical_id':'proof-a','enabled':True,'due_seconds':100},{'logical_id':'proof-b','enabled':True,'due_seconds':100}] +142- first=legacy.choose_due_target(targets,state,now[0]) +143- if not isinstance(first,dict) or first.get('logical_id')!='proof-b': raise RuntimeError('SCHEDULER_PROOF_PEER_NOT_SELECTED') +144: retry=int(a.get('retry_after_epoch',0)); preserved=int(a.get('last_success_epoch',0)) +145: if preserved!=1000 or a.get('last_success_generation')!='proof-a-good' or int(a.get('consecutive_failures',0))!=1 or retry!=101800: +146- raise RuntimeError('SCHEDULER_PROOF_FAILURE_STATE') +147- now[0]=retry+1 +148- second=legacy.choose_due_target(targets,legacy.load_state_all(),now[0]) +149- if not isinstance(second,dict) or second.get('logical_id')!='proof-a': raise RuntimeError('SCHEDULER_PROOF_RETRY_NOT_REELIGIBLE') +150: legacy.write_state('proof-a','SUCCESS',generation='proof-a-recovered',**common) +151- recovered=legacy.load_state_all()['proof-a'] +152: if int(recovered.get('consecutive_failures',-1))!=0 or int(recovered.get('retry_after_epoch',-1))!=0: +153- raise RuntimeError('SCHEDULER_PROOF_SUCCESS_RESET') +154: proof={'failed_target':'proof-a','peer_selected_during_backoff':first['logical_id'],'retry_after_epoch':retry,'last_success_preserved':preserved==1000,'retry_target_after_backoff':second['logical_id'],'success_resets_failures':True,'production_state_root_used':False} +155- print('SCHEDULER_FAILURE_ISOLATION_PROOF='+json.dumps(proof,separators=(',',':'),sort_keys=True),flush=True) +156- return 0 +157- finally: +158- g['STATE_ROOT']=old_root; g['epoch']=old_epoch +159- if argv==['scheduler-select-live']: +-- +174- if len(argv)==2 and argv[0]=='verify-target': +175- t=_target(wrapper,argv[1]) +176- if t.get('logical_id')=='guest-pve01-lxc-112': +177- raise RuntimeError('VERIFY_SYNTHETIC_CT112_FORBIDDEN') +178- try: +179: out=wrapper.backup_target(t,True) +180- except legacy.BackupError as e: +181- print('VERIFY_TARGET_RESULT=FAIL',flush=True) +182- print('VERIFY_TARGET_REASON='+str(e.code),flush=True) +183- return 2 +184- print('VERIFY_TARGET_RESULT='+json.dumps(out,separators=(',',':'),sort_keys=True),flush=True) +SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/scheduler_adapter.py SHA256=a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96 +14- n=int(v) +15- except Exception: +16- return default +17- return n if n>=0 else default +18- +19:def backoff_seconds(consecutive_failures:int)->int: +20: n=max(1,_as_nonneg_int(consecutive_failures,1)) +21- return min(MAX_BACKOFF,MIN_BACKOFF*(2**min(n-1,4))) +22- +23-def _read_prior(legacy,logical_id): +24- root=pathlib.Path(legacy.STATE_ROOT)/'state' +25- p=root/(str(logical_id)+'.json') +-- +60- if not isinstance(t,dict) or t.get('enabled') is not True: continue +61- lid=t.get('logical_id'); seconds=_as_nonneg_int(t.get('due_seconds'),0) +62- _need(isinstance(lid,str) and lid and seconds>0,'SCHEDULER_TARGET_SCHEMA') +63- s=state.get(lid,{}) +64- if not isinstance(s,dict): s={} +65: retry_after=_as_nonneg_int(s.get('retry_after_epoch'),0) +66: if retry_after>now: continue +67- last_success=_as_nonneg_int(s.get('last_success_epoch'),0) +68- age=(now-last_success) if last_success and now>=last_success else (10**12 if last_success==0 else 0) +69- if age\"$LOCK\"\nif ! flock -n 9; then echo \"STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$HEALTH\"; exit 0; fi\nlog(){ printf '%s RESTORE_CHECK %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/restore-check.log\"; }\nfail(){ rc=$?; printf 'STATUS=FAIL\\nTS=%s\\nRUN_ID=%s\\nerror_rc=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$rc\" > \"$HEALTH\"; log \"FAIL rc=$rc\"; exit \"$rc\"; }\ntrap fail ERR\ncommand -v rclone >/dev/null\nlog \"START run_id=$RUN_ID\"\ncat_chunked(){\n local remote_path=\"$1\" out=\"$2\" plist part\n : > \"$out\"\n plist=\"$(mktemp)\"\n if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n else\n rclone copyto \"$remote_path\" \"$out\" >/dev/null\n fi\n rm -f \"$plist\"\n [ -s \"$out\" ]\n}\ncheck_one_remote(){\n local label=\"$1\" remote=\"$2\" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status\n rbase=\"${remote}${REMOTE_BASE}/daily\"\n latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n [ -n \"$latest\" ] || { echo \"${label}_STATUS=NO_RUN\" >> \"$WORK/result.env\"; return 20; }\n archive_name=\"skladchik-reports-monitor-${latest}.tar.gz\"\n remote_archive=\"${rbase}/${latest}/${archive_name}\"\n outdir=\"${WORK}/${label}-${latest}\"\n mkdir -p \"$outdir\"\n archive=\"${outdir}/${archive_name}\"\n log \"${label} DOWNLOAD run=${latest}\"\n cat_chunked \"$remote_archive\" \"$archive\"\n cat_chunked \"${remote_archive}.sha256\" \"${archive}.sha256.remote\"\n expected=\"$(awk '{print $1; exit}' \"${archive}.sha256.remote\")\"\n actual=\"$(sha256sum \"$archive\" | awk '{print $1}')\"\n [ -n \"$expected\" ] && [ \"$expected\" = \"$actual\" ]\n list=\"${outdir}/tar-list.txt\"\n tar -tzf \"$archive\" > \"$list\"\n grep -Eq '(^|/)etc-skladchik-reports-monitor/config\\.ini$' \"$list\"\n grep -Eq '(^|/)etc-skladchik-reports-monitor/secrets\\.env$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/monitor\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/health_alert\\.py$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/state\\.sqlite$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/alert-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/cookies\\.jar$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\\.timer$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/operational_digest\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-runner\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-core$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/weekly-summary-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/age-alert-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/event-selftest-health\\.txt$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\\.timer$' \"$list\"\n extract=\"${outdir}/extract\"\n mkdir -p \"$extract\"\n tar -xzf \"$archive\" -C \"$extract\"\n sqlite_status=skipped\n if command -v sqlite3 >/dev/null && [ -s \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" ]; then\n [ \"$(sqlite3 \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" 'PRAGMA integrity_check;' | tail -n 1)\" = \"ok\" ]\n sqlite_status=ok\n fi\n py_status=skipped\n if command -v python3 >/dev/null; then\n python3 -m py_compile \"$extract/opt-skladchik-reports-monitor/monitor.py\" \"$extract/opt-skladchik-reports-monitor/health_alert.py\" \"$extract/opt-skladchik-reports-monitor/operational_digest.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-runner.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-core\"\n py_status=ok\n fi\n printf '%s_STATUS=OK\\n%s_RUN=%s\\n%s_ARCHIVE_BYTES=%s\\n%s_SHA256_OK=yes\\n%s_TAR_OK=yes\\n%s_REQUIRED_FILES_OK=yes\\n%s_COOKIE_JAR_OK=yes\\n%s_EXTENSION_HEALTH_FILES_OK=yes\\n%s_EXTENSION_SCRIPTS_OK=yes\\n%s_EXTENSION_UNITS_OK=yes\\n%s_SQLITE=%s\\n%s_PY_COMPILE=%s\\n' \"$label\" \"$label\" \"$latest\" \"$label\" \"$(stat -c %s \"$archive\")\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$sqlite_status\" \"$label\" \"$py_status\" >> \"$WORK/result.env\"\n log \"${label} OK run=${latest} bytes=$(stat -c %s \"$archive\") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}\"\n}\n: > \"$WORK/result.env\"\ncheck_one_remote CLOUD_1 \"$CLOUD_1\"\ncheck_one_remote CLOUD_2 \"$CLOUD_2\"\n. \"$WORK/result.env\"\nprintf 'STATUS=OK\\nTS=%s\\nRUN_ID=%s\\nREMOTE_BASE=%s\\nCLOUD_1_STATUS=%s\\nCLOUD_1_RUN=%s\\nCLOUD_2_STATUS=%s\\nCLOUD_2_RUN=%s\\nSHA256_OK=yes\\nTAR_OK=yes\\nREQUIRED_FILES_OK=yes\\nCOOKIE_JAR_OK=yes\\nEXTENSION_HEALTH_FILES_OK=yes\\nEXTENSION_SCRIPTS_OK=yes\\nEXTENSION_UNITS_OK=yes\\nSQLITE_OK=yes\\nPY_COMPILE_OK=yes\\nRESTORE_CHECK_VERSION=4\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$REMOTE_BASE\" \"${CLOUD_1_STATUS:-unknown}\" \"${CLOUD_1_RUN:-}\" \"${CLOUD_2_STATUS:-unknown}\" \"${CLOUD_2_RUN:-}\" > \"$HEALTH\"\nfind \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\nlog \"OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}\"\nRESTORE_SOURCE_SEGMENT_END=1\nRESTORE_SYSTEMD_BINDINGS_BEGIN=1\nRESTORE_SYSTEMD_BINDING_COUNT=0\nRESTORE_SYSTEMD_BINDINGS_END=1\nCLOUD_1_DAILY_LSF_RC=3\nCLOUD_1_DAILY_STDERR_SHA256=bb3ce24b6401f914e787d66ae4a3fb5460455a41fab035e6ede56f0af463781f\n2026/09/22 08:47:57 ERROR : : error listing: directory not found\n2026/09/22 08:47:57 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found\nCLOUD_2_DAILY_LSF_RC=3\nCLOUD_2_DAILY_STDERR_SHA256=a3c018181b7d581d2a06e107cb28685794108bc2066f6acc7abdfbc83545fefa\n2026/09/22 08:47:58 ERROR : : error listing: directory not found\n2026/09/22 08:47:58 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found\nCLOUDB_REMOTE_READ_CLASS=INCOMPLETE_OR_ERROR\nSOURCE_WORKTREE_VALID=true\nSOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614\nSOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_PUSHURL_CLASS=DISABLED\nSOURCE_REMOTE_HEAD_PROBE_RC=0\nSOURCE_REMOTE_BRANCH_PROBE_RC=0\nSOURCE_REMOTE_ALL_PROBE_RC=0\nSOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b\nSOURCE_REMOTE_BRANCH_HEAD=ABSENT\nSOURCE_LOCAL_COMMIT_REMOTE_REF_COUNT=0\nSOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT\nDECISION=HOLD_BACKUP_1115_ONE_OR_MORE_PATHS_UNSAFE\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1115_FAILURE\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1114_DUE_RESTORE_RCA\nCHANGES_MADE_BY_1115=false\nPRODUCT_MUTATION_BY_1115=false\nVM_MUTATION_BY_1115=false\nCLOUD_MUTATION_BY_1115=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1114_DUE_RESTORE_RCA_END=1\n" + "output": "WORKERS2_BACKUP_1115_ROOT_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1\nMODE=read-only\nCOMPONENT=homelab-backup-1115-root-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_FAILURE_AND_CANONICAL_SKLADCHIK_EVIDENCE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE;NO_REMOTE_MUTATION\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nUS_NETBIRD_STATE_SAFE_BEGIN=1\nUS_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26\n{\"schema\":1,\"logical_id\":\"us-netbird\",\"result\":\"FAIL\",\"generation\":\"v2-us-netbird-20260922T055316Z-d5c92acdd2e8\",\"updated_at_utc\":\"2026-09-22T05:53:31Z\",\"last_success_epoch\":1789704670,\"last_attempt_epoch\":1790056411,\"retry_after_epoch\":1790078011,\"consecutive_failures\":14,\"replica_count\":null}\n{\"path\":\"consecutive_failures\",\"value\":14}\n{\"path\":\"failure_backoff_seconds\",\"value\":21600}\n{\"path\":\"last_attempt_epoch\",\"value\":1790056411}\n{\"path\":\"last_success_epoch\",\"value\":1789704670}\n{\"path\":\"last_success_generation\",\"value\":\"v2-us-netbird-20260918T041032Z-d7648be4c16b\"}\n{\"path\":\"last_success_manifest_sha256\",\"value\":\"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5\"}\n{\"path\":\"last_success_payload_sha256\",\"value\":\"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf\"}\n{\"path\":\"reason\",\"value\":\"PIPE_FAILED\"}\n{\"path\":\"result\",\"value\":\"FAIL\"}\n{\"path\":\"retry_after_epoch\",\"value\":1790078011}\nUS_NETBIRD_STATE_SAFE_END=1\nUS_NETBIRD_SOURCE_HITS_BEGIN=1\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7\n155- for node in ('pve01','pve02','pve03'):\n156- out.append({'logical_id':'node-'+node,'scope':'node','type':'pve_node_bundle','node':node,'due_seconds':7*d,'enabled':True,'green1':node=='pve01'})\n157- out += [\n158- {'logical_id':'md-relay','scope':'vps','type':'vps_md','host':'[PRIVATE_IP]','port':22,'user':'ops','hostkey_fingerprint':'SHA256:DXJGwunC5tEVvnC3nRxyayTN8FXmyJcVgf+Oa6udQsY','due_seconds':d,'enabled':True,'green1':True},\n159- {'logical_id':'spb-ingress','scope':'vps','type':'vps_spb','host':'[PRIVATE_IP]','port':22,'user':'edgeadmin','hostkey_fingerprint':'SHA256:Ef+HsZb8lIXoadvYBXuGKIL3H7NyJMmf9EYCoJRoSv8','due_seconds':d,'enabled':True,'green1':True},\n160: {'logical_id':'us-netbird','scope':'vps','type':'vps_us','host':'[PRIVATE_IP]','port':22,'user':'root','hostkey_fingerprint':'SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE','due_seconds':d,'enabled':True,'green1':True},\n161- {'logical_id':'router-netcraze','scope':'router','type':'router','due_seconds':d,'enabled':True,'green1':True},\n162- ]\n163- sites=[\n164- ('codevipe',160,'pve02','/var/www/forums/codevipe/public','xf_codevipe','single_transaction'),\n165- ('dsmods',160,'pve02','/var/www/forums/dsmods/public','xf_dsmods','single_transaction'),\n--\n216- except subprocess.TimeoutExpired:\n217- for q in (src,z):\n218- try:q.kill()\n219- except Exception:pass\n220- raise BackupError('PIPE_TIMEOUT')\n221: if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}')\n222-\n223-def ensure_dir(p,mode=0o700):\n224- p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p\n225-\n226-def atomic_write(path,data,mode=0o600):\n--\n248- try:\n249- j=json.loads(p.read_text()); d[j['logical_id']]=j\n250- except Exception: continue\n251- return d\n252-\n253:def write_state(logical_id,result,**kw):\n254- if result not in RESULT_CLASSES: raise BackupError('RESULT_CLASS_INVALID')\n255- j={'schema':1,'logical_id':logical_id,'result':result,'updated_at_utc':utcnow(),**kw}\n256- if result=='SUCCESS': j['last_success_epoch']=epoch()\n257- atomic_write(STATE_ROOT/'state'/(logical_id+'.json'),canonical_json(j)+b'\\n',0o600)\n258- return j\n--\n384- if not names: raise BackupError('TAR_ZST_EMPTY')\n385- if required_member and required_member not in names: raise BackupError('REQUIRED_MEMBER_MISSING')\n386-\n387-def validate_payload(t,path):\n388- typ=t['type']\n389: if typ in ('pve_node_bundle','vps_md','vps_spb','vps_us','router'):\n390- validate_tar_zst(path)\n391- elif typ=='pve_guest':\n392- if t['guest_type']=='lxc': validate_tar_zst(path)\n393- else:\n394- # Green set is intentionally CT-only. Scheduled QEMU backups still get dual remote byte readback.\n--\n405-def cluster_inventory():\n406- p=run(['pvesh','get','/cluster/resources','--type','vm','--output-format','json'],timeout=60)\n407- try:return json.loads(p.stdout)\n408- except Exception: raise BackupError('PVE_INVENTORY_JSON')\n409-\n410:def ssh_remote(node,argv,input=None,timeout=1800,check=True):\n411- cmd='PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec '+shlex.join([str(x) for x in argv])\n412- if node=='pve01': return run([str(x) for x in argv],input=input,timeout=timeout,check=check)\n413- return run([SSH,'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node,cmd],input=input,timeout=timeout,check=check)\n414-\n415-def _pve_dumpdir_spec(guest_type):\n--\n471- if i+1>=len(argv) or argv[i+1]!=str(scratch): raise BackupError('PROVEN_DUMP_ARGV_TMPDIR_BINDING')\n472- elif '--tmpdir='+str(scratch) not in argv: raise BackupError('PROVEN_DUMP_ARGV_NO_TMPDIR')\n473- return argv,job\n474-\n475-def _default_unprivileged_lxc(node,vmid):\n476: q=ssh_remote(node,['pct','config',str(vmid)],timeout=30)\n477- lines=[x.strip() for x in q.stdout.decode('utf-8','replace').splitlines()]\n478- if 'unprivileged: 1' not in lines: raise BackupError('LXC_NOT_UNPRIVILEGED')\n479- if any(x.startswith('lxc.idmap:') or x.startswith('lxc.idmap =') for x in lines): raise BackupError('LXC_CUSTOM_IDMAP_UNSUPPORTED')\n480-\n481-def mapped_scratch_probe(node,scratch):\n482- tool=shutil.which('lxc-usernsexec') or '/usr/bin/lxc-usernsexec'\n483- pos=[tool,'-m','u:0:100000:65536','-m','g:0:100000:65536','--','/usr/bin/test','-x',str(scratch)]\n484: q=ssh_remote(node,pos,timeout=30,check=False)\n485- if q.returncode!=0: raise BackupError('MAPPED_UID_ACCESS_PROBE_FAILED')\n486-\n487-def _stream_argv_to_local(node,argv,dest,timeout=7200):\n488- dest=pathlib.Path(dest)\n489- if node=='pve01': cmd=argv\n--\n509- if typ!=t['guest_type']: raise BackupError('PVE_GUEST_TYPE_DRIFT')\n510- if typ=='lxc':\n511- _require_exact_source(P270,P270_SHA,'P270_IDENTITY');_require_exact_source(P282,P282_SHA,'P282_IDENTITY')\n512- _default_unprivileged_lxc(node,vmid)\n513- scratch=f'/var/tmp/homelab-vzdump-v2-{gen}'\n514: ssh_remote(node,['install','-d','-m','710','-o','0','-g','100000',scratch],timeout=30)\n515: q=ssh_remote(node,['stat','-c','%a:%u:%g',scratch],timeout=30)\n516- if q.stdout.decode().strip()!='710:0:100000': raise BackupError('PVE_MAPPED_SCRATCH_ACCESS_CONTRACT')\n517- try:\n518- mapped_scratch_probe(node,scratch)\n519- argv,job=proven_lxc_dump_argv(t,row,node,scratch)\n520- local=work/'payload.bin'\n521- _stream_argv_to_local(node,argv,local,timeout=7200)\n522- if local.stat().st_size>int(job['archive_cap_bytes']): raise BackupError('VZDUMP_ARCHIVE_CAP_EXCEEDED')\n523- except Exception:\n524: ssh_remote(node,['rm','-rf','--',scratch],timeout=60,check=False);raise\n525- cleanup={'kind':'pve_source_dir','node':node,'path':scratch}\n526- return local,{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status'),'producer_contract':'proven-270-282-stream'},cleanup\n527- # QEMU path remains unchanged: bounded dedicated dumpdir with later cloud readback/reconstruction.\n528- srcdir=f'/var/tmp/homelab-vzdump-v2-{gen}'\n529: ssh_remote(node,['install','-d','-m','700','-o','0','-g','0',srcdir],timeout=30)\n530- try:\n531: ssh_remote(node,['vzdump',str(vmid),'--mode','snapshot','--compress','zstd','--dumpdir',srcdir],timeout=7200)\n532: p=ssh_remote(node,['find',srcdir,'-maxdepth','1','-type','f','-printf','%p\\n'],timeout=30)\n533- files=[x for x in p.stdout.decode().splitlines() if x.strip()]\n534- if len(files)!=1: raise BackupError('VZDUMP_OUTPUT_NOT_UNIQUE')\n535- src=files[0];local=work/'payload.bin'\n536- if node=='pve01': local=pathlib.Path(src)\n537- else: run([SCP,'-q','-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node+':'+src,str(local)],timeout=7200)\n538- if not pathlib.Path(local).is_file() or pathlib.Path(local).stat().st_size<=0: raise BackupError('VZDUMP_PAYLOAD_MISSING')\n539- except Exception:\n540: ssh_remote(node,['rm','-rf','--',srcdir],timeout=60,check=False);raise\n541- return pathlib.Path(local),{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status')},{'kind':'pve_source_dir','node':node,'path':srcdir}\n542-\n543-def operator_key_fingerprint():\n544- if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING')\n545- p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30)\n--\n563- return good[0]\n564-\n565-def ensure_vps_known_hosts(targets):\n566- lines=[]\n567- for t in targets:\n568: if t['type'] not in ('vps_md','vps_spb','vps_us'): continue\n569- lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint']))\n570- atomic_write(VPS_KNOWN_HOSTS,('\\n'.join(lines)+'\\n').encode(),0o600)\n571-\n572-def vps_ssh_argv(t, *, use_agent=False):\n573- a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15']\n--\n598- tag=p.stdout.decode().strip()\n599- if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT')\n600- vps_run(t,['true'],timeout=30,use_agent=False)\n601- results[t['logical_id']]=tag\n602- # US was already operator-key based; prove it agentless too.\n603: us=[x for x in targets if x['type']=='vps_us'][0]\n604- vps_run(us,['true'],timeout=30,use_agent=False)\n605- results[us['logical_id']]='ALREADY_US_OPERATOR_KEY'\n606- return results\n607-\n608-def remote_hashset(t,command):\n609- p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False)\n610- return sha256_bytes(p.stdout)\n611-\n612:def capture_vps_md(t,gen,work):\n613- files=['etc/netbird/install.conf','var/lib/netbird/state.json','var/lib/netbird/service.json','var/lib/netbird/active_profile.json','var/lib/netbird/default.json']\n614- cmd=\"sudo -n sha256sum -- \"+' '.join(shlex.quote('/'+x) for x in files)\n615- pre=remote_hashset(t,cmd)\n616- remote=shlex.join(['sudo','-n','tar','--numeric-owner','-C','/','-cf','-']+files)\n617- argv=vps_ssh_argv(t,use_agent=False)+[remote]\n618- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=600)\n619- post=remote_hashset(t,cmd)\n620- if pre!=post: raise BackupError('MD_SOURCE_CHANGED_DURING_CAPTURE')\n621- return payload,{'generation':gen,'source_class':'vps_md','consistency':'FILESET_PRE_POST_HASH_STABILITY','hashset_sha256':pre},None\n622-\n623:def capture_vps_spb(t,gen,work):\n624- hcmd=\"find /etc/nginx -xdev -type f -print0 | sort -z | xargs -0 sha256sum\"\n625- pre=remote_hashset(t,hcmd)\n626- remote=shlex.join(['tar','--numeric-owner','-C','/','-cf','-','etc/nginx'])\n627- argv=vps_ssh_argv(t,use_agent=False)+[remote]\n628- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=900)\n--\n657- if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33)\n658- b=dst.serialize(); dst.close(); src.close()\n659- ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b))\n660-'''\n661-\n662:def capture_vps_us(t,gen,work):\n663- remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT])\n664- argv=vps_ssh_argv(t,use_agent=False)+[remote]\n665- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200)\n666: return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None\n667-\n668-def _simple_shell_assignments(text):\n669- vals={}\n670- for line in text.splitlines():\n671- m=re.match(r'^\\s*(?:export\\s+)?([A-Z_][A-Z0-9_]*)=(.*)$',line)\n--\n801-\n802-def qga_exec(node,vmid,guest_argv,input_bytes=None,timeout=600):\n803- args=['qm','guest','exec',str(vmid),'--synchronous','1','--timeout',str(int(timeout))]\n804- if input_bytes is not None: args += ['--pass-stdin','1']\n805- args += ['--']+[str(x) for x in guest_argv]\n806: p=ssh_remote(node,args,input=input_bytes,timeout=timeout+60)\n807- try:j=json.loads(p.stdout)\n808- except Exception: raise BackupError('QGA_JSON')\n809- rc=j.get('exitcode',j.get('exit-code'))\n810- if rc is None: rc=0 if j.get('exited') else None\n811- if rc not in (0,None): raise BackupError('QGA_GUEST_COMMAND_FAILED',str(rc))\n--\n854-\n855-def capture_node_bundle(t,gen,work):\n856- node=t['node']; raw=node_bundle_paths(node); paths=[]\n857- for rel in raw:\n858- if node=='pve01': ok=(pathlib.Path('/')/rel).exists()\n859: else: ok=(ssh_remote(node,['test','-e','/'+rel],timeout=20,check=False).returncode==0)\n860- if ok: paths.append(rel)\n861- required={'etc/network/interfaces','etc/hosts','etc/hostname','etc/pve/nodes/'+node}\n862- if not required.issubset(set(paths)): raise BackupError('NODE_BUNDLE_REQUIRED_PATH_MISSING')\n863- payload=work/'payload.tar.zst'\n864- argv=['tar','--numeric-owner','--xattrs','--acls','-C','/','-cf','-']+paths\n--\n870-\n871-def capture_target(t,gen,work):\n872- typ=t['type']\n873- if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n874- if typ=='pve_node_bundle': return capture_node_bundle(t,gen,work)\n875: if typ=='vps_md': return capture_vps_md(t,gen,work)\n876: if typ=='vps_spb': return capture_vps_spb(t,gen,work)\n877: if typ=='vps_us': return capture_vps_us(t,gen,work)\n878- if typ=='router': return capture_router(t,gen,work)\n879- if typ=='xenforo': return capture_xenforo(t,gen,work)\n880- raise BackupError('CAPTURE_TYPE_UNKNOWN')\n881-\n882-def cleanup_source(cleanup):\n883- if not cleanup: return\n884- if cleanup.get('kind')=='pve_source_dir':\n885: ssh_remote(cleanup['node'],['rm','-rf','--',cleanup['path']],timeout=60)\n886-\n887-def control_verify(t):\n888- engine=pathlib.Path('/usr/local/sbin/homelab-control-backup')\n889- if sha256_file(engine)!='0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57': raise BackupError('CONTROL_ENGINE_IDENTITY')\n890- root=pathlib.Path('/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2')\n--\n895- if j.get('status')=='SEALED' and j.get('generation'): rows.append((p.stat().st_mtime,j['generation']))\n896- except Exception: continue\n897- if not rows: raise BackupError('CONTROL_SEALED_GENERATION_MISSING')\n898- gen=sorted(rows)[-1][1]\n899- p=run([str(engine),'verify',gen],timeout=1800)\n900: write_state(t['logical_id'],'SUCCESS',generation=gen,verification='existing_control_engine_verify',last_success_epoch=epoch())\n901- return {'logical_id':t['logical_id'],'generation':gen,'result':'SUCCESS','payload_bytes':0,'replicas':2}\n902-\n903:def backup_target(t,green_verify=False):\n904- if t['type']=='control': return control_verify(t)\n905- gen=generation_id(t['logical_id']); work=ensure_dir(WORK_ROOT/gen,0o700)\n906- payload=None; cleanup=None; sealed=False\n907- try:\n908- payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n--\n916- ha=reconstruct_replica(manifest,manifest['replicas'][0],a); validate_payload(t,a)\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py SHA256=2b5eb4fc0f6f466a0d3e5a007af987092b6c17baa3ff9241a6afc2c2e97df1d5\n2-from __future__ import annotations\n3-import errno,fcntl,importlib.util,os,sys\n4-from pathlib import Path\n5-ROOT=Path(__file__).resolve().parent\n6-CT_ID='guest-pve01-lxc-112'\n7:LEGACY_IDS=('node-pve01','md-relay','spb-ingress','us-netbird','router-netcraze','xf-codevipe','xf-zakrutim','control-bootstrap-v2')\n8-LOCK=Path('/run/homelab-backup-heavy.lock')\n9-\n10-def _load(name,path):\n11- spec=importlib.util.spec_from_file_location(name,str(path))\n12- if spec is None or spec.loader is None: raise RuntimeError('MODULE_SPEC')\n--\n14-\n15-legacy=_load('_backupv2_legacy486',ROOT/'legacy_engine.py')\n16-ct112=_load('_backupv2_ct112_486',ROOT/'ct112_handler.py')\n17-driver=_load('_backupv2_driver454_479',ROOT/'representative_driver.py')\n18-_legacy_load_targets=legacy.load_targets\n19:_legacy_backup_target=legacy.backup_target\n20-\n21-def canonical_targets():\n22- rows=_legacy_load_targets()\n23- if not isinstance(rows,list) or len(rows)!=28: raise legacy.BackupError('TARGET_REGISTRY_COUNT')\n24- out=[dict(x) for x in rows]\n--\n35- 'guest_type':'lxc','due_seconds':due,'enabled':True,'green1':True}\n36-\n37-def load_targets():\n38- base=canonical_targets();return base+[_ct_descriptor(base)]\n39-\n40:def backup_target(t,green_verify=False):\n41- if not isinstance(t,dict) or t.get('logical_id')!=CT_ID:\n42: return _legacy_backup_target(t,green_verify)\n43- try:\n44: r=ct112.run_target(t,state_writer=legacy.write_state)\n45- return {'logical_id':CT_ID,'generation':r['generation'],'result':'SUCCESS','payload_bytes':r['payload_bytes'],\n46- 'replicas':2,'manifest_sha256':r['manifest_sha256'],'payload_sha256':r['payload_sha256']}\n47- except BaseException:\n48: try: legacy.write_state(CT_ID,'FAIL',reason='CT112_PIPELINE_FAILED')\n49- except BaseException: pass\n50- raise legacy.BackupError('CT112_PIPELINE_FAILED') from None\n51-\n52-legacy.load_targets=load_targets\n53:legacy.backup_target=backup_target\n54-\n55-def _legacy_reused_boundary_selftest():\n56- if legacy.PART_SIZE!=536870912: raise RuntimeError('LEGACY_PART_SIZE')\n57- if len(legacy.REMOTES)!=13 or len({p for _,p in legacy.REMOTES})!=13: raise RuntimeError('LEGACY_REMOTES')\n58- d=legacy.default_targets()\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py SHA256=ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793\n57- if gp==legacy_path:\n58- found[id(g)]=g\n59- if len(found)!=1:\n60- raise RuntimeError('LEGACY_PROVENANCE_CARDINALITY')\n61- g=next(iter(found.values()))\n62: required=('capture_pve_guest','backup_target','load_targets','cluster_inventory','ssh_remote',\n63- '_default_unprivileged_lxc','mapped_scratch_probe','_stream_argv_to_local','run',\n64: 'choose_due_target','write_state','epoch')\n65- if any(not callable(g.get(n)) for n in required):\n66- raise RuntimeError('LEGACY_PROVENANCE_API')\n67- original=g['capture_pve_guest']\n68- proxy=_GlobalsProxy(g)\n69- if not isinstance(g.get('SCP'),str) or not g['SCP'].startswith('/'):\n--\n73- raise RuntimeError('QEMU_ADAPTER_BIND_RESULT')\n74- patched=lxc_adapter.bind(proxy,qemu_patched)\n75- if not callable(patched):\n76- raise RuntimeError('LXC_ADAPTER_BIND_RESULT')\n77- g['capture_pve_guest']=patched\n78: scheduler_wired=scheduler_adapter.bind(proxy,g['choose_due_target'],g['write_state'])\n79: if g.get('choose_due_target') is not scheduler_wired['choose_due_target'] or g.get('write_state') is not scheduler_wired['write_state']:\n80- raise RuntimeError('SCHEDULER_ADAPTER_BIND_RESULT')\n81- return {'legacy_globals':g,'legacy':proxy,'original_capture':original,'qemu_patched_capture':qemu_patched,'patched_capture':patched,'scheduler_wired':scheduler_wired}\n82-\n83-def _load():\n84- _verify_base()\n--\n105- wrapper,lxc_adapter,qemu_adapter,scheduler_adapter,wired=_load()\n106- if wired['legacy_globals'].get('capture_pve_guest') is not wired['patched_capture']:\n107- raise RuntimeError('PATCH_BINDING')\n108- if wired['legacy_globals'].get('choose_due_target') is not wired['scheduler_wired']['choose_due_target']:\n109- raise RuntimeError('SCHEDULER_CHOOSE_BINDING')\n110: if wired['legacy_globals'].get('write_state') is not wired['scheduler_wired']['write_state']:\n111- raise RuntimeError('SCHEDULER_WRITE_BINDING')\n112- if [scheduler_adapter.backoff_seconds(n) for n in range(1,7)]!=[1800,3600,7200,14400,21600,21600]:\n113- raise RuntimeError('SCHEDULER_BACKOFF_CONTRACT')\n114: if not callable(wrapper.backup_target) or not callable(wrapper.main) or not callable(wrapper.load_targets):\n115- raise RuntimeError('WRAPPER_PUBLIC_API')\n116- print('SELFTEST_OVERLAY504=PASS',flush=True)\n117- return 0\n118-\n119-def main(argv=None):\n--\n131- try:\n132- with tempfile.TemporaryDirectory(prefix='homelab-scheduler-proof-',dir='/run') as td:\n133- g['STATE_ROOT']=pathlib.Path(td); (g['STATE_ROOT']/'state').mkdir(mode=0o700)\n134- g['epoch']=lambda: now[0]\n135- common={'payload_sha256':'a'*64,'manifest_sha256':'b'*64,'replicas':['proof-r1','proof-r2']}\n136: legacy.write_state('proof-a','SUCCESS',generation='proof-a-good',**common)\n137: legacy.write_state('proof-b','SUCCESS',generation='proof-b-good',**common)\n138- now[0]=100000\n139: legacy.write_state('proof-a','FAIL',generation='proof-a-bad',reason='PROOF_FAILURE',sealed=False)\n140- state=legacy.load_state_all(); a=state['proof-a']\n141- targets=[{'logical_id':'proof-a','enabled':True,'due_seconds':100},{'logical_id':'proof-b','enabled':True,'due_seconds':100}]\n142- first=legacy.choose_due_target(targets,state,now[0])\n143- if not isinstance(first,dict) or first.get('logical_id')!='proof-b': raise RuntimeError('SCHEDULER_PROOF_PEER_NOT_SELECTED')\n144: retry=int(a.get('retry_after_epoch',0)); preserved=int(a.get('last_success_epoch',0))\n145: if preserved!=1000 or a.get('last_success_generation')!='proof-a-good' or int(a.get('consecutive_failures',0))!=1 or retry!=101800:\n146- raise RuntimeError('SCHEDULER_PROOF_FAILURE_STATE')\n147- now[0]=retry+1\n148- second=legacy.choose_due_target(targets,legacy.load_state_all(),now[0])\n149- if not isinstance(second,dict) or second.get('logical_id')!='proof-a': raise RuntimeError('SCHEDULER_PROOF_RETRY_NOT_REELIGIBLE')\n150: legacy.write_state('proof-a','SUCCESS',generation='proof-a-recovered',**common)\n151- recovered=legacy.load_state_all()['proof-a']\n152: if int(recovered.get('consecutive_failures',-1))!=0 or int(recovered.get('retry_after_epoch',-1))!=0:\n153- raise RuntimeError('SCHEDULER_PROOF_SUCCESS_RESET')\n154: proof={'failed_target':'proof-a','peer_selected_during_backoff':first['logical_id'],'retry_after_epoch':retry,'last_success_preserved':preserved==1000,'retry_target_after_backoff':second['logical_id'],'success_resets_failures':True,'production_state_root_used':False}\n155- print('SCHEDULER_FAILURE_ISOLATION_PROOF='+json.dumps(proof,separators=(',',':'),sort_keys=True),flush=True)\n156- return 0\n157- finally:\n158- g['STATE_ROOT']=old_root; g['epoch']=old_epoch\n159- if argv==['scheduler-select-live']:\n--\n174- if len(argv)==2 and argv[0]=='verify-target':\n175- t=_target(wrapper,argv[1])\n176- if t.get('logical_id')=='guest-pve01-lxc-112':\n177- raise RuntimeError('VERIFY_SYNTHETIC_CT112_FORBIDDEN')\n178- try:\n179: out=wrapper.backup_target(t,True)\n180- except legacy.BackupError as e:\n181- print('VERIFY_TARGET_RESULT=FAIL',flush=True)\n182- print('VERIFY_TARGET_REASON='+str(e.code),flush=True)\n183- return 2\n184- print('VERIFY_TARGET_RESULT='+json.dumps(out,separators=(',',':'),sort_keys=True),flush=True)\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/scheduler_adapter.py SHA256=a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96\n14- n=int(v)\n15- except Exception:\n16- return default\n17- return n if n>=0 else default\n18-\n19:def backoff_seconds(consecutive_failures:int)->int:\n20: n=max(1,_as_nonneg_int(consecutive_failures,1))\n21- return min(MAX_BACKOFF,MIN_BACKOFF*(2**min(n-1,4)))\n22-\n23-def _read_prior(legacy,logical_id):\n24- root=pathlib.Path(legacy.STATE_ROOT)/'state'\n25- p=root/(str(logical_id)+'.json')\n--\n60- if not isinstance(t,dict) or t.get('enabled') is not True: continue\n61- lid=t.get('logical_id'); seconds=_as_nonneg_int(t.get('due_seconds'),0)\n62- _need(isinstance(lid,str) and lid and seconds>0,'SCHEDULER_TARGET_SCHEMA')\n63- s=state.get(lid,{})\n64- if not isinstance(s,dict): s={}\n65: retry_after=_as_nonneg_int(s.get('retry_after_epoch'),0)\n66: if retry_after>now: continue\n67- last_success=_as_nonneg_int(s.get('last_success_epoch'),0)\n68- age=(now-last_success) if last_success and now>=last_success else (10**12 if last_success==0 else 0)\n69- if age=len(argv) or argv[i+1]!=str(scratch): raise BackupError('PROVEN_DUMP_ARGV_TMPDIR_BINDING') +472- elif '--tmpdir='+str(scratch) not in argv: raise BackupError('PROVEN_DUMP_ARGV_NO_TMPDIR') +473- return argv,job +474- +475-def _default_unprivileged_lxc(node,vmid): +476: q=ssh_remote(node,['pct','config',str(vmid)],timeout=30) +477- lines=[x.strip() for x in q.stdout.decode('utf-8','replace').splitlines()] +478- if 'unprivileged: 1' not in lines: raise BackupError('LXC_NOT_UNPRIVILEGED') +479- if any(x.startswith('lxc.idmap:') or x.startswith('lxc.idmap =') for x in lines): raise BackupError('LXC_CUSTOM_IDMAP_UNSUPPORTED') +480- +481-def mapped_scratch_probe(node,scratch): +482- tool=shutil.which('lxc-usernsexec') or '/usr/bin/lxc-usernsexec' +483- pos=[tool,'-m','u:0:100000:65536','-m','g:0:100000:65536','--','/usr/bin/test','-x',str(scratch)] +484: q=ssh_remote(node,pos,timeout=30,check=False) +485- if q.returncode!=0: raise BackupError('MAPPED_UID_ACCESS_PROBE_FAILED') +486- +487-def _stream_argv_to_local(node,argv,dest,timeout=7200): +488- dest=pathlib.Path(dest) +489- if node=='pve01': cmd=argv +-- +509- if typ!=t['guest_type']: raise BackupError('PVE_GUEST_TYPE_DRIFT') +510- if typ=='lxc': +511- _require_exact_source(P270,P270_SHA,'P270_IDENTITY');_require_exact_source(P282,P282_SHA,'P282_IDENTITY') +512- _default_unprivileged_lxc(node,vmid) +513- scratch=f'/var/tmp/homelab-vzdump-v2-{gen}' +514: ssh_remote(node,['install','-d','-m','710','-o','0','-g','100000',scratch],timeout=30) +515: q=ssh_remote(node,['stat','-c','%a:%u:%g',scratch],timeout=30) +516- if q.stdout.decode().strip()!='710:0:100000': raise BackupError('PVE_MAPPED_SCRATCH_ACCESS_CONTRACT') +517- try: +518- mapped_scratch_probe(node,scratch) +519- argv,job=proven_lxc_dump_argv(t,row,node,scratch) +520- local=work/'payload.bin' +521- _stream_argv_to_local(node,argv,local,timeout=7200) +522- if local.stat().st_size>int(job['archive_cap_bytes']): raise BackupError('VZDUMP_ARCHIVE_CAP_EXCEEDED') +523- except Exception: +524: ssh_remote(node,['rm','-rf','--',scratch],timeout=60,check=False);raise +525- cleanup={'kind':'pve_source_dir','node':node,'path':scratch} +526- return local,{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status'),'producer_contract':'proven-270-282-stream'},cleanup +527- # QEMU path remains unchanged: bounded dedicated dumpdir with later cloud readback/reconstruction. +528- srcdir=f'/var/tmp/homelab-vzdump-v2-{gen}' +529: ssh_remote(node,['install','-d','-m','700','-o','0','-g','0',srcdir],timeout=30) +530- try: +531: ssh_remote(node,['vzdump',str(vmid),'--mode','snapshot','--compress','zstd','--dumpdir',srcdir],timeout=7200) +532: p=ssh_remote(node,['find',srcdir,'-maxdepth','1','-type','f','-printf','%p\n'],timeout=30) +533- files=[x for x in p.stdout.decode().splitlines() if x.strip()] +534- if len(files)!=1: raise BackupError('VZDUMP_OUTPUT_NOT_UNIQUE') +535- src=files[0];local=work/'payload.bin' +536- if node=='pve01': local=pathlib.Path(src) +537- else: run([SCP,'-q','-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node+':'+src,str(local)],timeout=7200) +538- if not pathlib.Path(local).is_file() or pathlib.Path(local).stat().st_size<=0: raise BackupError('VZDUMP_PAYLOAD_MISSING') +539- except Exception: +540: ssh_remote(node,['rm','-rf','--',srcdir],timeout=60,check=False);raise +541- return pathlib.Path(local),{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status')},{'kind':'pve_source_dir','node':node,'path':srcdir} +542- +543-def operator_key_fingerprint(): +544- if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING') +545- p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30) +-- +563- return good[0] +564- +565-def ensure_vps_known_hosts(targets): +566- lines=[] +567- for t in targets: +568: if t['type'] not in ('vps_md','vps_spb','vps_us'): continue +569- lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint'])) +570- atomic_write(VPS_KNOWN_HOSTS,('\n'.join(lines)+'\n').encode(),0o600) +571- +572-def vps_ssh_argv(t, *, use_agent=False): +573- a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15'] +-- +598- tag=p.stdout.decode().strip() +599- if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT') +600- vps_run(t,['true'],timeout=30,use_agent=False) +601- results[t['logical_id']]=tag +602- # US was already operator-key based; prove it agentless too. +603: us=[x for x in targets if x['type']=='vps_us'][0] +604- vps_run(us,['true'],timeout=30,use_agent=False) +605- results[us['logical_id']]='ALREADY_US_OPERATOR_KEY' +606- return results +607- +608-def remote_hashset(t,command): +609- p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False) +610- return sha256_bytes(p.stdout) +611- +612:def capture_vps_md(t,gen,work): +613- files=['etc/netbird/install.conf','var/lib/netbird/state.json','var/lib/netbird/service.json','var/lib/netbird/active_profile.json','var/lib/netbird/default.json'] +614- cmd="sudo -n sha256sum -- "+' '.join(shlex.quote('/'+x) for x in files) +615- pre=remote_hashset(t,cmd) +616- remote=shlex.join(['sudo','-n','tar','--numeric-owner','-C','/','-cf','-']+files) +617- argv=vps_ssh_argv(t,use_agent=False)+[remote] +618- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=600) +619- post=remote_hashset(t,cmd) +620- if pre!=post: raise BackupError('MD_SOURCE_CHANGED_DURING_CAPTURE') +621- return payload,{'generation':gen,'source_class':'vps_md','consistency':'FILESET_PRE_POST_HASH_STABILITY','hashset_sha256':pre},None +622- +623:def capture_vps_spb(t,gen,work): +624- hcmd="find /etc/nginx -xdev -type f -print0 | sort -z | xargs -0 sha256sum" +625- pre=remote_hashset(t,hcmd) +626- remote=shlex.join(['tar','--numeric-owner','-C','/','-cf','-','etc/nginx']) +627- argv=vps_ssh_argv(t,use_agent=False)+[remote] +628- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=900) +-- +657- if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33) +658- b=dst.serialize(); dst.close(); src.close() +659- ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b)) +660-''' +661- +662:def capture_vps_us(t,gen,work): +663- remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT]) +664- argv=vps_ssh_argv(t,use_agent=False)+[remote] +665- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200) +666: return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None +667- +668-def _simple_shell_assignments(text): +669- vals={} +670- for line in text.splitlines(): +671- m=re.match(r'^\s*(?:export\s+)?([A-Z_][A-Z0-9_]*)=(.*)$',line) +-- +801- +802-def qga_exec(node,vmid,guest_argv,input_bytes=None,timeout=600): +803- args=['qm','guest','exec',str(vmid),'--synchronous','1','--timeout',str(int(timeout))] +804- if input_bytes is not None: args += ['--pass-stdin','1'] +805- args += ['--']+[str(x) for x in guest_argv] +806: p=ssh_remote(node,args,input=input_bytes,timeout=timeout+60) +807- try:j=json.loads(p.stdout) +808- except Exception: raise BackupError('QGA_JSON') +809- rc=j.get('exitcode',j.get('exit-code')) +810- if rc is None: rc=0 if j.get('exited') else None +811- if rc not in (0,None): raise BackupError('QGA_GUEST_COMMAND_FAILED',str(rc)) +-- +854- +855-def capture_node_bundle(t,gen,work): +856- node=t['node']; raw=node_bundle_paths(node); paths=[] +857- for rel in raw: +858- if node=='pve01': ok=(pathlib.Path('/')/rel).exists() +859: else: ok=(ssh_remote(node,['test','-e','/'+rel],timeout=20,check=False).returncode==0) +860- if ok: paths.append(rel) +861- required={'etc/network/interfaces','etc/hosts','etc/hostname','etc/pve/nodes/'+node} +862- if not required.issubset(set(paths)): raise BackupError('NODE_BUNDLE_REQUIRED_PATH_MISSING') +863- payload=work/'payload.tar.zst' +864- argv=['tar','--numeric-owner','--xattrs','--acls','-C','/','-cf','-']+paths +-- +870- +871-def capture_target(t,gen,work): +872- typ=t['type'] +873- if typ=='pve_guest': return capture_pve_guest(t,gen,work) +874- if typ=='pve_node_bundle': return capture_node_bundle(t,gen,work) +875: if typ=='vps_md': return capture_vps_md(t,gen,work) +876: if typ=='vps_spb': return capture_vps_spb(t,gen,work) +877: if typ=='vps_us': return capture_vps_us(t,gen,work) +878- if typ=='router': return capture_router(t,gen,work) +879- if typ=='xenforo': return capture_xenforo(t,gen,work) +880- raise BackupError('CAPTURE_TYPE_UNKNOWN') +881- +882-def cleanup_source(cleanup): +883- if not cleanup: return +884- if cleanup.get('kind')=='pve_source_dir': +885: ssh_remote(cleanup['node'],['rm','-rf','--',cleanup['path']],timeout=60) +886- +887-def control_verify(t): +888- engine=pathlib.Path('/usr/local/sbin/homelab-control-backup') +889- if sha256_file(engine)!='0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57': raise BackupError('CONTROL_ENGINE_IDENTITY') +890- root=pathlib.Path('/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2') +-- +895- if j.get('status')=='SEALED' and j.get('generation'): rows.append((p.stat().st_mtime,j['generation'])) +896- except Exception: continue +897- if not rows: raise BackupError('CONTROL_SEALED_GENERATION_MISSING') +898- gen=sorted(rows)[-1][1] +899- p=run([str(engine),'verify',gen],timeout=1800) +900: write_state(t['logical_id'],'SUCCESS',generation=gen,verification='existing_control_engine_verify',last_success_epoch=epoch()) +901- return {'logical_id':t['logical_id'],'generation':gen,'result':'SUCCESS','payload_bytes':0,'replicas':2} +902- +903:def backup_target(t,green_verify=False): +904- if t['type']=='control': return control_verify(t) +905- gen=generation_id(t['logical_id']); work=ensure_dir(WORK_ROOT/gen,0o700) +906- payload=None; cleanup=None; sealed=False +907- try: +908- payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +-- +916- ha=reconstruct_replica(manifest,manifest['replicas'][0],a); validate_payload(t,a) +SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py SHA256=2b5eb4fc0f6f466a0d3e5a007af987092b6c17baa3ff9241a6afc2c2e97df1d5 +2-from __future__ import annotations +3-import errno,fcntl,importlib.util,os,sys +4-from pathlib import Path +5-ROOT=Path(__file__).resolve().parent +6-CT_ID='guest-pve01-lxc-112' +7:LEGACY_IDS=('node-pve01','md-relay','spb-ingress','us-netbird','router-netcraze','xf-codevipe','xf-zakrutim','control-bootstrap-v2') +8-LOCK=Path('/run/homelab-backup-heavy.lock') +9- +10-def _load(name,path): +11- spec=importlib.util.spec_from_file_location(name,str(path)) +12- if spec is None or spec.loader is None: raise RuntimeError('MODULE_SPEC') +-- +14- +15-legacy=_load('_backupv2_legacy486',ROOT/'legacy_engine.py') +16-ct112=_load('_backupv2_ct112_486',ROOT/'ct112_handler.py') +17-driver=_load('_backupv2_driver454_479',ROOT/'representative_driver.py') +18-_legacy_load_targets=legacy.load_targets +19:_legacy_backup_target=legacy.backup_target +20- +21-def canonical_targets(): +22- rows=_legacy_load_targets() +23- if not isinstance(rows,list) or len(rows)!=28: raise legacy.BackupError('TARGET_REGISTRY_COUNT') +24- out=[dict(x) for x in rows] +-- +35- 'guest_type':'lxc','due_seconds':due,'enabled':True,'green1':True} +36- +37-def load_targets(): +38- base=canonical_targets();return base+[_ct_descriptor(base)] +39- +40:def backup_target(t,green_verify=False): +41- if not isinstance(t,dict) or t.get('logical_id')!=CT_ID: +42: return _legacy_backup_target(t,green_verify) +43- try: +44: r=ct112.run_target(t,state_writer=legacy.write_state) +45- return {'logical_id':CT_ID,'generation':r['generation'],'result':'SUCCESS','payload_bytes':r['payload_bytes'], +46- 'replicas':2,'manifest_sha256':r['manifest_sha256'],'payload_sha256':r['payload_sha256']} +47- except BaseException: +48: try: legacy.write_state(CT_ID,'FAIL',reason='CT112_PIPELINE_FAILED') +49- except BaseException: pass +50- raise legacy.BackupError('CT112_PIPELINE_FAILED') from None +51- +52-legacy.load_targets=load_targets +53:legacy.backup_target=backup_target +54- +55-def _legacy_reused_boundary_selftest(): +56- if legacy.PART_SIZE!=536870912: raise RuntimeError('LEGACY_PART_SIZE') +57- if len(legacy.REMOTES)!=13 or len({p for _,p in legacy.REMOTES})!=13: raise RuntimeError('LEGACY_REMOTES') +58- d=legacy.default_targets() +SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py SHA256=ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793 +57- if gp==legacy_path: +58- found[id(g)]=g +59- if len(found)!=1: +60- raise RuntimeError('LEGACY_PROVENANCE_CARDINALITY') +61- g=next(iter(found.values())) +62: required=('capture_pve_guest','backup_target','load_targets','cluster_inventory','ssh_remote', +63- '_default_unprivileged_lxc','mapped_scratch_probe','_stream_argv_to_local','run', +64: 'choose_due_target','write_state','epoch') +65- if any(not callable(g.get(n)) for n in required): +66- raise RuntimeError('LEGACY_PROVENANCE_API') +67- original=g['capture_pve_guest'] +68- proxy=_GlobalsProxy(g) +69- if not isinstance(g.get('SCP'),str) or not g['SCP'].startswith('/'): +-- +73- raise RuntimeError('QEMU_ADAPTER_BIND_RESULT') +74- patched=lxc_adapter.bind(proxy,qemu_patched) +75- if not callable(patched): +76- raise RuntimeError('LXC_ADAPTER_BIND_RESULT') +77- g['capture_pve_guest']=patched +78: scheduler_wired=scheduler_adapter.bind(proxy,g['choose_due_target'],g['write_state']) +79: if g.get('choose_due_target') is not scheduler_wired['choose_due_target'] or g.get('write_state') is not scheduler_wired['write_state']: +80- raise RuntimeError('SCHEDULER_ADAPTER_BIND_RESULT') +81- return {'legacy_globals':g,'legacy':proxy,'original_capture':original,'qemu_patched_capture':qemu_patched,'patched_capture':patched,'scheduler_wired':scheduler_wired} +82- +83-def _load(): +84- _verify_base() +-- +105- wrapper,lxc_adapter,qemu_adapter,scheduler_adapter,wired=_load() +106- if wired['legacy_globals'].get('capture_pve_guest') is not wired['patched_capture']: +107- raise RuntimeError('PATCH_BINDING') +108- if wired['legacy_globals'].get('choose_due_target') is not wired['scheduler_wired']['choose_due_target']: +109- raise RuntimeError('SCHEDULER_CHOOSE_BINDING') +110: if wired['legacy_globals'].get('write_state') is not wired['scheduler_wired']['write_state']: +111- raise RuntimeError('SCHEDULER_WRITE_BINDING') +112- if [scheduler_adapter.backoff_seconds(n) for n in range(1,7)]!=[1800,3600,7200,14400,21600,21600]: +113- raise RuntimeError('SCHEDULER_BACKOFF_CONTRACT') +114: if not callable(wrapper.backup_target) or not callable(wrapper.main) or not callable(wrapper.load_targets): +115- raise RuntimeError('WRAPPER_PUBLIC_API') +116- print('SELFTEST_OVERLAY504=PASS',flush=True) +117- return 0 +118- +119-def main(argv=None): +-- +131- try: +132- with tempfile.TemporaryDirectory(prefix='homelab-scheduler-proof-',dir='/run') as td: +133- g['STATE_ROOT']=pathlib.Path(td); (g['STATE_ROOT']/'state').mkdir(mode=0o700) +134- g['epoch']=lambda: now[0] +135- common={'payload_sha256':'a'*64,'manifest_sha256':'b'*64,'replicas':['proof-r1','proof-r2']} +136: legacy.write_state('proof-a','SUCCESS',generation='proof-a-good',**common) +137: legacy.write_state('proof-b','SUCCESS',generation='proof-b-good',**common) +138- now[0]=100000 +139: legacy.write_state('proof-a','FAIL',generation='proof-a-bad',reason='PROOF_FAILURE',sealed=False) +140- state=legacy.load_state_all(); a=state['proof-a'] +141- targets=[{'logical_id':'proof-a','enabled':True,'due_seconds':100},{'logical_id':'proof-b','enabled':True,'due_seconds':100}] +142- first=legacy.choose_due_target(targets,state,now[0]) +143- if not isinstance(first,dict) or first.get('logical_id')!='proof-b': raise RuntimeError('SCHEDULER_PROOF_PEER_NOT_SELECTED') +144: retry=int(a.get('retry_after_epoch',0)); preserved=int(a.get('last_success_epoch',0)) +145: if preserved!=1000 or a.get('last_success_generation')!='proof-a-good' or int(a.get('consecutive_failures',0))!=1 or retry!=101800: +146- raise RuntimeError('SCHEDULER_PROOF_FAILURE_STATE') +147- now[0]=retry+1 +148- second=legacy.choose_due_target(targets,legacy.load_state_all(),now[0]) +149- if not isinstance(second,dict) or second.get('logical_id')!='proof-a': raise RuntimeError('SCHEDULER_PROOF_RETRY_NOT_REELIGIBLE') +150: legacy.write_state('proof-a','SUCCESS',generation='proof-a-recovered',**common) +151- recovered=legacy.load_state_all()['proof-a'] +152: if int(recovered.get('consecutive_failures',-1))!=0 or int(recovered.get('retry_after_epoch',-1))!=0: +153- raise RuntimeError('SCHEDULER_PROOF_SUCCESS_RESET') +154: proof={'failed_target':'proof-a','peer_selected_during_backoff':first['logical_id'],'retry_after_epoch':retry,'last_success_preserved':preserved==1000,'retry_target_after_backoff':second['logical_id'],'success_resets_failures':True,'production_state_root_used':False} +155- print('SCHEDULER_FAILURE_ISOLATION_PROOF='+json.dumps(proof,separators=(',',':'),sort_keys=True),flush=True) +156- return 0 +157- finally: +158- g['STATE_ROOT']=old_root; g['epoch']=old_epoch +159- if argv==['scheduler-select-live']: +-- +174- if len(argv)==2 and argv[0]=='verify-target': +175- t=_target(wrapper,argv[1]) +176- if t.get('logical_id')=='guest-pve01-lxc-112': +177- raise RuntimeError('VERIFY_SYNTHETIC_CT112_FORBIDDEN') +178- try: +179: out=wrapper.backup_target(t,True) +180- except legacy.BackupError as e: +181- print('VERIFY_TARGET_RESULT=FAIL',flush=True) +182- print('VERIFY_TARGET_REASON='+str(e.code),flush=True) +183- return 2 +184- print('VERIFY_TARGET_RESULT='+json.dumps(out,separators=(',',':'),sort_keys=True),flush=True) +SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/scheduler_adapter.py SHA256=a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96 +14- n=int(v) +15- except Exception: +16- return default +17- return n if n>=0 else default +18- +19:def backoff_seconds(consecutive_failures:int)->int: +20: n=max(1,_as_nonneg_int(consecutive_failures,1)) +21- return min(MAX_BACKOFF,MIN_BACKOFF*(2**min(n-1,4))) +22- +23-def _read_prior(legacy,logical_id): +24- root=pathlib.Path(legacy.STATE_ROOT)/'state' +25- p=root/(str(logical_id)+'.json') +-- +60- if not isinstance(t,dict) or t.get('enabled') is not True: continue +61- lid=t.get('logical_id'); seconds=_as_nonneg_int(t.get('due_seconds'),0) +62- _need(isinstance(lid,str) and lid and seconds>0,'SCHEDULER_TARGET_SCHEMA') +63- s=state.get(lid,{}) +64- if not isinstance(s,dict): s={} +65: retry_after=_as_nonneg_int(s.get('retry_after_epoch'),0) +66: if retry_after>now: continue +67- last_success=_as_nonneg_int(s.get('last_success_epoch'),0) +68- age=(now-last_success) if last_success and now>=last_success else (10**12 if last_success==0 else 0) +69- if age"$LOCK" -if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi -log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; } -fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; } -trap fail ERR -command -v rclone >/dev/null -log "START run_id=$RUN_ID" -cat_chunked(){ - local remote_path="$1" out="$2" plist part - : > "$out" - plist="$(mktemp)" - if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then - while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort) - else - rclone copyto "$remote_path" "$out" >/dev/null - fi - rm -f "$plist" - [ -s "$out" ] -} -check_one_remote(){ - local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status - rbase="${remote}${REMOTE_BASE}/daily" - latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)" - [ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; } - archive_name="skladchik-reports-monitor-${latest}.tar.gz" - remote_archive="${rbase}/${latest}/${archive_name}" - outdir="${WORK}/${label}-${latest}" - mkdir -p "$outdir" - archive="${outdir}/${archive_name}" - log "${label} DOWNLOAD run=${latest}" - cat_chunked "$remote_archive" "$archive" - cat_chunked "${remote_archive}.sha256" "${archive}.sha256.remote" - expected="$(awk '{print $1; exit}' "${archive}.sha256.remote")" - actual="$(sha256sum "$archive" | awk '{print $1}')" - [ -n "$expected" ] && [ "$expected" = "$actual" ] - list="${outdir}/tar-list.txt" - tar -tzf "$archive" > "$list" - grep -Eq '(^|/)etc-skladchik-reports-monitor/config\.ini$' "$list" - grep -Eq '(^|/)etc-skladchik-reports-monitor/secrets\.env$' "$list" - grep -Eq '(^|/)opt-skladchik-reports-monitor/monitor\.py$' "$list" - grep -Eq '(^|/)opt-skladchik-reports-monitor/health_alert\.py$' "$list" - grep -Eq '(^|/)var-lib-skladchik-reports-monitor/state\.sqlite$' "$list" - grep -Eq '(^|/)var-lib-skladchik-reports-monitor/health\.txt$' "$list" - grep -Eq '(^|/)var-lib-skladchik-reports-monitor/alert-health\.txt$' "$list" - grep -Eq '(^|/)var-lib-skladchik-reports-monitor/cookies\.jar$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor\.service$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor\.timer$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.service$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.timer$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.service$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.timer$' "$list" - grep -Eq '(^|/)opt-skladchik-reports-monitor/operational_digest\.py$' "$list" - grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-runner\.py$' "$list" - grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-core$' "$list" - grep -Eq '(^|/)var-lib-skladchik-reports-monitor/weekly-summary-health\.txt$' "$list" - grep -Eq '(^|/)var-lib-skladchik-reports-monitor/age-alert-health\.txt$' "$list" - grep -Eq '(^|/)var-lib-skladchik-reports-monitor/event-selftest-health\.txt$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.service$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.timer$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.service$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.timer$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.service$' "$list" - grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.timer$' "$list" - extract="${outdir}/extract" - mkdir -p "$extract" - tar -xzf "$archive" -C "$extract" - sqlite_status=skipped - if command -v sqlite3 >/dev/null && [ -s "$extract/var-lib-skladchik-reports-monitor/state.sqlite" ]; then - [ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ] - sqlite_status=ok - fi - py_status=skipped - if command -v python3 >/dev/null; then - python3 -m py_compile "$extract/opt-skladchik-reports-monitor/monitor.py" "$extract/opt-skladchik-reports-monitor/health_alert.py" "$extract/opt-skladchik-reports-monitor/operational_digest.py" "$extract/opt-skladchik-reports-monitor/event-selftest-runner.py" "$extract/opt-skladchik-reports-monitor/event-selftest-core" - py_status=ok - fi - printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env" - log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}" -} -: > "$WORK/result.env" -check_one_remote CLOUD_1 "$CLOUD_1" -check_one_remote CLOUD_2 "$CLOUD_2" -. "$WORK/result.env" -printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH" -find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done -log "OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}" -RESTORE_SOURCE_SEGMENT_END=1 -RESTORE_SYSTEMD_BINDINGS_BEGIN=1 -RESTORE_SYSTEMD_BINDING_COUNT=0 -RESTORE_SYSTEMD_BINDINGS_END=1 -CLOUD_1_DAILY_LSF_RC=3 -CLOUD_1_DAILY_STDERR_SHA256=bb3ce24b6401f914e787d66ae4a3fb5460455a41fab035e6ede56f0af463781f -2026/09/22 08:47:57 ERROR : : error listing: directory not found -2026/09/22 08:47:57 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found -CLOUD_2_DAILY_LSF_RC=3 -CLOUD_2_DAILY_STDERR_SHA256=a3c018181b7d581d2a06e107cb28685794108bc2066f6acc7abdfbc83545fefa -2026/09/22 08:47:58 ERROR : : error listing: directory not found -2026/09/22 08:47:58 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found -CLOUDB_REMOTE_READ_CLASS=INCOMPLETE_OR_ERROR -SOURCE_WORKTREE_VALID=true +2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_TARGET=us-netbird +2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_RESULT=FAIL +2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_REASON=PIPE_FAILED +2026-09-22T08:53:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. +2026-09-22T08:53:31+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +US_NETBIRD_RECENT_JOURNAL_END=1 +BACKUP_V2_TIMER_STATE_BEGIN=1 +MainPID=0 +Result=exit-code +ExecMainStartTimestamp=Tue 2026-09-22 08:53:16 MSK +ExecMainExitTimestamp=Tue 2026-09-22 08:53:31 MSK +ExecMainStatus=2 +ActiveState=failed +SubState=failed +Unit=homelab-backup-v2.service +NextElapseUSecRealtime= +LastTriggerUSec=Tue 2026-09-22 08:53:16 MSK +LoadState=loaded +ActiveState=active +SubState=waiting +UnitFileState=enabled +BACKUP_V2_TIMER_STATE_END=1 +CANONICAL_SKLADCHIK_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json +CANONICAL_SKLADCHIK_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc +CANONICAL_SKLADCHIK_STATE_BEGIN=1 +{"schema":"homelab-cloud-quorum-payload-v1","status":"SEALED","logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","archive_size_bytes":178950,"archive_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","part_count":1,"parts_manifest_sha256":"664ef74ec0f30cd3e7fef848794a8813e390f55295c5c6874c1f30867708f740","cloud_quorum":2,"cloud_a_verified":true,"cloud_b_verified":true,"part_hashes_verified":true,"seal_written":true,"cloud_a_seal_verified":true,"cloud_b_seal_verified":true,"local_payload_present":false,"local_cleanup":"OK","sealed_at_utc":"2026-09-21T17:50:41Z","updated_at_utc":"2026-09-21T17:50:43Z","remote_a_base":"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z","remote_b_base":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z"} +CANONICAL_SKLADCHIK_STATE_END=1 +CANONICAL_SKLADCHIK_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json +CANONICAL_SKLADCHIK_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e +CANONICAL_SKLADCHIK_RESTORE_BEGIN=1 +{"result":"OK","logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","restore_source":"Cloud A","transport_state":"SEALED","payload_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","payload_sha256_verified":true,"payload_decompression_verified":null,"archive_recovery_readback_verified":null,"parts_manifest_verified":null,"bounded_staging":null,"full_local_payload_written":null,"validation_level":null,"validated_at_utc":"2026-09-21T17:50:46Z"} +CANONICAL_SKLADCHIK_RESTORE_END=1 +CANONICAL_SKLADCHIK_STRICT_STATE=true +CANONICAL_SKLADCHIK_STRICT_RESTORE=false +CANONICAL_CLOUDB_MANIFEST_READ_RC=0 +CANONICAL_CLOUDB_MANIFEST_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +CANONICAL_CLOUDB_MANIFEST_SHA256=664ef74ec0f30cd3e7fef848794a8813e390f55295c5c6874c1f30867708f740 +CANONICAL_CLOUDB_MANIFEST_ROWS=1 +CANONICAL_CLOUDB_MANIFEST_TOTAL=178950 +CANONICAL_CLOUDB_MANIFEST_MATCH=true +AUDIT_LEGACY_CLOUDB_CONTRACT_BEGIN=1 + except Exception: pass + if lrc!=0: issues.append({"severity":"CRITICAL","code":"CLOUD_UNREADABLE","logical_id":"cloud-"+label,"root_cause":"RCLONE_LSF_FAILED"}) + cloud[label]=rec + +try: dr=json.loads(DRSEAL.read_text(encoding="utf-8")) +except Exception: dr=None +if not isinstance(dr,dict) or dr.get("status")!="SEALED": + issues.append({"severity":"CRITICAL","code":"DR_SEAL_MISSING","logical_id":"-","root_cause":"SITE_DR_FINAL_SEAL_NOT_SEALED"}) + +cloudb_ok=False +if CLOUDB_HEALTH.is_file(): + txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace") + kv={} + for line in txt.splitlines(): + if "=" in line: + k,v=line.split("=",1); kv[k]=v + cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes") +if not cloudb_ok: + issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"}) + +source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +if hrc==0: source["local_head"]=ho.strip() +frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) +prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) +source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") +if frc==0 and fo.strip(): + lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) + if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] +if source["remote_head"]!=source["local_head"]: + issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) + +status="GREEN" if not issues else "RED" +for t in targets: + print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None)) +for i in issues: +AUDIT_LEGACY_CLOUDB_CONTRACT_END=1 SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614 SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation -SOURCE_PUSHURL_CLASS=DISABLED -SOURCE_REMOTE_HEAD_PROBE_RC=0 -SOURCE_REMOTE_BRANCH_PROBE_RC=0 -SOURCE_REMOTE_ALL_PROBE_RC=0 SOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b SOURCE_REMOTE_BRANCH_HEAD=ABSENT -SOURCE_LOCAL_COMMIT_REMOTE_REF_COUNT=0 +SOURCE_REMOTE_HEAD_RC=0 +SOURCE_REMOTE_BRANCH_RC=0 SOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT -DECISION=HOLD_BACKUP_1115_ONE_OR_MORE_PATHS_UNSAFE -NEXT_GATE=RECONCILE_ONLY_BACKUP_1115_FAILURE -TASK_RESULT=HOLD_HOMELAB_BACKUP_1114_DUE_RESTORE_RCA -CHANGES_MADE_BY_1115=false -PRODUCT_MUTATION_BY_1115=false -VM_MUTATION_BY_1115=false -CLOUD_MUTATION_BY_1115=false -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} -WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_END=1 +US_NETBIRD_FAILURE_CLASS=STATE_FAILURE_DETAIL_PRESENT +DECISION=HOLD_BACKUP_1116_ROOT_CAUSE_INCOMPLETE +NEXT_GATE=RECONCILE_ONLY_BACKUP_1116_FAILURE +TASK_RESULT=HOLD_HOMELAB_BACKUP_1115_ROOT_RCA +CHANGES_MADE_BY_1116=false +PRODUCT_MUTATION_BY_1116=false +VM_MUTATION_BY_1116=false +CLOUD_MUTATION_BY_1116=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1115_ROOT_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END