diff --git a/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-V2-CURRENT-FAILURE-RCA-1221R1.json b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-V2-CURRENT-FAILURE-RCA-1221R1.json new file mode 100644 index 00000000..a272e7d1 --- /dev/null +++ b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-V2-CURRENT-FAILURE-RCA-1221R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260927-HOMELAB-BACKUP-V2-CURRENT-FAILURE-RCA-1221R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-v2-current-failure-rca", + "started_at_utc": "2026-09-27T18:22:59Z", + "finished_at_utc": "2026-09-27T18:23:07Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "d4cfa85a3f59165d51b64505096167a15ce486f1a2105a55cd0c206b16d5353a", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "de7467ab7d2c1584968349e75f34c054b7b406741bd4ab8a3a6e0e559ecf7d2a", + "sanitized_output_sha256": "ebca4b307aeed79780cce791fc44145115b68a88d817904d875aaae99b5dbbef", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "BACKUP_V2_CURRENT_FAILURE_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-V2-CURRENT-FAILURE-RCA-1221R1\nMODE=read-only\nCOMPONENT=homelab-backup-v2-current-failure-rca\nMUTATION_BOUNDARY=NONE_READ_ONLY_NO_SYSTEMD_ACTION_NO_BACKUP_NO_AUDIT_RUN_NO_LOCK_BREAK_NO_RETENTION_NO_DELETE_NO_MAIL_NO_GIT_WRITE_NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED\nREFERENCE_SHA256_EXPECTED=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED\nERROR_REGISTER_SHA256_EXPECTED=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\n=== IMMUTABLE_HISTORY ===\nSOURCE_1211_STATUS=OK\nSOURCE_1211_RC=0\nSOURCE_1211_DECISION=PASS_BACKUP_RPO_ADAPTIVE_CLOSE\nSOURCE_1211_NEXT_GATE=NONE\nSOURCE_1211_TASK_RESULT=PASS_HOMELAB_BACKUP_RPO_ADAPTIVE_CLOSE\nSOURCE_1220_STATUS=OK\nSOURCE_1220_RC=0\nSOURCE_1220_DECISION=PASS_BACKUP_ALERT_DEBOUNCE\nSOURCE_1220_NEXT_GATE=NONE\nSOURCE_73337_STATUS=OK\nSOURCE_73337_DECISION=PASS_NETBIRD_SKLADCHIK_POSTREPAIR_FINAL_ACCEPTANCE\nSOURCE_73337_NEXT_GATE=NONE\nSOURCE_CONTEXT_PRESENT=true\nSOURCE_CONTEXT_BACKUP_V2_FAILED=true\nSOURCE_CONTEXT_BACKUP_AUDIT_FAILED=true\n=== CURRENT_FAILED_UNITS ===\n● homelab-backup-audit.service loaded failed failed Homelab unified backup and DR audit\n● homelab-backup-v2.service loaded failed failed Homelab Backup V2 manifest-driven scheduler\n=== BACKUP_V2_SERVICE_SHOW ===\nResult=exit-code\nExecMainStartTimestamp=Sun 2026-09-27 21:08:52 MSK\nExecMainExitTimestamp=Sun 2026-09-27 21:08:59 MSK\nExecMainCode=1\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-backup-v2 ; argv[]=/usr/local/sbin/homelab-backup-v2 scheduled ; ignore_errors=no ; start_time=[Sun 2026-09-27 21:08:52 MSK] ; stop_time=[Sun 2026-09-27 21:08:59 MSK] ; pid=19346 ; code=exited ; status=1 }\nId=homelab-backup-v2.service\nDescription=Homelab Backup V2 manifest-driven scheduler\nLoadState=loaded\nActiveState=failed\nSubState=failed\nFragmentPath=/etc/systemd/system/homelab-backup-v2.service\nDropInPaths=\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Sun 2026-09-27 21:08:59 MSK\nBACKUP_V2_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service\nBACKUP_V2_UNIT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565\nBACKUP_V2_UNIT_META=644:root:root:2026-09-15 00:28:03.719796025 +0300\n=== BACKUP_V2_SERVICE_CAT ===\n# /etc/systemd/system/homelab-backup-v2.service\n[Unit]\nDescription=Homelab Backup V2 manifest-driven scheduler\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nUMask=0077\nExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nTimeoutStartSec=8h\nBACKUP_V2_EXECSTART_RAW={ path=/usr/local/sbin/homelab-backup-v2 ; argv[]=/usr/local/sbin/homelab-backup-v2 scheduled ; ignore_errors=no ; start_time=[Sun 2026-09-27 21:08:52 MSK] ; stop_time=[Sun 2026-09-27 21:08:59 MSK] ; pid=19346 ; code=exited ; status=1 }\nBACKUP_V2_EXECSTART_PATH=/usr/local/sbin/homelab-backup-v2\nBACKUP_V2_EXECSTART_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nBACKUP_V2_EXECSTART_META=750:root:root:2026-09-17 10:33:39.880371590 +0300\n=== BACKUP_V2_EXECSTART_SOURCE_BEGIN ===\n#!/usr/bin/python3\nfrom __future__ import annotations\nimport hashlib, importlib, os, pathlib, stat, sys\nACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b')\nEXPECTED={'fresh_lxc_helper.py': 'dbb3e6dbb71b10797df736431d9501a752a0f8c439163c9a0bddb7f76747f011', 'producer_exec_helper.py': 'b3b97099bcd7126670f866e81faae6a42a96d869d9989e6a67f6ea6c871c90b3', 'lxc_adapter.py': '3fa7d194010d76f7c8bc95035ae8876262e9337903e310f45f45df88c266dc8d', 'qemu_adapter.py': '1906db42bd34bbe85eb46fd9797b3572d46e0a41c29d1c29be5f9df5e448f15b', 'scheduler_adapter.py': 'a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96', 'entry504.py': 'ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793'}\ndef sha(p):\n h=hashlib.sha256()\n with open(p,'rb',buffering=0) as f:\n while True:\n b=f.read(1048576)\n if not b: break\n h.update(b)\n return h.hexdigest()\ndef main():\n st=os.lstat(ACCEPTED)\n if not stat.S_ISDIR(st.st_mode) or stat.S_ISLNK(st.st_mode) or st.st_uid!=0 or st.st_gid!=0 or stat.S_IMODE(st.st_mode)!=0o500: raise SystemExit('RELEASE_ROOT_IDENTITY')\n names=sorted(p.name for p in ACCEPTED.iterdir())\n if names!=sorted(EXPECTED): raise SystemExit('RELEASE_FILESET')\n for name,want in EXPECTED.items():\n p=ACCEPTED/name; s=os.lstat(p)\n if not stat.S_ISREG(s.st_mode) or stat.S_ISLNK(s.st_mode) or s.st_uid!=0 or s.st_gid!=0 or stat.S_IMODE(s.st_mode)!=0o400 or sha(p)!=want: raise SystemExit('RELEASE_FILE_IDENTITY_'+name)\n sys.dont_write_bytecode=True\n sys.path.insert(0,str(ACCEPTED))\n e=importlib.import_module('entry504')\n return int(e.main())\nif __name__=='__main__': raise SystemExit(main())\n=== BACKUP_V2_EXECSTART_SOURCE_END ===\n=== BACKUP_V2_TIMER_SHOW ===\nNextElapseUSecRealtime=\nLastTriggerUSec=Sun 2026-09-27 21:08:52 MSK\nResult=success\nId=homelab-backup-v2.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nFragmentPath=/etc/systemd/system/homelab-backup-v2.timer\nDropInPaths=\nUnitFileState=enabled\nBACKUP_V2_TIMER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.timer\nBACKUP_V2_TIMER_SHA256=5469057602452b303a01ccb8057085c1b0e447973e5ed198cebf7dd5daf58e9d\n=== BACKUP_V2_TIMER_CAT ===\n# /etc/systemd/system/homelab-backup-v2.timer\n[Unit]\nDescription=Homelab Backup V2 scheduler timer\n\n[Timer]\nOnActiveSec=20min\nOnUnitActiveSec=30min\nRandomizedDelaySec=10min\nUnit=homelab-backup-v2.service\n\n[Install]\nWantedBy=timers.target\n=== BACKUP_AUDIT_SERVICE_SHOW ===\nResult=exit-code\nExecMainStartTimestamp=Sun 2026-09-27 21:05:58 MSK\nExecMainExitTimestamp=Sun 2026-09-27 21:06:02 MSK\nExecMainCode=1\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-backup-audit ; argv[]=/usr/local/sbin/homelab-backup-audit scheduled ; ignore_errors=no ; start_time=[Sun 2026-09-27 21:05:58 MSK] ; stop_time=[Sun 2026-09-27 21:06:02 MSK] ; pid=16838 ; code=exited ; status=1 }\nId=homelab-backup-audit.service\nDescription=Homelab unified backup and DR audit\nLoadState=loaded\nActiveState=failed\nSubState=failed\nFragmentPath=/etc/systemd/system/homelab-backup-audit.service\nDropInPaths=\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Sun 2026-09-27 21:06:02 MSK\nBACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service\nBACKUP_AUDIT_UNIT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa\nBACKUP_AUDIT_UNIT_META=644:root:root:2026-08-11 08:46:44.076278347 +0300\n=== BACKUP_AUDIT_SERVICE_CAT ===\n# /etc/systemd/system/homelab-backup-audit.service\n[Unit]\nDescription=Homelab unified backup and DR audit\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-audit scheduled\nUser=root\nGroup=root\nUMask=0077\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nBACKUP_AUDIT_EXECSTART_RAW={ path=/usr/local/sbin/homelab-backup-audit ; argv[]=/usr/local/sbin/homelab-backup-audit scheduled ; ignore_errors=no ; start_time=[Sun 2026-09-27 21:05:58 MSK] ; stop_time=[Sun 2026-09-27 21:06:02 MSK] ; pid=16838 ; code=exited ; status=1 }\nBACKUP_AUDIT_EXECSTART_PATH=/usr/local/sbin/homelab-backup-audit\nBACKUP_AUDIT_EXECSTART_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nBACKUP_AUDIT_EXECSTART_META=755:root:root:2026-09-22 12:19:10.163905297 +0300\n=== BACKUP_AUDIT_EXECSTART_SOURCE_BEGIN ===\n#!/usr/bin/env python3\nimport sys,os,json,time,subprocess,pathlib,re,datetime\nMODE=sys.argv[1] if len(sys.argv)>1 else \"all\"\nif MODE not in (\"all\",\"write-health\",\"scheduled\",\"contract\"):\n print(\"AUDIT_ERROR=USAGE_all_write-health_scheduled_contract\",file=sys.stderr); sys.exit(64)\nif MODE==\"contract\":\n print(\"AUDIT_CONTRACT=v2-plus-sealed-quorum\")\n print(\"AUDIT_SCHEMA=homelab-backup-audit-v3\")\n sys.exit(0)\n\nREG=pathlib.Path(\"/etc/homelab-backup/targets.json\")\nSTATE=pathlib.Path(\"/var/lib/homelab-backup-v2/state\")\nQROOT=pathlib.Path(\"/var/lib/homelab-backup/cloud-quorum\")\nVMREST=pathlib.Path(\"/var/lib/homelab-backup/vm-restore-validation\")\nAPPREST=pathlib.Path(\"/var/lib/homelab-backup/app-restore-validation\")\nREGH=\"/usr/local/libexec/homelab-backup-target-registry\"\nAPPQ=\"/usr/local/sbin/homelab-app-cloud-quorum-queue\"\nGFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nGFSP=\"/etc/homelab-backup/gfs-retention.json\"\nRET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nWT=\"/root/homelab-ops-cr-2026-1005\"\nBRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nDRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nCLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nHEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\n\ndef run(cmd,timeout=60,env=None):\n try:\n p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)\n return p.returncode,p.stdout,p.stderr\n except subprocess.TimeoutExpired as e:\n return 124,e.stdout or \"\",e.stderr or \"\"\n\ndef collect(obj,out):\n if isinstance(obj,dict):\n if isinstance(obj.get(\"logical_id\"),str): out.append(obj)\n for v in obj.values(): collect(v,out)\n elif isinstance(obj,list):\n for v in obj: collect(v,out)\n\ndef parse_epoch(obj):\n for k in (\"sealed_epoch\",\"last_success_epoch\",\"updated_epoch\"):\n v=obj.get(k)\n if isinstance(v,(int,float)) and v>0: return int(v)\n for k in (\"sealed_at_utc\",\"updated_at_utc\",\"completed_at_utc\"):\n v=obj.get(k)\n if isinstance(v,str) and v:\n try:\n return int(datetime.datetime.fromisoformat(v.replace(\"Z\",\"+00:00\")).timestamp())\n except Exception: pass\n return None\n\ndef latest_legacy(logical,kind):\n d=QROOT/logical\n best=None; best_epoch=-1\n if not d.is_dir(): return None\n for p in d.glob(\"*.json\"):\n try: obj=json.loads(p.read_text(encoding=\"utf-8\"))\n except Exception: continue\n epoch=parse_epoch(obj)\n score=epoch if epoch is not None else int(p.stat().st_mtime)\n if score>best_epoch:\n best_epoch=score; best=(p,obj,score)\n if not best: return None\n p,obj,epoch=best\n gen=obj.get(\"generation\")\n strict=(obj.get(\"status\")==\"SEALED\" and obj.get(\"cloud_quorum\")==2 and obj.get(\"cloud_a_verified\") is True and obj.get(\"cloud_b_verified\") is True and obj.get(\"part_hashes_verified\") is True and obj.get(\"seal_written\") is True and obj.get(\"local_payload_present\") is False and obj.get(\"local_cleanup\")==\"OK\")\n rroot=VMREST if kind==\"VM\" else APPREST\n rev=(rroot/logical/(str(gen)+\".json\")) if gen else None\n restore=False\n if rev and rev.is_file():\n try:\n rj=json.loads(rev.read_text(encoding=\"utf-8\"))\n restore=(rj.get(\"result\")==\"OK\")\n except Exception: pass\n age=max(0,int(time.time()-epoch)) if epoch else None\n return {\"generation\":gen,\"strict\":strict,\"restore_ok\":restore,\"age_seconds\":age,\"state_file\":str(p),\"obj\":obj}\n\ndef unit_ok(unit):\n a=run([\"systemctl\",\"is-active\",unit],10)[1].strip()\n e=run([\"systemctl\",\"is-enabled\",unit],10)[1].strip()\n return a==\"active\" and e==\"enabled\",a,e\n\ndef parse_app_plan(txt):\n d={}\n for line in txt.splitlines():\n if line.startswith(\"QUEUE_FRESHNESS_PLAN APP=\"):\n r={}\n for tok in line.split()[1:]:\n if \"=\" in tok:\n k,v=tok.split(\"=\",1); r[k]=v\n if r.get(\"APP\"): d[r[\"APP\"]]=r.get(\"DECISION\")\n return d\n\nissues=[]; targets=[]; now=int(time.time())\ntry: robj=json.loads(REG.read_text(encoding=\"utf-8\"))\nexcept Exception:\n print(\"AUDIT_ERROR=REGISTRY_UNREADABLE\",file=sys.stderr); sys.exit(69)\nrows=[]; collect(robj,rows)\nactive={}\nfor r in rows:\n lid=r.get(\"logical_id\")\n if lid and str(r.get(\"lifecycle\",\"\")).upper()==\"ACTIVE\":\n active[lid]={\"logical_id\":lid,\"kind\":str(r.get(\"kind\",\"\")).upper(),\"id\":str(r.get(\"id\")) if r.get(\"id\") is not None else None}\n\nvmrc,vmo,_=run([REGH,\"freshness-seconds\",\"VM\"],20)\naprc,apo,_=run([REGH,\"freshness-seconds\",\"APP\"],20)\nvm_sla=int(vmo.strip()) if vmrc==0 and vmo.strip().isdigit() else 72000\napp_sla=int(apo.strip()) if aprc==0 and apo.strip().isdigit() else 72000\n\nrrc,rout,_=run([\"pvesh\",\"get\",\"/cluster/resources\",\"--type\",\"vm\",\"--output-format\",\"json\"],30)\nresources={}\nif rrc==0:\n try:\n for x in json.loads(rout): resources[str(x.get(\"vmid\"))]=x\n except Exception: pass\n\nqrc,qout,qerr=run([APPQ,\"freshness-plan\"],180)\napp_dec=parse_app_plan(qout if qrc==0 else \"\")\nif qrc!=0: issues.append({\"severity\":\"CRITICAL\",\"code\":\"APP_FRESHNESS_PLAN_FAILED\",\"logical_id\":\"-\",\"root_cause\":\"APP_QUEUE_RC_\"+str(qrc)})\n\nfor lid,r in sorted(active.items()):\n kind=r[\"kind\"]; ident=r[\"id\"]; rec={\"logical_id\":lid,\"kind\":kind,\"state\":\"UNVERIFIED\"}\n if kind==\"VM\":\n typ=str(resources.get(ident,{}).get(\"type\",\"qemu\"))\n sid=(\"ct\"+ident) if typ==\"lxc\" else (\"vm\"+ident)\n v2=None\n try: v2=json.loads((STATE/(sid+\".json\")).read_text(encoding=\"utf-8\"))\n except Exception: pass\n v2_ok=False; v2_age=None; repcount=None\n if isinstance(v2,dict):\n ls=v2.get(\"last_success_epoch\")\n v2_age=max(0,int(now-ls)) if isinstance(ls,(int,float)) and ls>0 else None\n reps=v2.get(\"replicas\"); repcount=len(reps) if isinstance(reps,(list,dict)) else None\n v2_ok=(v2.get(\"result\")==\"SUCCESS\" and v2_age is not None and v2_age=3:\n issues.append({\"severity\":\"HIGH\",\"code\":\"RETRY_EXHAUSTION\",\"logical_id\":lid,\"root_cause\":\"CONSECUTIVE_FAILURES_\"+str(cf)})\n legacy=latest_legacy(lid,\"VM\")\n legacy_ok=bool(legacy and legacy[\"strict\"] and legacy[\"restore_ok\"] and legacy[\"age_seconds\"] is not None and legacy[\"age_seconds\"]0 else None\n fresh=(obj.get(\"result\")==\"SUCCESS\" and age is not None and age<86400)\n rec.update({\"state\":\"FRESH\" if fresh else \"STALE\",\"age_seconds\":age,\"generation\":obj.get(\"generation\") if isinstance(obj,dict) else None})\n if not fresh: issues.append({\"severity\":\"HIGH\",\"code\":\"RPO_BREACH\",\"logical_id\":lid,\"root_cause\":\"CONTROL_STATE_NOT_FRESH\"})\n targets.append(rec)\n\nif len(targets)!=len(active):\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"LIFECYCLE_COVERAGE_GAP\",\"logical_id\":\"-\",\"root_cause\":\"ACTIVE_TARGET_COUNT_MISMATCH\"})\n\nfor unit,code in [\n (\"homelab-backup-v2.timer\",\"V2_TIMER_BAD\"),\n (\"homelab-app-cloud-quorum-queue.timer\",\"APP_TIMER_BAD\"),\n (\"homelab-backup-retention-gc.timer\",\"RETENTION_TIMER_BAD\"),\n (\"homelab-alertmanager-backup-health.timer\",\"ALERT_TIMER_BAD\"),\n]:\n ok,a,e=unit_ok(unit)\n if not ok: issues.append({\"severity\":\"HIGH\",\"code\":code,\"logical_id\":unit,\"root_cause\":\"UNIT_\"+a+\"_\"+e})\n\nfor lock in [\"/run/homelab-backup-heavy.lock\",\"/run/lock/homelab-app-cloud-quorum-queue.lock\",\"/run/lock/homelab-vm-cloud-quorum-queue.lock\",\"/run/lock/homelab-backup-dispatcher.lock\"]:\n p=pathlib.Path(lock)\n if p.exists():\n lrc,_,_=run([\"flock\",\"-n\",str(p),\"-c\",\"true\"],5)\n if lrc!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"LOCK_BUSY\",\"logical_id\":lock,\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\"})\n\nenv=os.environ.copy(); env[\"HOMELAB_BACKUP_GFS_POLICY\"]=GFSP; env[\"HOMELAB_BACKUP_TARGET_REGISTRY\"]=str(REG)\ngrc,gout,_=run([GFSH,\"validate\"],20,env)\nif grc!=0 or \"GFS_POLICY=OK\" not in gout:\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"GFS_INVALID\",\"logical_id\":\"-\",\"root_cause\":\"GFS_VALIDATE_FAILED\"})\nrr,_,_=run([RET,\"contract\"],120)\nif rr!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_CONTRACT_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_CONTRACT_RC_\"+str(rr)})\nrhp=pathlib.Path(\"/var/lib/homelab-health/backup-retention-safe-gc.json\")\nif not rhp.is_file():\n issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_FILE_MISSING\"})\nelse:\n try:\n rh=json.loads(rhp.read_text(encoding=\"utf-8\"))\n if rh.get(\"status\")!=\"OK\": issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_NOT_OK\"})\n except Exception: issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_UNREADABLE\"})\n\ncloud={}\nfor label,remote in [(\"A\",\"pve01-mail-01-crypt:homelab-backups\"),(\"B\",\"pve01-mail-02-crypt:homelab-backups\")]:\n lrc,lo,_=run([\"rclone\",\"lsf\",remote,\"--max-depth\",\"1\",\"--dirs-only\"],60)\n arc,ao,_=run([\"rclone\",\"about\",remote,\"--json\"],60)\n rec={\"readable\":lrc==0,\"top_dirs\":len([x for x in lo.splitlines() if x.strip()])}\n if arc==0:\n try:\n j=json.loads(ao); rec.update({k:j.get(k) for k in (\"total\",\"used\",\"free\")})\n if j.get(\"total\") and j.get(\"free\") is not None:\n frac=j[\"free\"]/j[\"total\"]; rec[\"free_fraction\"]=round(frac,6)\n if frac<0.15: issues.append({\"severity\":\"HIGH\",\"code\":\"CAPACITY_LOW\",\"logical_id\":\"cloud-\"+label,\"root_cause\":\"FREE_LT_15_PERCENT\"})\n except Exception: pass\n if lrc!=0: issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_UNREADABLE\",\"logical_id\":\"cloud-\"+label,\"root_cause\":\"RCLONE_LSF_FAILED\"})\n cloud[label]=rec\n\ntry: dr=json.loads(DRSEAL.read_text(encoding=\"utf-8\"))\nexcept Exception: dr=None\nif not isinstance(dr,dict) or dr.get(\"status\")!=\"SEALED\":\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"DR_SEAL_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"SITE_DR_FINAL_SEAL_NOT_SEALED\"})\n\ncloudb_ok=False\ntry:\n import pathlib as _pathlib, json as _json, subprocess as _subprocess, hashlib as _hashlib\n _state_root=_pathlib.Path(\"/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor\")\n _restore_root=_pathlib.Path(\"/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor\")\n _states=sorted([p for p in _state_root.glob(\"*.json\") if p.is_file() and not p.is_symlink()],key=lambda p:p.stat().st_mtime,reverse=True)\n for _sp in _states:\n try: _st=_json.loads(_sp.read_text(encoding=\"utf-8\"))\n except Exception: continue\n if not (\n _st.get(\"schema\")==\"homelab-cloud-quorum-payload-v1\" and\n _st.get(\"logical_id\")==\"app-skladchik-reports-monitor\" and\n _st.get(\"status\")==\"SEALED\" and _st.get(\"cloud_quorum\")==2 and\n _st.get(\"cloud_a_verified\") is True and _st.get(\"cloud_b_verified\") is True and\n _st.get(\"part_hashes_verified\") is True and _st.get(\"seal_written\") is True and\n _st.get(\"cloud_a_seal_verified\") is True and _st.get(\"cloud_b_seal_verified\") is True and\n _st.get(\"local_payload_present\") is False and _st.get(\"local_cleanup\")==\"OK\"):\n continue\n _gen=_st.get(\"generation\"); _archive_sha=_st.get(\"archive_sha256\")\n _manifest_sha=_st.get(\"parts_manifest_sha256\"); _part_count=_st.get(\"part_count\")\n _archive_size=_st.get(\"archive_size_bytes\"); _remote_b=_st.get(\"remote_b_base\")\n if not (isinstance(_gen,str) and isinstance(_archive_sha,str) and len(_archive_sha)==64 and\n isinstance(_manifest_sha,str) and len(_manifest_sha)==64 and isinstance(_part_count,int) and\n _part_count>0 and isinstance(_archive_size,int) and _archive_size>0 and isinstance(_remote_b,str)):\n continue\n _evp=_restore_root/(_gen+\".json\")\n try: _ev=_json.loads(_evp.read_text(encoding=\"utf-8\"))\n except Exception: continue\n if not (\n _ev.get(\"schema\")==\"homelab-app-restore-validation-v1\" and\n _ev.get(\"logical_id\")==\"app-skladchik-reports-monitor\" and\n _ev.get(\"generation\")==_gen and _ev.get(\"transport_state\")==\"SEALED\" and\n _ev.get(\"restore_source\")==\"Cloud A\" and _ev.get(\"payload_sha256\")==_archive_sha and\n _ev.get(\"payload_sha256_verified\") is True and _ev.get(\"result\")==\"OK\"):\n continue\n _mp=_subprocess.run([\"rclone\",\"cat\",_remote_b.rstrip(\"/\")+\"/PARTS.tsv\"],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=60)\n if _mp.returncode!=0 or _hashlib.sha256(_mp.stdout).hexdigest()!=_manifest_sha: continue\n _rows=[]; _valid=True\n try:\n for _line in _mp.stdout.decode(\"utf-8\",\"strict\").splitlines():\n if not _line: continue\n _f=_line.split(\"\\t\")\n if len(_f)!=4: _valid=False; break\n _idx,_name,_size_s,_psha=_f\n if not (_idx.isdigit() and len(_idx)==8 and _size_s.isdigit() and len(_psha)==64):\n _valid=False; break\n _size=int(_size_s)\n _pp=_subprocess.run([\"rclone\",\"cat\",_remote_b.rstrip(\"/\")+\"/payload.parts/\"+_name],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=120)\n if _pp.returncode!=0 or len(_pp.stdout)!=_size or _hashlib.sha256(_pp.stdout).hexdigest()!=_psha:\n _valid=False; break\n _rows.append((_size,_pp.stdout))\n except Exception:\n _valid=False\n if not _valid or len(_rows)!=_part_count or sum(x[0] for x in _rows)!=_archive_size: continue\n _payload=b\"\".join(x[1] for x in _rows)\n if _hashlib.sha256(_payload).hexdigest()!=_archive_sha: continue\n cloudb_ok=True\n break\nexcept Exception:\n cloudb_ok=False\nif not cloudb_ok:\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK\"})\n\nsource={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\nhrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\nif hrc==0: source[\"local_head\"]=ho.strip()\nfrc,fo,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.url\"],10)\nprc,po,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.pushurl\"],10)\nsource[\"pushurl_disabled\"]=(prc==0 and po.strip()==\"DISABLED\")\nif frc==0 and fo.strip():\n lrc,lo,_=run([\"git\",\"ls-remote\",fo.strip(),\"refs/heads/\"+BRANCH],30)\n if lrc==0 and lo.strip(): source[\"remote_head\"]=lo.split()[0]\nif source[\"remote_head\"]!=source[\"local_head\"]:\n issues.append({\"severity\":\"MEDIUM\",\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\"})\n\nstatus=\"GREEN\" if not issues else \"RED\"\nfor t in targets:\n print(\"TARGET|\"+\"|\".join(f\"{k}={v}\" for k,v in t.items() if v is not None))\nfor i in issues:\n print(\"ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}\".format(**i))\nprint(f\"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}\")\nhealth={\"schema\":\"homelab-backup-audit-v3\",\"status\":status,\"generated_at_epoch\":now,\"targets\":targets,\"issues\":issues,\"cloud\":cloud,\"source\":source,\"cloud_b_restore_drill\":cloudb_ok,\"dr_proven\":bool(isinstance(dr,dict) and dr.get(\"status\")==\"SEALED\")}\nif MODE in (\"write-health\",\"scheduled\"):\n for hp in HEALTHS:\n hp.parent.mkdir(parents=True,exist_ok=True)\n tmp=hp.with_name(hp.name+\".tmp\")\n tmp.write_text(json.dumps(health,separators=(\",\",\":\"),sort_keys=True)+\"\\n\",encoding=\"utf-8\")\n os.chmod(tmp,0o644); os.replace(tmp,hp)\nsys.exit(0 if not issues else 1)\n=== BACKUP_AUDIT_EXECSTART_SOURCE_END ===\n=== BACKUP_AUDIT_TIMER_SHOW ===\nNextElapseUSecRealtime=Sun 2026-09-27 21:36:58 MSK\nLastTriggerUSec=Sun 2026-09-27 21:05:58 MSK\nResult=success\nId=homelab-backup-audit.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nFragmentPath=/etc/systemd/system/homelab-backup-audit.timer\nDropInPaths=\nUnitFileState=enabled\nBACKUP_AUDIT_TIMER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.timer\nBACKUP_AUDIT_TIMER_SHA256=987695f83ee993bd4d716b8365308211110ac4c684f0bd9b17dfdb1e523e466a\n=== BACKUP_AUDIT_TIMER_CAT ===\n# /etc/systemd/system/homelab-backup-audit.timer\n[Unit]\nDescription=Run homelab unified backup and DR audit\n\n[Timer]\nOnCalendar=*-*-* *:05,35:00\nRandomizedDelaySec=2min\nPersistent=true\nAccuracySec=1min\nUnit=homelab-backup-audit.service\n\n[Install]\nWantedBy=timers.target\n=== BACKUP_V2_JOURNAL_LAST12H ===\n2026-09-27T16:56:08.780555+03:00 pve01 homelab-backup-v2[3412388]: lxc_adapter.AdapterError: FRESH_HELPER_FAILED\n2026-09-27T16:56:08.785999+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T16:56:08.786073+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\n2026-09-27T16:56:08.786211+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-27T16:56:08.786299+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 7.475s CPU time, 159M memory peak.\n2026-09-27T17:32:01.150519+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-27T17:32:01.367425+03:00 pve01 homelab-backup-v2[3534166]: SELFTEST_REPLACEMENT486=PASS\n2026-09-27T17:32:08.781016+03:00 pve01 homelab-backup-v2[3534166]: SCHEDULE_TARGET=ct110\n2026-09-27T17:32:08.788194+03:00 pve01 homelab-backup-v2[3534166]: Traceback (most recent call last):\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/usr/local/sbin/homelab-backup-v2\", line 26, in \n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return int(e.main())\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try:return legacy.scheduled_run()\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try: backup_target(t,green_verify=False)\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return _legacy_backup_target(t,green_verify)\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return int(e.main())\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try:return legacy.scheduled_run()\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try: backup_target(t,green_verify=False)\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return _legacy_backup_target(t,green_verify)\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return int(e.main())\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try:return legacy.scheduled_run()\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try: backup_target(t,green_verify=False)\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return _legacy_backup_target(t,green_verify)\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return int(e.main())\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try:return legacy.scheduled_run()\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try: backup_target(t,green_verify=False)\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return _legacy_backup_target(t,green_verify)\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return int(e.main())\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try:return legacy.scheduled_run()\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try: backup_target(t,green_verify=False)\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return _legacy_backup_target(t,green_verify)\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return int(e.main())\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try:return legacy.scheduled_run()\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try: backup_target(t,green_verify=False)\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return _legacy_backup_target(t,green_verify)\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return int(e.main())\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try:return legacy.scheduled_run()\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try: backup_target(t,green_verify=False)\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return _legacy_backup_target(t,green_verify)\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: _need(p.returncode==0 and not p.stderr and 01 else "all" +if MODE not in ("all","write-health","scheduled","contract"): + print("AUDIT_ERROR=USAGE_all_write-health_scheduled_contract",file=sys.stderr); sys.exit(64) +if MODE=="contract": + print("AUDIT_CONTRACT=v2-plus-sealed-quorum") + print("AUDIT_SCHEMA=homelab-backup-audit-v3") + sys.exit(0) + +REG=pathlib.Path("/etc/homelab-backup/targets.json") +STATE=pathlib.Path("/var/lib/homelab-backup-v2/state") +QROOT=pathlib.Path("/var/lib/homelab-backup/cloud-quorum") +VMREST=pathlib.Path("/var/lib/homelab-backup/vm-restore-validation") +APPREST=pathlib.Path("/var/lib/homelab-backup/app-restore-validation") +REGH="/usr/local/libexec/homelab-backup-target-registry" +APPQ="/usr/local/sbin/homelab-app-cloud-quorum-queue" +GFSH="/usr/local/libexec/homelab-backup-gfs-policy" +GFSP="/etc/homelab-backup/gfs-retention.json" +RET="/usr/local/sbin/homelab-backup-retention-gc" +WT="/root/homelab-ops-cr-2026-1005" +BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" +DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") +CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] + +def run(cmd,timeout=60,env=None): + try: + p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env) + return p.returncode,p.stdout,p.stderr + except subprocess.TimeoutExpired as e: + return 124,e.stdout or "",e.stderr or "" + +def collect(obj,out): + if isinstance(obj,dict): + if isinstance(obj.get("logical_id"),str): out.append(obj) + for v in obj.values(): collect(v,out) + elif isinstance(obj,list): + for v in obj: collect(v,out) + +def parse_epoch(obj): + for k in ("sealed_epoch","last_success_epoch","updated_epoch"): + v=obj.get(k) + if isinstance(v,(int,float)) and v>0: return int(v) + for k in ("sealed_at_utc","updated_at_utc","completed_at_utc"): + v=obj.get(k) + if isinstance(v,str) and v: + try: + return int(datetime.datetime.fromisoformat(v.replace("Z","+00:00")).timestamp()) + except Exception: pass + return None + +def latest_legacy(logical,kind): + d=QROOT/logical + best=None; best_epoch=-1 + if not d.is_dir(): return None + for p in d.glob("*.json"): + try: obj=json.loads(p.read_text(encoding="utf-8")) + except Exception: continue + epoch=parse_epoch(obj) + score=epoch if epoch is not None else int(p.stat().st_mtime) + if score>best_epoch: + best_epoch=score; best=(p,obj,score) + if not best: return None + p,obj,epoch=best + gen=obj.get("generation") + strict=(obj.get("status")=="SEALED" and obj.get("cloud_quorum")==2 and obj.get("cloud_a_verified") is True and obj.get("cloud_b_verified") is True and obj.get("part_hashes_verified") is True and obj.get("seal_written") is True and obj.get("local_payload_present") is False and obj.get("local_cleanup")=="OK") + rroot=VMREST if kind=="VM" else APPREST + rev=(rroot/logical/(str(gen)+".json")) if gen else None + restore=False + if rev and rev.is_file(): + try: + rj=json.loads(rev.read_text(encoding="utf-8")) + restore=(rj.get("result")=="OK") + except Exception: pass + age=max(0,int(time.time()-epoch)) if epoch else None + return {"generation":gen,"strict":strict,"restore_ok":restore,"age_seconds":age,"state_file":str(p),"obj":obj} + +def unit_ok(unit): + a=run(["systemctl","is-active",unit],10)[1].strip() + e=run(["systemctl","is-enabled",unit],10)[1].strip() + return a=="active" and e=="enabled",a,e + +def parse_app_plan(txt): + d={} + for line in txt.splitlines(): + if line.startswith("QUEUE_FRESHNESS_PLAN APP="): + r={} + for tok in line.split()[1:]: + if "=" in tok: + k,v=tok.split("=",1); r[k]=v + if r.get("APP"): d[r["APP"]]=r.get("DECISION") + return d + +issues=[]; targets=[]; now=int(time.time()) +try: robj=json.loads(REG.read_text(encoding="utf-8")) +except Exception: + print("AUDIT_ERROR=REGISTRY_UNREADABLE",file=sys.stderr); sys.exit(69) +rows=[]; collect(robj,rows) +active={} +for r in rows: + lid=r.get("logical_id") + if lid and str(r.get("lifecycle","")).upper()=="ACTIVE": + active[lid]={"logical_id":lid,"kind":str(r.get("kind","")).upper(),"id":str(r.get("id")) if r.get("id") is not None else None} + +vmrc,vmo,_=run([REGH,"freshness-seconds","VM"],20) +aprc,apo,_=run([REGH,"freshness-seconds","APP"],20) +vm_sla=int(vmo.strip()) if vmrc==0 and vmo.strip().isdigit() else 72000 +app_sla=int(apo.strip()) if aprc==0 and apo.strip().isdigit() else 72000 + +rrc,rout,_=run(["pvesh","get","/cluster/resources","--type","vm","--output-format","json"],30) +resources={} +if rrc==0: + try: + for x in json.loads(rout): resources[str(x.get("vmid"))]=x + except Exception: pass + +qrc,qout,qerr=run([APPQ,"freshness-plan"],180) +app_dec=parse_app_plan(qout if qrc==0 else "") +if qrc!=0: issues.append({"severity":"CRITICAL","code":"APP_FRESHNESS_PLAN_FAILED","logical_id":"-","root_cause":"APP_QUEUE_RC_"+str(qrc)}) + +for lid,r in sorted(active.items()): + kind=r["kind"]; ident=r["id"]; rec={"logical_id":lid,"kind":kind,"state":"UNVERIFIED"} + if kind=="VM": + typ=str(resources.get(ident,{}).get("type","qemu")) + sid=("ct"+ident) if typ=="lxc" else ("vm"+ident) + v2=None + try: v2=json.loads((STATE/(sid+".json")).read_text(encoding="utf-8")) + except Exception: pass + v2_ok=False; v2_age=None; repcount=None + if isinstance(v2,dict): + ls=v2.get("last_success_epoch") + v2_age=max(0,int(now-ls)) if isinstance(ls,(int,float)) and ls>0 else None + reps=v2.get("replicas"); repcount=len(reps) if isinstance(reps,(list,dict)) else None + v2_ok=(v2.get("result")=="SUCCESS" and v2_age is not None and v2_age=3: + issues.append({"severity":"HIGH","code":"RETRY_EXHAUSTION","logical_id":lid,"root_cause":"CONSECUTIVE_FAILURES_"+str(cf)}) + legacy=latest_legacy(lid,"VM") + legacy_ok=bool(legacy and legacy["strict"] and legacy["restore_ok"] and legacy["age_seconds"] is not None and legacy["age_seconds"]0 else None + fresh=(obj.get("result")=="SUCCESS" and age is not None and age<86400) + rec.update({"state":"FRESH" if fresh else "STALE","age_seconds":age,"generation":obj.get("generation") if isinstance(obj,dict) else None}) + if not fresh: issues.append({"severity":"HIGH","code":"RPO_BREACH","logical_id":lid,"root_cause":"CONTROL_STATE_NOT_FRESH"}) + targets.append(rec) + +if len(targets)!=len(active): + issues.append({"severity":"CRITICAL","code":"LIFECYCLE_COVERAGE_GAP","logical_id":"-","root_cause":"ACTIVE_TARGET_COUNT_MISMATCH"}) + +for unit,code in [ + ("homelab-backup-v2.timer","V2_TIMER_BAD"), + ("homelab-app-cloud-quorum-queue.timer","APP_TIMER_BAD"), + ("homelab-backup-retention-gc.timer","RETENTION_TIMER_BAD"), + ("homelab-alertmanager-backup-health.timer","ALERT_TIMER_BAD"), +]: + ok,a,e=unit_ok(unit) + if not ok: issues.append({"severity":"HIGH","code":code,"logical_id":unit,"root_cause":"UNIT_"+a+"_"+e}) + +for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]: + p=pathlib.Path(lock) + if p.exists(): + lrc,_,_=run(["flock","-n",str(p),"-c","true"],5) + if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"}) + +env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG) +grc,gout,_=run([GFSH,"validate"],20,env) +if grc!=0 or "GFS_POLICY=OK" not in gout: + issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"}) +rr,_,_=run([RET,"contract"],120) +if rr!=0: issues.append({"severity":"HIGH","code":"RETENTION_CONTRACT_BAD","logical_id":"-","root_cause":"RETENTION_CONTRACT_RC_"+str(rr)}) +rhp=pathlib.Path("/var/lib/homelab-health/backup-retention-safe-gc.json") +if not rhp.is_file(): + issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_MISSING","logical_id":"-","root_cause":"RETENTION_HEALTH_FILE_MISSING"}) +else: + try: + rh=json.loads(rhp.read_text(encoding="utf-8")) + if rh.get("status")!="OK": issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_NOT_OK"}) + except Exception: issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_UNREADABLE"}) + +cloud={} +for label,remote in [("A","pve01-mail-01-crypt:homelab-backups"),("B","pve01-mail-02-crypt:homelab-backups")]: + lrc,lo,_=run(["rclone","lsf",remote,"--max-depth","1","--dirs-only"],60) + arc,ao,_=run(["rclone","about",remote,"--json"],60) + rec={"readable":lrc==0,"top_dirs":len([x for x in lo.splitlines() if x.strip()])} + if arc==0: + try: + j=json.loads(ao); rec.update({k:j.get(k) for k in ("total","used","free")}) + if j.get("total") and j.get("free") is not None: + frac=j["free"]/j["total"]; rec["free_fraction"]=round(frac,6) + if frac<0.15: issues.append({"severity":"HIGH","code":"CAPACITY_LOW","logical_id":"cloud-"+label,"root_cause":"FREE_LT_15_PERCENT"}) + except Exception: pass + if lrc!=0: issues.append({"severity":"CRITICAL","code":"CLOUD_UNREADABLE","logical_id":"cloud-"+label,"root_cause":"RCLONE_LSF_FAILED"}) + cloud[label]=rec + +try: dr=json.loads(DRSEAL.read_text(encoding="utf-8")) +except Exception: dr=None +if not isinstance(dr,dict) or dr.get("status")!="SEALED": + issues.append({"severity":"CRITICAL","code":"DR_SEAL_MISSING","logical_id":"-","root_cause":"SITE_DR_FINAL_SEAL_NOT_SEALED"}) + +cloudb_ok=False +try: + import pathlib as _pathlib, json as _json, subprocess as _subprocess, hashlib as _hashlib + _state_root=_pathlib.Path("/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor") + _restore_root=_pathlib.Path("/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor") + _states=sorted([p for p in _state_root.glob("*.json") if p.is_file() and not p.is_symlink()],key=lambda p:p.stat().st_mtime,reverse=True) + for _sp in _states: + try: _st=_json.loads(_sp.read_text(encoding="utf-8")) + except Exception: continue + if not ( + _st.get("schema")=="homelab-cloud-quorum-payload-v1" and + _st.get("logical_id")=="app-skladchik-reports-monitor" and + _st.get("status")=="SEALED" and _st.get("cloud_quorum")==2 and + _st.get("cloud_a_verified") is True and _st.get("cloud_b_verified") is True and + _st.get("part_hashes_verified") is True and _st.get("seal_written") is True and + _st.get("cloud_a_seal_verified") is True and _st.get("cloud_b_seal_verified") is True and + _st.get("local_payload_present") is False and _st.get("local_cleanup")=="OK"): + continue + _gen=_st.get("generation"); _archive_sha=_st.get("archive_sha256") + _manifest_sha=_st.get("parts_manifest_sha256"); _part_count=_st.get("part_count") + _archive_size=_st.get("archive_size_bytes"); _remote_b=_st.get("remote_b_base") + if not (isinstance(_gen,str) and isinstance(_archive_sha,str) and len(_archive_sha)==64 and + isinstance(_manifest_sha,str) and len(_manifest_sha)==64 and isinstance(_part_count,int) and + _part_count>0 and isinstance(_archive_size,int) and _archive_size>0 and isinstance(_remote_b,str)): + continue + _evp=_restore_root/(_gen+".json") + try: _ev=_json.loads(_evp.read_text(encoding="utf-8")) + except Exception: continue + if not ( + _ev.get("schema")=="homelab-app-restore-validation-v1" and + _ev.get("logical_id")=="app-skladchik-reports-monitor" and + _ev.get("generation")==_gen and _ev.get("transport_state")=="SEALED" and + _ev.get("restore_source")=="Cloud A" and _ev.get("payload_sha256")==_archive_sha and + _ev.get("payload_sha256_verified") is True and _ev.get("result")=="OK"): + continue + _mp=_subprocess.run(["rclone","cat",_remote_b.rstrip("/")+"/PARTS.tsv"],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=60) + if _mp.returncode!=0 or _hashlib.sha256(_mp.stdout).hexdigest()!=_manifest_sha: continue + _rows=[]; _valid=True + try: + for _line in _mp.stdout.decode("utf-8","strict").splitlines(): + if not _line: continue + _f=_line.split("\t") + if len(_f)!=4: _valid=False; break + _idx,_name,_size_s,_psha=_f + if not (_idx.isdigit() and len(_idx)==8 and _size_s.isdigit() and len(_psha)==64): + _valid=False; break + _size=int(_size_s) + _pp=_subprocess.run(["rclone","cat",_remote_b.rstrip("/")+"/payload.parts/"+_name],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=120) + if _pp.returncode!=0 or len(_pp.stdout)!=_size or _hashlib.sha256(_pp.stdout).hexdigest()!=_psha: + _valid=False; break + _rows.append((_size,_pp.stdout)) + except Exception: + _valid=False + if not _valid or len(_rows)!=_part_count or sum(x[0] for x in _rows)!=_archive_size: continue + _payload=b"".join(x[1] for x in _rows) + if _hashlib.sha256(_payload).hexdigest()!=_archive_sha: continue + cloudb_ok=True + break +except Exception: + cloudb_ok=False +if not cloudb_ok: + issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"}) + +source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +if hrc==0: source["local_head"]=ho.strip() +frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) +prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) +source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") +if frc==0 and fo.strip(): + lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) + if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] +if source["remote_head"]!=source["local_head"]: + issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) + +status="GREEN" if not issues else "RED" +for t in targets: + print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None)) +for i in issues: + print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i)) +print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}") +health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")} +if MODE in ("write-health","scheduled"): + for hp in HEALTHS: + hp.parent.mkdir(parents=True,exist_ok=True) + tmp=hp.with_name(hp.name+".tmp") + tmp.write_text(json.dumps(health,separators=(",",":"),sort_keys=True)+"\n",encoding="utf-8") + os.chmod(tmp,0o644); os.replace(tmp,hp) +sys.exit(0 if not issues else 1) +=== BACKUP_AUDIT_EXECSTART_SOURCE_END === +=== BACKUP_AUDIT_TIMER_SHOW === +NextElapseUSecRealtime=Sun 2026-09-27 21:36:58 MSK +LastTriggerUSec=Sun 2026-09-27 21:05:58 MSK +Result=success +Id=homelab-backup-audit.timer +LoadState=loaded +ActiveState=active +SubState=waiting +FragmentPath=/etc/systemd/system/homelab-backup-audit.timer +DropInPaths= +UnitFileState=enabled +BACKUP_AUDIT_TIMER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.timer +BACKUP_AUDIT_TIMER_SHA256=987695f83ee993bd4d716b8365308211110ac4c684f0bd9b17dfdb1e523e466a +=== BACKUP_AUDIT_TIMER_CAT === +# /etc/systemd/system/homelab-backup-audit.timer +[Unit] +Description=Run homelab unified backup and DR audit + +[Timer] +OnCalendar=*-*-* *:05,35:00 +RandomizedDelaySec=2min +Persistent=true +AccuracySec=1min +Unit=homelab-backup-audit.service + +[Install] +WantedBy=timers.target +=== BACKUP_V2_JOURNAL_LAST12H === +2026-09-27T16:56:08.780555+03:00 pve01 homelab-backup-v2[3412388]: lxc_adapter.AdapterError: FRESH_HELPER_FAILED +2026-09-27T16:56:08.785999+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T16:56:08.786073+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. +2026-09-27T16:56:08.786211+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-27T16:56:08.786299+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 7.475s CPU time, 159M memory peak. +2026-09-27T17:32:01.150519+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-27T17:32:01.367425+03:00 pve01 homelab-backup-v2[3534166]: SELFTEST_REPLACEMENT486=PASS +2026-09-27T17:32:08.781016+03:00 pve01 homelab-backup-v2[3534166]: SCHEDULE_TARGET=ct110 +2026-09-27T17:32:08.788194+03:00 pve01 homelab-backup-v2[3534166]: Traceback (most recent call last): +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/usr/local/sbin/homelab-backup-v2", line 26, in +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return int(e.main()) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try:return legacy.scheduled_run() +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try: backup_target(t,green_verify=False) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return _legacy_backup_target(t,green_verify) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return int(e.main()) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try:return legacy.scheduled_run() +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try: backup_target(t,green_verify=False) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return _legacy_backup_target(t,green_verify) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return int(e.main()) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try:return legacy.scheduled_run() +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try: backup_target(t,green_verify=False) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return _legacy_backup_target(t,green_verify) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return int(e.main()) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try:return legacy.scheduled_run() +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try: backup_target(t,green_verify=False) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return _legacy_backup_target(t,green_verify) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return int(e.main()) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try:return legacy.scheduled_run() +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try: backup_target(t,green_verify=False) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return _legacy_backup_target(t,green_verify) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return int(e.main()) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try:return legacy.scheduled_run() +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try: backup_target(t,green_verify=False) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return _legacy_backup_target(t,green_verify) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return int(e.main()) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try:return legacy.scheduled_run() +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try: backup_target(t,green_verify=False) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return _legacy_backup_target(t,green_verify) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: _need(p.returncode==0 and not p.stderr and 01 else \"all\"\nif MODE not in (\"all\",\"write-health\",\"scheduled\",\"contract\"):\n print(\"AUDIT_ERROR=USAGE_all_write-health_scheduled_contract\",file=sys.stderr); sys.exit(64)\nif MODE==\"contract\":\n print(\"AUDIT_CONTRACT=v2-plus-sealed-quorum\")\n print(\"AUDIT_SCHEMA=homelab-backup-audit-v3\")\n sys.exit(0)\n\nREG=pathlib.Path(\"/etc/homelab-backup/targets.json\")\nSTATE=pathlib.Path(\"/var/lib/homelab-backup-v2/state\")\nQROOT=pathlib.Path(\"/var/lib/homelab-backup/cloud-quorum\")\nVMREST=pathlib.Path(\"/var/lib/homelab-backup/vm-restore-validation\")\nAPPREST=pathlib.Path(\"/var/lib/homelab-backup/app-restore-validation\")\nREGH=\"/usr/local/libexec/homelab-backup-target-registry\"\nAPPQ=\"/usr/local/sbin/homelab-app-cloud-quorum-queue\"\nGFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nGFSP=\"/etc/homelab-backup/gfs-retention.json\"\nRET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nWT=\"/root/homelab-ops-cr-2026-1005\"\nBRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nDRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nCLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nHEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\n\ndef run(cmd,timeout=60,env=None):\n try:\n p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)\n return p.returncode,p.stdout,p.stderr\n except subprocess.TimeoutExpired as e:\n return 124,e.stdout or \"\",e.stderr or \"\"\n\ndef collect(obj,out):\n if isinstance(obj,dict):\n if isinstance(obj.get(\"logical_id\"),str): out.append(obj)\n for v in obj.values(): collect(v,out)\n elif isinstance(obj,list):\n for v in obj: collect(v,out)\n\ndef parse_epoch(obj):\n for k in (\"sealed_epoch\",\"last_success_epoch\",\"updated_epoch\"):\n v=obj.get(k)\n if isinstance(v,(int,float)) and v>0: return int(v)\n for k in (\"sealed_at_utc\",\"updated_at_utc\",\"completed_at_utc\"):\n v=obj.get(k)\n if isinstance(v,str) and v:\n try:\n return int(datetime.datetime.fromisoformat(v.replace(\"Z\",\"+00:00\")).timestamp())\n except Exception: pass\n return None\n\ndef latest_legacy(logical,kind):\n d=QROOT/logical\n best=None; best_epoch=-1\n if not d.is_dir(): return None\n for p in d.glob(\"*.json\"):\n try: obj=json.loads(p.read_text(encoding=\"utf-8\"))\n except Exception: continue\n epoch=parse_epoch(obj)\n score=epoch if epoch is not None else int(p.stat().st_mtime)\n if score>best_epoch:\n best_epoch=score; best=(p,obj,score)\n if not best: return None\n p,obj,epoch=best\n gen=obj.get(\"generation\")\n strict=(obj.get(\"status\")==\"SEALED\" and obj.get(\"cloud_quorum\")==2 and obj.get(\"cloud_a_verified\") is True and obj.get(\"cloud_b_verified\") is True and obj.get(\"part_hashes_verified\") is True and obj.get(\"seal_written\") is True and obj.get(\"local_payload_present\") is False and obj.get(\"local_cleanup\")==\"OK\")\n rroot=VMREST if kind==\"VM\" else APPREST\n rev=(rroot/logical/(str(gen)+\".json\")) if gen else None\n restore=False\n if rev and rev.is_file():\n try:\n rj=json.loads(rev.read_text(encoding=\"utf-8\"))\n restore=(rj.get(\"result\")==\"OK\")\n except Exception: pass\n age=max(0,int(time.time()-epoch)) if epoch else None\n return {\"generation\":gen,\"strict\":strict,\"restore_ok\":restore,\"age_seconds\":age,\"state_file\":str(p),\"obj\":obj}\n\ndef unit_ok(unit):\n a=run([\"systemctl\",\"is-active\",unit],10)[1].strip()\n e=run([\"systemctl\",\"is-enabled\",unit],10)[1].strip()\n return a==\"active\" and e==\"enabled\",a,e\n\ndef parse_app_plan(txt):\n d={}\n for line in txt.splitlines():\n if line.startswith(\"QUEUE_FRESHNESS_PLAN APP=\"):\n r={}\n for tok in line.split()[1:]:\n if \"=\" in tok:\n k,v=tok.split(\"=\",1); r[k]=v\n if r.get(\"APP\"): d[r[\"APP\"]]=r.get(\"DECISION\")\n return d\n\nissues=[]; targets=[]; now=int(time.time())\ntry: robj=json.loads(REG.read_text(encoding=\"utf-8\"))\nexcept Exception:\n print(\"AUDIT_ERROR=REGISTRY_UNREADABLE\",file=sys.stderr); sys.exit(69)\nrows=[]; collect(robj,rows)\nactive={}\nfor r in rows:\n lid=r.get(\"logical_id\")\n if lid and str(r.get(\"lifecycle\",\"\")).upper()==\"ACTIVE\":\n active[lid]={\"logical_id\":lid,\"kind\":str(r.get(\"kind\",\"\")).upper(),\"id\":str(r.get(\"id\")) if r.get(\"id\") is not None else None}\n\nvmrc,vmo,_=run([REGH,\"freshness-seconds\",\"VM\"],20)\naprc,apo,_=run([REGH,\"freshness-seconds\",\"APP\"],20)\nvm_sla=int(vmo.strip()) if vmrc==0 and vmo.strip().isdigit() else 72000\napp_sla=int(apo.strip()) if aprc==0 and apo.strip().isdigit() else 72000\n\nrrc,rout,_=run([\"pvesh\",\"get\",\"/cluster/resources\",\"--type\",\"vm\",\"--output-format\",\"json\"],30)\nresources={}\nif rrc==0:\n try:\n for x in json.loads(rout): resources[str(x.get(\"vmid\"))]=x\n except Exception: pass\n\nqrc,qout,qerr=run([APPQ,\"freshness-plan\"],180)\napp_dec=parse_app_plan(qout if qrc==0 else \"\")\nif qrc!=0: issues.append({\"severity\":\"CRITICAL\",\"code\":\"APP_FRESHNESS_PLAN_FAILED\",\"logical_id\":\"-\",\"root_cause\":\"APP_QUEUE_RC_\"+str(qrc)})\n\nfor lid,r in sorted(active.items()):\n kind=r[\"kind\"]; ident=r[\"id\"]; rec={\"logical_id\":lid,\"kind\":kind,\"state\":\"UNVERIFIED\"}\n if kind==\"VM\":\n typ=str(resources.get(ident,{}).get(\"type\",\"qemu\"))\n sid=(\"ct\"+ident) if typ==\"lxc\" else (\"vm\"+ident)\n v2=None\n try: v2=json.loads((STATE/(sid+\".json\")).read_text(encoding=\"utf-8\"))\n except Exception: pass\n v2_ok=False; v2_age=None; repcount=None\n if isinstance(v2,dict):\n ls=v2.get(\"last_success_epoch\")\n v2_age=max(0,int(now-ls)) if isinstance(ls,(int,float)) and ls>0 else None\n reps=v2.get(\"replicas\"); repcount=len(reps) if isinstance(reps,(list,dict)) else None\n v2_ok=(v2.get(\"result\")==\"SUCCESS\" and v2_age is not None and v2_age=3:\n issues.append({\"severity\":\"HIGH\",\"code\":\"RETRY_EXHAUSTION\",\"logical_id\":lid,\"root_cause\":\"CONSECUTIVE_FAILURES_\"+str(cf)})\n legacy=latest_legacy(lid,\"VM\")\n legacy_ok=bool(legacy and legacy[\"strict\"] and legacy[\"restore_ok\"] and legacy[\"age_seconds\"] is not None and legacy[\"age_seconds\"]0 else None\n fresh=(obj.get(\"result\")==\"SUCCESS\" and age is not None and age<86400)\n rec.update({\"state\":\"FRESH\" if fresh else \"STALE\",\"age_seconds\":age,\"generation\":obj.get(\"generation\") if isinstance(obj,dict) else None})\n if not fresh: issues.append({\"severity\":\"HIGH\",\"code\":\"RPO_BREACH\",\"logical_id\":lid,\"root_cause\":\"CONTROL_STATE_NOT_FRESH\"})\n targets.append(rec)\n\nif len(targets)!=len(active):\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"LIFECYCLE_COVERAGE_GAP\",\"logical_id\":\"-\",\"root_cause\":\"ACTIVE_TARGET_COUNT_MISMATCH\"})\n\nfor unit,code in [\n (\"homelab-backup-v2.timer\",\"V2_TIMER_BAD\"),\n (\"homelab-app-cloud-quorum-queue.timer\",\"APP_TIMER_BAD\"),\n (\"homelab-backup-retention-gc.timer\",\"RETENTION_TIMER_BAD\"),\n (\"homelab-alertmanager-backup-health.timer\",\"ALERT_TIMER_BAD\"),\n]:\n ok,a,e=unit_ok(unit)\n if not ok: issues.append({\"severity\":\"HIGH\",\"code\":code,\"logical_id\":unit,\"root_cause\":\"UNIT_\"+a+\"_\"+e})\n\nfor lock in [\"/run/homelab-backup-heavy.lock\",\"/run/lock/homelab-app-cloud-quorum-queue.lock\",\"/run/lock/homelab-vm-cloud-quorum-queue.lock\",\"/run/lock/homelab-backup-dispatcher.lock\"]:\n p=pathlib.Path(lock)\n if p.exists():\n lrc,_,_=run([\"flock\",\"-n\",str(p),\"-c\",\"true\"],5)\n if lrc!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"LOCK_BUSY\",\"logical_id\":lock,\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\"})\n\nenv=os.environ.copy(); env[\"HOMELAB_BACKUP_GFS_POLICY\"]=GFSP; env[\"HOMELAB_BACKUP_TARGET_REGISTRY\"]=str(REG)\ngrc,gout,_=run([GFSH,\"validate\"],20,env)\nif grc!=0 or \"GFS_POLICY=OK\" not in gout:\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"GFS_INVALID\",\"logical_id\":\"-\",\"root_cause\":\"GFS_VALIDATE_FAILED\"})\nrr,_,_=run([RET,\"contract\"],120)\nif rr!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_CONTRACT_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_CONTRACT_RC_\"+str(rr)})\nrhp=pathlib.Path(\"/var/lib/homelab-health/backup-retention-safe-gc.json\")\nif not rhp.is_file():\n issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_FILE_MISSING\"})\nelse:\n try:\n rh=json.loads(rhp.read_text(encoding=\"utf-8\"))\n if rh.get(\"status\")!=\"OK\": issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_NOT_OK\"})\n except Exception: issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_UNREADABLE\"})\n\ncloud={}\nfor label,remote in [(\"A\",\"pve01-mail-01-crypt:homelab-backups\"),(\"B\",\"pve01-mail-02-crypt:homelab-backups\")]:\n lrc,lo,_=run([\"rclone\",\"lsf\",remote,\"--max-depth\",\"1\",\"--dirs-only\"],60)\n arc,ao,_=run([\"rclone\",\"about\",remote,\"--json\"],60)\n rec={\"readable\":lrc==0,\"top_dirs\":len([x for x in lo.splitlines() if x.strip()])}\n if arc==0:\n try:\n j=json.loads(ao); rec.update({k:j.get(k) for k in (\"total\",\"used\",\"free\")})\n if j.get(\"total\") and j.get(\"free\") is not None:\n frac=j[\"free\"]/j[\"total\"]; rec[\"free_fraction\"]=round(frac,6)\n if frac<0.15: issues.append({\"severity\":\"HIGH\",\"code\":\"CAPACITY_LOW\",\"logical_id\":\"cloud-\"+label,\"root_cause\":\"FREE_LT_15_PERCENT\"})\n except Exception: pass\n if lrc!=0: issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_UNREADABLE\",\"logical_id\":\"cloud-\"+label,\"root_cause\":\"RCLONE_LSF_FAILED\"})\n cloud[label]=rec\n\ntry: dr=json.loads(DRSEAL.read_text(encoding=\"utf-8\"))\nexcept Exception: dr=None\nif not isinstance(dr,dict) or dr.get(\"status\")!=\"SEALED\":\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"DR_SEAL_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"SITE_DR_FINAL_SEAL_NOT_SEALED\"})\n\ncloudb_ok=False\ntry:\n import pathlib as _pathlib, json as _json, subprocess as _subprocess, hashlib as _hashlib\n _state_root=_pathlib.Path(\"/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor\")\n _restore_root=_pathlib.Path(\"/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor\")\n _states=sorted([p for p in _state_root.glob(\"*.json\") if p.is_file() and not p.is_symlink()],key=lambda p:p.stat().st_mtime,reverse=True)\n for _sp in _states:\n try: _st=_json.loads(_sp.read_text(encoding=\"utf-8\"))\n except Exception: continue\n if not (\n _st.get(\"schema\")==\"homelab-cloud-quorum-payload-v1\" and\n _st.get(\"logical_id\")==\"app-skladchik-reports-monitor\" and\n _st.get(\"status\")==\"SEALED\" and _st.get(\"cloud_quorum\")==2 and\n _st.get(\"cloud_a_verified\") is True and _st.get(\"cloud_b_verified\") is True and\n _st.get(\"part_hashes_verified\") is True and _st.get(\"seal_written\") is True and\n _st.get(\"cloud_a_seal_verified\") is True and _st.get(\"cloud_b_seal_verified\") is True and\n _st.get(\"local_payload_present\") is False and _st.get(\"local_cleanup\")==\"OK\"):\n continue\n _gen=_st.get(\"generation\"); _archive_sha=_st.get(\"archive_sha256\")\n _manifest_sha=_st.get(\"parts_manifest_sha256\"); _part_count=_st.get(\"part_count\")\n _archive_size=_st.get(\"archive_size_bytes\"); _remote_b=_st.get(\"remote_b_base\")\n if not (isinstance(_gen,str) and isinstance(_archive_sha,str) and len(_archive_sha)==64 and\n isinstance(_manifest_sha,str) and len(_manifest_sha)==64 and isinstance(_part_count,int) and\n _part_count>0 and isinstance(_archive_size,int) and _archive_size>0 and isinstance(_remote_b,str)):\n continue\n _evp=_restore_root/(_gen+\".json\")\n try: _ev=_json.loads(_evp.read_text(encoding=\"utf-8\"))\n except Exception: continue\n if not (\n _ev.get(\"schema\")==\"homelab-app-restore-validation-v1\" and\n _ev.get(\"logical_id\")==\"app-skladchik-reports-monitor\" and\n _ev.get(\"generation\")==_gen and _ev.get(\"transport_state\")==\"SEALED\" and\n _ev.get(\"restore_source\")==\"Cloud A\" and _ev.get(\"payload_sha256\")==_archive_sha and\n _ev.get(\"payload_sha256_verified\") is True and _ev.get(\"result\")==\"OK\"):\n continue\n _mp=_subprocess.run([\"rclone\",\"cat\",_remote_b.rstrip(\"/\")+\"/PARTS.tsv\"],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=60)\n if _mp.returncode!=0 or _hashlib.sha256(_mp.stdout).hexdigest()!=_manifest_sha: continue\n _rows=[]; _valid=True\n try:\n for _line in _mp.stdout.decode(\"utf-8\",\"strict\").splitlines():\n if not _line: continue\n _f=_line.split(\"\\t\")\n if len(_f)!=4: _valid=False; break\n _idx,_name,_size_s,_psha=_f\n if not (_idx.isdigit() and len(_idx)==8 and _size_s.isdigit() and len(_psha)==64):\n _valid=False; break\n _size=int(_size_s)\n _pp=_subprocess.run([\"rclone\",\"cat\",_remote_b.rstrip(\"/\")+\"/payload.parts/\"+_name],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=120)\n if _pp.returncode!=0 or len(_pp.stdout)!=_size or _hashlib.sha256(_pp.stdout).hexdigest()!=_psha:\n _valid=False; break\n _rows.append((_size,_pp.stdout))\n except Exception:\n _valid=False\n if not _valid or len(_rows)!=_part_count or sum(x[0] for x in _rows)!=_archive_size: continue\n _payload=b\"\".join(x[1] for x in _rows)\n if _hashlib.sha256(_payload).hexdigest()!=_archive_sha: continue\n cloudb_ok=True\n break\nexcept Exception:\n cloudb_ok=False\nif not cloudb_ok:\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK\"})\n\nsource={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\nhrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\nif hrc==0: source[\"local_head\"]=ho.strip()\nfrc,fo,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.url\"],10)\nprc,po,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.pushurl\"],10)\nsource[\"pushurl_disabled\"]=(prc==0 and po.strip()==\"DISABLED\")\nif frc==0 and fo.strip():\n lrc,lo,_=run([\"git\",\"ls-remote\",fo.strip(),\"refs/heads/\"+BRANCH],30)\n if lrc==0 and lo.strip(): source[\"remote_head\"]=lo.split()[0]\nif source[\"remote_head\"]!=source[\"local_head\"]:\n issues.append({\"severity\":\"MEDIUM\",\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\"})\n\nstatus=\"GREEN\" if not issues else \"RED\"\nfor t in targets:\n print(\"TARGET|\"+\"|\".join(f\"{k}={v}\" for k,v in t.items() if v is not None))\nfor i in issues:\n print(\"ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}\".format(**i))\nprint(f\"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}\")\nhealth={\"schema\":\"homelab-backup-audit-v3\",\"status\":status,\"generated_at_epoch\":now,\"targets\":targets,\"issues\":issues,\"cloud\":cloud,\"source\":source,\"cloud_b_restore_drill\":cloudb_ok,\"dr_proven\":bool(isinstance(dr,dict) and dr.get(\"status\")==\"SEALED\")}\nif MODE in (\"write-health\",\"scheduled\"):\n for hp in HEALTHS:\n hp.parent.mkdir(parents=True,exist_ok=True)\n tmp=hp.with_name(hp.name+\".tmp\")\n tmp.write_text(json.dumps(health,separators=(\",\",\":\"),sort_keys=True)+\"\\n\",encoding=\"utf-8\")\n os.chmod(tmp,0o644); os.replace(tmp,hp)\nsys.exit(0 if not issues else 1)\n=== BACKUP_AUDIT_EXECSTART_SOURCE_END ===\n=== BACKUP_AUDIT_TIMER_SHOW ===\nNextElapseUSecRealtime=Sun 2026-09-27 21:36:58 MSK\nLastTriggerUSec=Sun 2026-09-27 21:05:58 MSK\nResult=success\nId=homelab-backup-audit.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nFragmentPath=/etc/systemd/system/homelab-backup-audit.timer\nDropInPaths=\nUnitFileState=enabled\nBACKUP_AUDIT_TIMER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.timer\nBACKUP_AUDIT_TIMER_SHA256=987695f83ee993bd4d716b8365308211110ac4c684f0bd9b17dfdb1e523e466a\n=== BACKUP_AUDIT_TIMER_CAT ===\n# /etc/systemd/system/homelab-backup-audit.timer\n[Unit]\nDescription=Run homelab unified backup and DR audit\n\n[Timer]\nOnCalendar=*-*-* *:05,35:00\nRandomizedDelaySec=2min\nPersistent=true\nAccuracySec=1min\nUnit=homelab-backup-audit.service\n\n[Install]\nWantedBy=timers.target\n=== BACKUP_V2_JOURNAL_LAST12H ===\n2026-09-27T16:56:08.780555+03:00 pve01 homelab-backup-v2[3412388]: lxc_adapter.AdapterError: FRESH_HELPER_FAILED\n2026-09-27T16:56:08.785999+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T16:56:08.786073+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\n2026-09-27T16:56:08.786211+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-27T16:56:08.786299+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 7.475s CPU time, 159M memory peak.\n2026-09-27T17:32:01.150519+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-27T17:32:01.367425+03:00 pve01 homelab-backup-v2[3534166]: SELFTEST_REPLACEMENT486=PASS\n2026-09-27T17:32:08.781016+03:00 pve01 homelab-backup-v2[3534166]: SCHEDULE_TARGET=ct110\n2026-09-27T17:32:08.788194+03:00 pve01 homelab-backup-v2[3534166]: Traceback (most recent call last):\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/usr/local/sbin/homelab-backup-v2\", line 26, in \n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return int(e.main())\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try:return legacy.scheduled_run()\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try: backup_target(t,green_verify=False)\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return _legacy_backup_target(t,green_verify)\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return int(e.main())\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try:return legacy.scheduled_run()\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try: backup_target(t,green_verify=False)\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return _legacy_backup_target(t,green_verify)\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return int(e.main())\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try:return legacy.scheduled_run()\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try: backup_target(t,green_verify=False)\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return _legacy_backup_target(t,green_verify)\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return int(e.main())\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try:return legacy.scheduled_run()\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try: backup_target(t,green_verify=False)\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return _legacy_backup_target(t,green_verify)\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return int(e.main())\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try:return legacy.scheduled_run()\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try: backup_target(t,green_verify=False)\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return _legacy_backup_target(t,green_verify)\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return int(e.main())\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try:return legacy.scheduled_run()\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try: backup_target(t,green_verify=False)\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return _legacy_backup_target(t,green_verify)\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return int(e.main())\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try:return legacy.scheduled_run()\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try: backup_target(t,green_verify=False)\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return _legacy_backup_target(t,green_verify)\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: _need(p.returncode==0 and not p.stderr and 01 else "all" +if MODE not in ("all","write-health","scheduled","contract"): + print("AUDIT_ERROR=USAGE_all_write-health_scheduled_contract",file=sys.stderr); sys.exit(64) +if MODE=="contract": + print("AUDIT_CONTRACT=v2-plus-sealed-quorum") + print("AUDIT_SCHEMA=homelab-backup-audit-v3") + sys.exit(0) +REG=pathlib.Path("/etc/homelab-backup/targets.json") +STATE=pathlib.Path("/var/lib/homelab-backup-v2/state") +QROOT=pathlib.Path("/var/lib/homelab-backup/cloud-quorum") +VMREST=pathlib.Path("/var/lib/homelab-backup/vm-restore-validation") +APPREST=pathlib.Path("/var/lib/homelab-backup/app-restore-validation") +REGH="/usr/local/libexec/homelab-backup-target-registry" +APPQ="/usr/local/sbin/homelab-app-cloud-quorum-queue" +GFSH="/usr/local/libexec/homelab-backup-gfs-policy" +GFSP="/etc/homelab-backup/gfs-retention.json" +RET="/usr/local/sbin/homelab-backup-retention-gc" +WT="/root/homelab-ops-cr-2026-1005" +BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" +DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") +CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] -## NEWFI_PRIVATE_STRUCTURE92_WRONG_MARKER_SCOPE_[PRIVATE_IP] CLOSURE -- Status: CLOSED -- RCA: NEWFI92 inspected the pve01 NEWFI90 attempt marker, which had not been finalized because the outer shell exited immediately after the guest returned RC3 following its successful rollback. The guest marker contained the true final rollback state. -- Correction: NEWFI93 required both the guest FINISHED/ROLLED_BACK record and a fresh exact demo-state readback before reconciling the outer NEWFI90 marker. NEWFI92 itself never reserved an attempt or mutated state. +def run(cmd,timeout=60,env=None): + try: + p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env) + return p.returncode,p.stdout,p.stderr + except subprocess.TimeoutExpired as e: + return 124,e.stdout or "",e.stderr or "" +def collect(obj,out): + if isinstance(obj,dict): + if isinstance(obj.get("logical_id"),str): out.append(obj) + for v in obj.values(): collect(v,out) + elif isinstance(obj,list): + for v in obj: collect(v,out) -## NEWFI_PRIVATE_NEWFI95_NESTED_TRIPLE_QUOTE_PARSE_[PRIVATE_IP] -- Status: OPEN -- Command: NEWFI95 -- Symptom: after the content-structure precheck passed, the pve03 Python wrapper failed to parse at the PHP line beginning with $root. No Portal discovery or application mutation was reached. -- RCA: an outer raw triple-single-quoted guest Python payload contained an inner raw triple-single-quoted PHP payload. The inner delimiter terminated the outer Python string, causing PHP source to be parsed as Python. -- Safety: NEWFI95 was read-only, created no attempt marker, and did not reach the intended nested guest execution. Do not replay NEWFI95. +def parse_epoch(obj): + for k in ("sealed_epoch","last_success_epoch","updated_epoch"): + v=obj.get(k) + if isinstance(v,(int,float)) and v>0: return int(v) + for k in ("sealed_at_utc","updated_at_utc","completed_at_utc"): + v=obj.get(k) + if isinstance(v,str) and v: + try: + return int(datetime.datetime.fromisoformat(v.replace("Z","+00:00")).timestamp()) + except Exception: pass + return None +def latest_legacy(logical,kind): + d=QROOT/logical + best=None; best_epoch=-1 + if not d.is_dir(): return None + for p in d.glob("*.json"): + try: obj=json.loads(p.read_text(encoding="utf-8")) + except Exception: continue + epoch=parse_epoch(obj) + score=epoch if epoch is not None else int(p.stat().st_mtime) + if score>best_epoch: + best_epoch=score; best=(p,obj,score) + if not best: return None + p,obj,epoch=best + gen=obj.get("generation") + strict=(obj.get("status")=="SEALED" and obj.get("cloud_quorum")==2 and obj.get("cloud_a_verified") is True and obj.get("cloud_b_verified") is True and obj.get("part_hashes_verified") is True and obj.get("seal_written") is True and obj.get("local_payload_present") is False and obj.get("local_cleanup")=="OK") + rroot=VMREST if kind=="VM" else APPREST + rev=(rroot/logical/(str(gen)+".json")) if gen else None + restore=False + if rev and rev.is_file(): + try: + rj=json.loads(rev.read_text(encoding="utf-8")) + restore=(rj.get("result")=="OK") + except Exception: pass + age=max(0,int(time.time()-epoch)) if epoch else None + return {"generation":gen,"strict":strict,"restore_ok":restore,"age_seconds":age,"state_file":str(p),"obj":obj} + +def unit_ok(unit): + a=run(["systemctl","is-active",unit],10)[1].strip() + e=run(["systemctl","is-enabled",unit],10)[1].strip() + return a=="active" and e=="enabled",a,e + +def parse_app_plan(txt): + d={} + for line in txt.splitlines(): + if line.startswith("QUEUE_FRESHNESS_PLAN APP="): + r={} + for tok in line.split()[1:]: + if "=" in tok: + k,v=tok.split("=",1); r[k]=v + if r.get("APP"): d[r["APP"]]=r.get("DECISION") + return d + +issues=[]; targets=[]; now=int(time.time()) +try: robj=json.loads(REG.read_text(encoding="utf-8")) +except Exception: + print("AUDIT_ERROR=REGISTRY_UNREADABLE",file=sys.stderr); sys.exit(69) +rows=[]; collect(robj,rows) +active={} +for r in rows: + lid=r.get("logical_id") + if lid and str(r.get("lifecycle","")).upper()=="ACTIVE": + active[lid]={"logical_id":lid,"kind":str(r.get("kind","")).upper(),"id":str(r.get("id")) if r.get("id") is not None else None} + +vmrc,vmo,_=run([REGH,"freshness-seconds","VM"],20) +aprc,apo,_=run([REGH,"freshness-seconds","APP"],20) +vm_sla=int(vmo.strip()) if vmrc==0 and vmo.strip().isdigit() else 72000 +app_sla=int(apo.strip()) if aprc==0 and apo.strip().isdigit() else 72000 + +rrc,rout,_=run(["pvesh","get","/cluster/resources","--type","vm","--output-format","json"],30) +resources={} +if rrc==0: + try: + for x in json.loads(rout): resources[str(x.get("vmid"))]=x + except Exception: pass + +qrc,qout,qerr=run([APPQ,"freshness-plan"],180) +app_dec=parse_app_plan(qout if qrc==0 else "") +if qrc!=0: issues.append({"severity":"CRITICAL","code":"APP_FRESHNESS_PLAN_FAILED","logical_id":"-","root_cause":"APP_QUEUE_RC_"+str(qrc)}) + +for lid,r in sorted(active.items()): + kind=r["kind"]; ident=r["id"]; rec={"logical_id":lid,"kind":kind,"state":"UNVERIFIED"} + if kind=="VM": + typ=str(resources.get(ident,{}).get("type","qemu")) + sid=("ct"+ident) if typ=="lxc" else ("vm"+ident) + v2=None + try: v2=json.loads((STATE/(sid+".json")).read_text(encoding="utf-8")) + except Exception: pass + v2_ok=False; v2_age=None; repcount=None + if isinstance(v2,dict): + ls=v2.get("last_success_epoch") + v2_age=max(0,int(now-ls)) if isinstance(ls,(int,float)) and ls>0 else None + reps=v2.get("replicas"); repcount=len(reps) if isinstance(reps,(list,dict)) else None + v2_ok=(v2.get("result")=="SUCCESS" and v2_age is not None and v2_age=3: + issues.append({"severity":"HIGH","code":"RETRY_EXHAUSTION","logical_id":lid,"root_cause":"CONSECUTIVE_FAILURES_"+str(cf)}) + legacy=latest_legacy(lid,"VM") + legacy_ok=bool(legacy and legacy["strict"] and legacy["restore_ok"] and legacy["age_seconds"] is not None and legacy["age_seconds"]0 else None + fresh=(obj.get("result")=="SUCCESS" and age is not None and age<86400) + rec.update({"state":"FRESH" if fresh else "STALE","age_seconds":age,"generation":obj.get("generation") if isinstance(obj,dict) else None}) + if not fresh: issues.append({"severity":"HIGH","code":"RPO_BREACH","logical_id":lid,"root_cause":"CONTROL_STATE_NOT_FRESH"}) + targets.append(rec) + +if len(targets)!=len(active): + issues.append({"severity":"CRITICAL","code":"LIFECYCLE_COVERAGE_GAP","logical_id":"-","root_cause":"ACTIVE_TARGET_COUNT_MISMATCH"}) + +for unit,code in [ + ("homelab-backup-v2.timer","V2_TIMER_BAD"), + ("homelab-app-cloud-quorum-queue.timer","APP_TIMER_BAD"), + ("homelab-backup-retention-gc.timer","RETENTION_TIMER_BAD"), + ("homelab-alertmanager-backup-health.timer","ALERT_TIMER_BAD"), +]: + ok,a,e=unit_ok(unit) + if not ok: issues.append({"severity":"HIGH","code":code,"logical_id":unit,"root_cause":"UNIT_"+a+"_"+e}) + +for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]: + p=pathlib.Path(lock) + if p.exists(): + lrc,_,_=run(["flock","-n",str(p),"-c","true"],5) + if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"}) + +env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG) +grc,gout,_=run([GFSH,"validate"],20,env) +if grc!=0 or "GFS_POLICY=OK" not in gout: + issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"}) +rr,_,_=run([RET,"contract"],120) +if rr!=0: issues.append({"severity":"HIGH","code":"RETENTION_CONTRACT_BAD","logical_id":"-","root_cause":"RETENTION_CONTRACT_RC_"+str(rr)}) +rhp=pathlib.Path("/var/lib/homelab-health/backup-retention-safe-gc.json") +if not rhp.is_file(): + issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_MISSING","logical_id":"-","root_cause":"RETENTION_HEALTH_FILE_MISSING"}) +else: + try: + rh=json.loads(rhp.read_text(encoding="utf-8")) + if rh.get("status")!="OK": issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_NOT_OK"}) + except Exception: issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_UNREADABLE"}) + +cloud={} +for label,remote in [("A","pve01-mail-01-crypt:homelab-backups"),("B","pve01-mail-02-crypt:homelab-backups")]: + lrc,lo,_=run(["rclone","lsf",remote,"--max-depth","1","--dirs-only"],60) + arc,ao,_=run(["rclone","about",remote,"--json"],60) + rec={"readable":lrc==0,"top_dirs":len([x for x in lo.splitlines() if x.strip()])} + if arc==0: + try: + j=json.loads(ao); rec.update({k:j.get(k) for k in ("total","used","free")}) + if j.get("total") and j.get("free") is not None: + frac=j["free"]/j["total"]; rec["free_fraction"]=round(frac,6) + if frac<0.15: issues.append({"severity":"HIGH","code":"CAPACITY_LOW","logical_id":"cloud-"+label,"root_cause":"FREE_LT_15_PERCENT"}) + except Exception: pass + if lrc!=0: issues.append({"severity":"CRITICAL","code":"CLOUD_UNREADABLE","logical_id":"cloud-"+label,"root_cause":"RCLONE_LSF_FAILED"}) + cloud[label]=rec + +try: dr=json.loads(DRSEAL.read_text(encoding="utf-8")) +except Exception: dr=None +if not isinstance(dr,dict) or dr.get("status")!="SEALED": + issues.append({"severity":"CRITICAL","code":"DR_SEAL_MISSING","logical_id":"-","root_cause":"SITE_DR_FINAL_SEAL_NOT_SEALED"}) + +cloudb_ok=False +try: + import pathlib as _pathlib, json as _json, subprocess as _subprocess, hashlib as _hashlib + _state_root=_pathlib.Path("/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor") + _restore_root=_pathlib.Path("/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor") + _states=sorted([p for p in _state_root.glob("*.json") if p.is_file() and not p.is_symlink()],key=lambda p:p.stat().st_mtime,reverse=True) + for _sp in _states: + try: _st=_json.loads(_sp.read_text(encoding="utf-8")) + except Exception: continue + if not ( + _st.get("schema")=="homelab-cloud-quorum-payload-v1" and + _st.get("logical_id")=="app-skladchik-reports-monitor" and + _st.get("status")=="SEALED" and _st.get("cloud_quorum")==2 and + _st.get("cloud_a_verified") is True and _st.get("cloud_b_verified") is True and + _st.get("part_hashes_verified") is True and _st.get("seal_written") is True and + _st.get("cloud_a_seal_verified") is True and _st.get("cloud_b_seal_verified") is True and + _st.get("local_payload_present") is False and _st.get("local_cleanup")=="OK"): + continue + _gen=_st.get("generation"); _archive_sha=_st.get("archive_sha256") + _manifest_sha=_st.get("parts_manifest_sha256"); _part_count=_st.get("part_count") + _archive_size=_st.get("archive_size_bytes"); _remote_b=_st.get("remote_b_base") + if not (isinstance(_gen,str) and isinstance(_archive_sha,str) and len(_archive_sha)==64 and + isinstance(_manifest_sha,str) and len(_manifest_sha)==64 and isinstance(_part_count,int) and + _part_count>0 and isinstance(_archive_size,int) and _archive_size>0 and isinstance(_remote_b,str)): + continue + _evp=_restore_root/(_gen+".json") + try: _ev=_json.loads(_evp.read_text(encoding="utf-8")) + except Exception: continue + if not ( + _ev.get("schema")=="homelab-app-restore-validation-v1" and + _ev.get("logical_id")=="app-skladchik-reports-monitor" and + _ev.get("generation")==_gen and _ev.get("transport_state")=="SEALED" and + _ev.get("restore_source")=="Cloud A" and _ev.get("payload_sha256")==_archive_sha and + _ev.get("payload_sha256_verified") is True and _ev.get("result")=="OK"): + continue + _mp=_subprocess.run(["rclone","cat",_remote_b.rstrip("/")+"/PARTS.tsv"],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=60) + if _mp.returncode!=0 or _hashlib.sha256(_mp.stdout).hexdigest()!=_manifest_sha: continue + _rows=[]; _valid=True + try: + for _line in _mp.stdout.decode("utf-8","strict").splitlines(): + if not _line: continue + _f=_line.split("\t") + if len(_f)!=4: _valid=False; break + _idx,_name,_size_s,_psha=_f + if not (_idx.isdigit() and len(_idx)==8 and _size_s.isdigit() and len(_psha)==64): + _valid=False; break + _size=int(_size_s) + _pp=_subprocess.run(["rclone","cat",_remote_b.rstrip("/")+"/payload.parts/"+_name],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=120) + if _pp.returncode!=0 or len(_pp.stdout)!=_size or _hashlib.sha256(_pp.stdout).hexdigest()!=_psha: + _valid=False; break + _rows.append((_size,_pp.stdout)) + except Exception: + _valid=False + if not _valid or len(_rows)!=_part_count or sum(x[0] for x in _rows)!=_archive_size: continue + _payload=b"".join(x[1] for x in _rows) + if _hashlib.sha256(_payload).hexdigest()!=_archive_sha: continue + cloudb_ok=True + break +except Exception: + cloudb_ok=False +if not cloudb_ok: + issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"}) + +source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +if hrc==0: source["local_head"]=ho.strip() +frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) +prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) +source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") +if frc==0 and fo.strip(): + lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) + if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] +if source["remote_head"]!=source["local_head"]: + issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) + +status="GREEN" if not issues else "RED" +for t in targets: + print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None)) +for i in issues: + print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i)) +print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}") +health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")} +if MODE in ("write-health","scheduled"): + for hp in HEALTHS: + hp.parent.mkdir(parents=True,exist_ok=True) + tmp=hp.with_name(hp.name+".tmp") + tmp.write_text(json.dumps(health,separators=(",",":"),sort_keys=True)+"\n",encoding="utf-8") + os.chmod(tmp,0o644); os.replace(tmp,hp) +sys.exit(0 if not issues else 1) +=== BACKUP_AUDIT_EXECSTART_SOURCE_END === +=== BACKUP_AUDIT_TIMER_SHOW === +NextElapseUSecRealtime=Sun 2026-09-27 21:36:58 MSK +LastTriggerUSec=Sun 2026-09-27 21:05:58 MSK +Result=success +Id=homelab-backup-audit.timer +LoadState=loaded +ActiveState=active +SubState=waiting +FragmentPath=/etc/systemd/system/homelab-backup-audit.timer +DropInPaths= +UnitFileState=enabled +BACKUP_AUDIT_TIMER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.timer +BACKUP_AUDIT_TIMER_SHA256=987695f83ee993bd4d716b8365308211110ac4c684f0bd9b17dfdb1e523e466a +=== BACKUP_AUDIT_TIMER_CAT === +# /etc/systemd/system/homelab-backup-audit.timer +[Unit] +Description=Run homelab unified backup and DR audit + +[Timer] +OnCalendar=*-*-* *:05,35:00 +RandomizedDelaySec=2min +Persistent=true +AccuracySec=1min +Unit=homelab-backup-audit.service + +[Install] +WantedBy=timers.target +=== BACKUP_V2_JOURNAL_LAST12H === +2026-09-27T16:56:08.780555+03:00 pve01 homelab-backup-v2[3412388]: lxc_adapter.AdapterError: FRESH_HELPER_FAILED +2026-09-27T16:56:08.785999+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T16:56:08.786073+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. +2026-09-27T16:56:08.786211+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-27T16:56:08.786299+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 7.475s CPU time, 159M memory peak. +2026-09-27T17:32:01.150519+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-27T17:32:01.367425+03:00 pve01 homelab-backup-v2[3534166]: SELFTEST_REPLACEMENT486=PASS +2026-09-27T17:32:08.781016+03:00 pve01 homelab-backup-v2[3534166]: SCHEDULE_TARGET=ct110 +2026-09-27T17:32:08.788194+03:00 pve01 homelab-backup-v2[3534166]: Traceback (most recent call last): +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/usr/local/sbin/homelab-backup-v2", line 26, in +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return int(e.main()) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try:return legacy.scheduled_run() +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try: backup_target(t,green_verify=False) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return _legacy_backup_target(t,green_verify) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return int(e.main()) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try:return legacy.scheduled_run() +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try: backup_target(t,green_verify=False) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return _legacy_backup_target(t,green_verify) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return int(e.main()) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try:return legacy.scheduled_run() +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try: backup_target(t,green_verify=False) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return _legacy_backup_target(t,green_verify) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return int(e.main()) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try:return legacy.scheduled_run() +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try: backup_target(t,green_verify=False) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return _legacy_backup_target(t,green_verify) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return int(e.main()) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try:return legacy.scheduled_run() +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try: backup_target(t,green_verify=False) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return _legacy_backup_target(t,green_verify) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return int(e.main()) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try:return legacy.scheduled_run() +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try: backup_target(t,green_verify=False) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return _legacy_backup_target(t,green_verify) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: _need(p.returncode==0 and not p.stderr and 0 +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if __name__=='__main__': raise SystemExit(main()) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return int(e.main()) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return wrapper.main([sys.argv[0],*argv]) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if cmd=='scheduled':return _scheduled_locked() +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try:return legacy.scheduled_run() +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~~~~^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try: backup_target(t,green_verify=False) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return _legacy_backup_target(t,green_verify) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) +2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper +2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: _need(p.returncode==0 and not p.stderr and 0