From cf8c48db57af86fde92d1d60f1842f2a23697428 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Sun, 27 Sep 2026 18:29:19 +0000 Subject: [PATCH] runtime: publish SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1 --- ...-BACKUP-CT110-FRESH-HELPER-RCA-1222R1.json | 38 + ...B-BACKUP-CT110-FRESH-HELPER-RCA-1222R1.txt | 48 + runtime/latest.json | 16 +- runtime/latest.txt | 1650 +---------------- 4 files changed, 110 insertions(+), 1642 deletions(-) create mode 100644 runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1.json create mode 100644 runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1.txt diff --git a/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1.json b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1.json new file mode 100644 index 00000000..e3281a3a --- /dev/null +++ b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-ct110-fresh-helper-rca", + "started_at_utc": "2026-09-27T18:29:18Z", + "finished_at_utc": "2026-09-27T18:29:18Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "ab00b7d1889981b4cc323889177dbd754141ec085c7c285248721efca35ab7df", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "767dcb094f402e94fdcbac522590466690a8d61219a0a78266cb811a2f7399c1", + "sanitized_output_sha256": "767dcb094f402e94fdcbac522590466690a8d61219a0a78266cb811a2f7399c1", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "BACKUP_CT110_FRESH_HELPER_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1\nMODE=read-only\nCOMPONENT=homelab-backup-ct110-fresh-helper-rca\nMUTATION_BOUNDARY=ONLY_TEMP_DIAGNOSTIC_SCRATCH_UNDER_RUN;NO_CAPTURE_PVE_GUEST;NO_VZDUMP;NO_BACKUP_UPLOAD;NO_RETENTION;NO_AUDIT_RUN;NO_SYSTEMD_ACTION;NO_LOCK_BREAK;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1221_STATUS=OK TS=2026-07-10T23:30:49Z TYPE=cluster-admin-incident-engine-backup BAD=0 JOB=homelab-pve01-app-cluster-admin-incident-engine-20260710T232958Z ARCHIVE_SIZE_BYTES=157380 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK POST_FIX_CODE=included\nSOURCE_1221_RC=0\nSOURCE_1221_DECISION=PASS_BACKUP_V2_CURRENT_FAILURE_RCA_COLLECTED\nSOURCE_1221_NEXT_GATE=READ_EXACT_1221_AND_FIX_ONLY_PROVEN_BACKUP_CAUSE\nDECISION=FAIL_1222_PRECONDITION\nNEXT_GATE=FIX_ONLY_FAILED_1222_PRECONDITION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1.txt b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1.txt new file mode 100644 index 00000000..dea6d341 --- /dev/null +++ b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1.txt @@ -0,0 +1,48 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-ct110-fresh-helper-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=ab00b7d1889981b4cc323889177dbd754141ec085c7c285248721efca35ab7df +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=767dcb094f402e94fdcbac522590466690a8d61219a0a78266cb811a2f7399c1 +SANITIZED_OUTPUT_SHA256=767dcb094f402e94fdcbac522590466690a8d61219a0a78266cb811a2f7399c1 +OUTPUT_BEGIN +BACKUP_CT110_FRESH_HELPER_RCA_BEGIN=true +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1 +MODE=read-only +COMPONENT=homelab-backup-ct110-fresh-helper-rca +MUTATION_BOUNDARY=ONLY_TEMP_DIAGNOSTIC_SCRATCH_UNDER_RUN;NO_CAPTURE_PVE_GUEST;NO_VZDUMP;NO_BACKUP_UPLOAD;NO_RETENTION;NO_AUDIT_RUN;NO_SYSTEMD_ACTION;NO_LOCK_BREAK;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE +=== AUTHORITY === +HOST=pve01 +CLUSTER_QUORATE=Yes +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +SOURCE_1221_STATUS=OK TS=2026-07-10T23:30:49Z TYPE=cluster-admin-incident-engine-backup BAD=0 JOB=homelab-pve01-app-cluster-admin-incident-engine-20260710T232958Z ARCHIVE_SIZE_BYTES=157380 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK POST_FIX_CODE=included +SOURCE_1221_RC=0 +SOURCE_1221_DECISION=PASS_BACKUP_V2_CURRENT_FAILURE_RCA_COLLECTED +SOURCE_1221_NEXT_GATE=READ_EXACT_1221_AND_FIX_ONLY_PROVEN_BACKUP_CAUSE +DECISION=FAIL_1222_PRECONDITION +NEXT_GATE=FIX_ONLY_FAILED_1222_PRECONDITION +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index a272e7d1..e3281a3a 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260927-HOMELAB-BACKUP-V2-CURRENT-FAILURE-RCA-1221R1", + "command_id": "SUPPORT-260927-HOMELAB-BACKUP-CT110-FRESH-HELPER-RCA-1222R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "homelab-backup-v2-current-failure-rca", - "started_at_utc": "2026-09-27T18:22:59Z", - "finished_at_utc": "2026-09-27T18:23:07Z", + "component": "homelab-backup-ct110-fresh-helper-rca", + "started_at_utc": "2026-09-27T18:29:18Z", + "finished_at_utc": "2026-09-27T18:29:18Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "d4cfa85a3f59165d51b64505096167a15ce486f1a2105a55cd0c206b16d5353a", + "command_sha256": "ab00b7d1889981b4cc323889177dbd754141ec085c7c285248721efca35ab7df", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "de7467ab7d2c1584968349e75f34c054b7b406741bd4ab8a3a6e0e559ecf7d2a", - "sanitized_output_sha256": "ebca4b307aeed79780cce791fc44145115b68a88d817904d875aaae99b5dbbef", + "raw_evidence_sha256": "767dcb094f402e94fdcbac522590466690a8d61219a0a78266cb811a2f7399c1", + "sanitized_output_sha256": "767dcb094f402e94fdcbac522590466690a8d61219a0a78266cb811a2f7399c1", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "BACKUP_V2_CURRENT_FAILURE_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-V2-CURRENT-FAILURE-RCA-1221R1\nMODE=read-only\nCOMPONENT=homelab-backup-v2-current-failure-rca\nMUTATION_BOUNDARY=NONE_READ_ONLY_NO_SYSTEMD_ACTION_NO_BACKUP_NO_AUDIT_RUN_NO_LOCK_BREAK_NO_RETENTION_NO_DELETE_NO_MAIL_NO_GIT_WRITE_NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED\nREFERENCE_SHA256_EXPECTED=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED\nERROR_REGISTER_SHA256_EXPECTED=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\n=== IMMUTABLE_HISTORY ===\nSOURCE_1211_STATUS=OK\nSOURCE_1211_RC=0\nSOURCE_1211_DECISION=PASS_BACKUP_RPO_ADAPTIVE_CLOSE\nSOURCE_1211_NEXT_GATE=NONE\nSOURCE_1211_TASK_RESULT=PASS_HOMELAB_BACKUP_RPO_ADAPTIVE_CLOSE\nSOURCE_1220_STATUS=OK\nSOURCE_1220_RC=0\nSOURCE_1220_DECISION=PASS_BACKUP_ALERT_DEBOUNCE\nSOURCE_1220_NEXT_GATE=NONE\nSOURCE_73337_STATUS=OK\nSOURCE_73337_DECISION=PASS_NETBIRD_SKLADCHIK_POSTREPAIR_FINAL_ACCEPTANCE\nSOURCE_73337_NEXT_GATE=NONE\nSOURCE_CONTEXT_PRESENT=true\nSOURCE_CONTEXT_BACKUP_V2_FAILED=true\nSOURCE_CONTEXT_BACKUP_AUDIT_FAILED=true\n=== CURRENT_FAILED_UNITS ===\n● homelab-backup-audit.service loaded failed failed Homelab unified backup and DR audit\n● homelab-backup-v2.service loaded failed failed Homelab Backup V2 manifest-driven scheduler\n=== BACKUP_V2_SERVICE_SHOW ===\nResult=exit-code\nExecMainStartTimestamp=Sun 2026-09-27 21:08:52 MSK\nExecMainExitTimestamp=Sun 2026-09-27 21:08:59 MSK\nExecMainCode=1\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-backup-v2 ; argv[]=/usr/local/sbin/homelab-backup-v2 scheduled ; ignore_errors=no ; start_time=[Sun 2026-09-27 21:08:52 MSK] ; stop_time=[Sun 2026-09-27 21:08:59 MSK] ; pid=19346 ; code=exited ; status=1 }\nId=homelab-backup-v2.service\nDescription=Homelab Backup V2 manifest-driven scheduler\nLoadState=loaded\nActiveState=failed\nSubState=failed\nFragmentPath=/etc/systemd/system/homelab-backup-v2.service\nDropInPaths=\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Sun 2026-09-27 21:08:59 MSK\nBACKUP_V2_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service\nBACKUP_V2_UNIT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565\nBACKUP_V2_UNIT_META=644:root:root:2026-09-15 00:28:03.719796025 +0300\n=== BACKUP_V2_SERVICE_CAT ===\n# /etc/systemd/system/homelab-backup-v2.service\n[Unit]\nDescription=Homelab Backup V2 manifest-driven scheduler\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nUMask=0077\nExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nTimeoutStartSec=8h\nBACKUP_V2_EXECSTART_RAW={ path=/usr/local/sbin/homelab-backup-v2 ; argv[]=/usr/local/sbin/homelab-backup-v2 scheduled ; ignore_errors=no ; start_time=[Sun 2026-09-27 21:08:52 MSK] ; stop_time=[Sun 2026-09-27 21:08:59 MSK] ; pid=19346 ; code=exited ; status=1 }\nBACKUP_V2_EXECSTART_PATH=/usr/local/sbin/homelab-backup-v2\nBACKUP_V2_EXECSTART_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nBACKUP_V2_EXECSTART_META=750:root:root:2026-09-17 10:33:39.880371590 +0300\n=== BACKUP_V2_EXECSTART_SOURCE_BEGIN ===\n#!/usr/bin/python3\nfrom __future__ import annotations\nimport hashlib, importlib, os, pathlib, stat, sys\nACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b')\nEXPECTED={'fresh_lxc_helper.py': 'dbb3e6dbb71b10797df736431d9501a752a0f8c439163c9a0bddb7f76747f011', 'producer_exec_helper.py': 'b3b97099bcd7126670f866e81faae6a42a96d869d9989e6a67f6ea6c871c90b3', 'lxc_adapter.py': '3fa7d194010d76f7c8bc95035ae8876262e9337903e310f45f45df88c266dc8d', 'qemu_adapter.py': '1906db42bd34bbe85eb46fd9797b3572d46e0a41c29d1c29be5f9df5e448f15b', 'scheduler_adapter.py': 'a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96', 'entry504.py': 'ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793'}\ndef sha(p):\n h=hashlib.sha256()\n with open(p,'rb',buffering=0) as f:\n while True:\n b=f.read(1048576)\n if not b: break\n h.update(b)\n return h.hexdigest()\ndef main():\n st=os.lstat(ACCEPTED)\n if not stat.S_ISDIR(st.st_mode) or stat.S_ISLNK(st.st_mode) or st.st_uid!=0 or st.st_gid!=0 or stat.S_IMODE(st.st_mode)!=0o500: raise SystemExit('RELEASE_ROOT_IDENTITY')\n names=sorted(p.name for p in ACCEPTED.iterdir())\n if names!=sorted(EXPECTED): raise SystemExit('RELEASE_FILESET')\n for name,want in EXPECTED.items():\n p=ACCEPTED/name; s=os.lstat(p)\n if not stat.S_ISREG(s.st_mode) or stat.S_ISLNK(s.st_mode) or s.st_uid!=0 or s.st_gid!=0 or stat.S_IMODE(s.st_mode)!=0o400 or sha(p)!=want: raise SystemExit('RELEASE_FILE_IDENTITY_'+name)\n sys.dont_write_bytecode=True\n sys.path.insert(0,str(ACCEPTED))\n e=importlib.import_module('entry504')\n return int(e.main())\nif __name__=='__main__': raise SystemExit(main())\n=== BACKUP_V2_EXECSTART_SOURCE_END ===\n=== BACKUP_V2_TIMER_SHOW ===\nNextElapseUSecRealtime=\nLastTriggerUSec=Sun 2026-09-27 21:08:52 MSK\nResult=success\nId=homelab-backup-v2.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nFragmentPath=/etc/systemd/system/homelab-backup-v2.timer\nDropInPaths=\nUnitFileState=enabled\nBACKUP_V2_TIMER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.timer\nBACKUP_V2_TIMER_SHA256=5469057602452b303a01ccb8057085c1b0e447973e5ed198cebf7dd5daf58e9d\n=== BACKUP_V2_TIMER_CAT ===\n# /etc/systemd/system/homelab-backup-v2.timer\n[Unit]\nDescription=Homelab Backup V2 scheduler timer\n\n[Timer]\nOnActiveSec=20min\nOnUnitActiveSec=30min\nRandomizedDelaySec=10min\nUnit=homelab-backup-v2.service\n\n[Install]\nWantedBy=timers.target\n=== BACKUP_AUDIT_SERVICE_SHOW ===\nResult=exit-code\nExecMainStartTimestamp=Sun 2026-09-27 21:05:58 MSK\nExecMainExitTimestamp=Sun 2026-09-27 21:06:02 MSK\nExecMainCode=1\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-backup-audit ; argv[]=/usr/local/sbin/homelab-backup-audit scheduled ; ignore_errors=no ; start_time=[Sun 2026-09-27 21:05:58 MSK] ; stop_time=[Sun 2026-09-27 21:06:02 MSK] ; pid=16838 ; code=exited ; status=1 }\nId=homelab-backup-audit.service\nDescription=Homelab unified backup and DR audit\nLoadState=loaded\nActiveState=failed\nSubState=failed\nFragmentPath=/etc/systemd/system/homelab-backup-audit.service\nDropInPaths=\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Sun 2026-09-27 21:06:02 MSK\nBACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service\nBACKUP_AUDIT_UNIT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa\nBACKUP_AUDIT_UNIT_META=644:root:root:2026-08-11 08:46:44.076278347 +0300\n=== BACKUP_AUDIT_SERVICE_CAT ===\n# /etc/systemd/system/homelab-backup-audit.service\n[Unit]\nDescription=Homelab unified backup and DR audit\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-audit scheduled\nUser=root\nGroup=root\nUMask=0077\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nBACKUP_AUDIT_EXECSTART_RAW={ path=/usr/local/sbin/homelab-backup-audit ; argv[]=/usr/local/sbin/homelab-backup-audit scheduled ; ignore_errors=no ; start_time=[Sun 2026-09-27 21:05:58 MSK] ; stop_time=[Sun 2026-09-27 21:06:02 MSK] ; pid=16838 ; code=exited ; status=1 }\nBACKUP_AUDIT_EXECSTART_PATH=/usr/local/sbin/homelab-backup-audit\nBACKUP_AUDIT_EXECSTART_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nBACKUP_AUDIT_EXECSTART_META=755:root:root:2026-09-22 12:19:10.163905297 +0300\n=== BACKUP_AUDIT_EXECSTART_SOURCE_BEGIN ===\n#!/usr/bin/env python3\nimport sys,os,json,time,subprocess,pathlib,re,datetime\nMODE=sys.argv[1] if len(sys.argv)>1 else \"all\"\nif MODE not in (\"all\",\"write-health\",\"scheduled\",\"contract\"):\n print(\"AUDIT_ERROR=USAGE_all_write-health_scheduled_contract\",file=sys.stderr); sys.exit(64)\nif MODE==\"contract\":\n print(\"AUDIT_CONTRACT=v2-plus-sealed-quorum\")\n print(\"AUDIT_SCHEMA=homelab-backup-audit-v3\")\n sys.exit(0)\n\nREG=pathlib.Path(\"/etc/homelab-backup/targets.json\")\nSTATE=pathlib.Path(\"/var/lib/homelab-backup-v2/state\")\nQROOT=pathlib.Path(\"/var/lib/homelab-backup/cloud-quorum\")\nVMREST=pathlib.Path(\"/var/lib/homelab-backup/vm-restore-validation\")\nAPPREST=pathlib.Path(\"/var/lib/homelab-backup/app-restore-validation\")\nREGH=\"/usr/local/libexec/homelab-backup-target-registry\"\nAPPQ=\"/usr/local/sbin/homelab-app-cloud-quorum-queue\"\nGFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nGFSP=\"/etc/homelab-backup/gfs-retention.json\"\nRET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nWT=\"/root/homelab-ops-cr-2026-1005\"\nBRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nDRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nCLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nHEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\n\ndef run(cmd,timeout=60,env=None):\n try:\n p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)\n return p.returncode,p.stdout,p.stderr\n except subprocess.TimeoutExpired as e:\n return 124,e.stdout or \"\",e.stderr or \"\"\n\ndef collect(obj,out):\n if isinstance(obj,dict):\n if isinstance(obj.get(\"logical_id\"),str): out.append(obj)\n for v in obj.values(): collect(v,out)\n elif isinstance(obj,list):\n for v in obj: collect(v,out)\n\ndef parse_epoch(obj):\n for k in (\"sealed_epoch\",\"last_success_epoch\",\"updated_epoch\"):\n v=obj.get(k)\n if isinstance(v,(int,float)) and v>0: return int(v)\n for k in (\"sealed_at_utc\",\"updated_at_utc\",\"completed_at_utc\"):\n v=obj.get(k)\n if isinstance(v,str) and v:\n try:\n return int(datetime.datetime.fromisoformat(v.replace(\"Z\",\"+00:00\")).timestamp())\n except Exception: pass\n return None\n\ndef latest_legacy(logical,kind):\n d=QROOT/logical\n best=None; best_epoch=-1\n if not d.is_dir(): return None\n for p in d.glob(\"*.json\"):\n try: obj=json.loads(p.read_text(encoding=\"utf-8\"))\n except Exception: continue\n epoch=parse_epoch(obj)\n score=epoch if epoch is not None else int(p.stat().st_mtime)\n if score>best_epoch:\n best_epoch=score; best=(p,obj,score)\n if not best: return None\n p,obj,epoch=best\n gen=obj.get(\"generation\")\n strict=(obj.get(\"status\")==\"SEALED\" and obj.get(\"cloud_quorum\")==2 and obj.get(\"cloud_a_verified\") is True and obj.get(\"cloud_b_verified\") is True and obj.get(\"part_hashes_verified\") is True and obj.get(\"seal_written\") is True and obj.get(\"local_payload_present\") is False and obj.get(\"local_cleanup\")==\"OK\")\n rroot=VMREST if kind==\"VM\" else APPREST\n rev=(rroot/logical/(str(gen)+\".json\")) if gen else None\n restore=False\n if rev and rev.is_file():\n try:\n rj=json.loads(rev.read_text(encoding=\"utf-8\"))\n restore=(rj.get(\"result\")==\"OK\")\n except Exception: pass\n age=max(0,int(time.time()-epoch)) if epoch else None\n return {\"generation\":gen,\"strict\":strict,\"restore_ok\":restore,\"age_seconds\":age,\"state_file\":str(p),\"obj\":obj}\n\ndef unit_ok(unit):\n a=run([\"systemctl\",\"is-active\",unit],10)[1].strip()\n e=run([\"systemctl\",\"is-enabled\",unit],10)[1].strip()\n return a==\"active\" and e==\"enabled\",a,e\n\ndef parse_app_plan(txt):\n d={}\n for line in txt.splitlines():\n if line.startswith(\"QUEUE_FRESHNESS_PLAN APP=\"):\n r={}\n for tok in line.split()[1:]:\n if \"=\" in tok:\n k,v=tok.split(\"=\",1); r[k]=v\n if r.get(\"APP\"): d[r[\"APP\"]]=r.get(\"DECISION\")\n return d\n\nissues=[]; targets=[]; now=int(time.time())\ntry: robj=json.loads(REG.read_text(encoding=\"utf-8\"))\nexcept Exception:\n print(\"AUDIT_ERROR=REGISTRY_UNREADABLE\",file=sys.stderr); sys.exit(69)\nrows=[]; collect(robj,rows)\nactive={}\nfor r in rows:\n lid=r.get(\"logical_id\")\n if lid and str(r.get(\"lifecycle\",\"\")).upper()==\"ACTIVE\":\n active[lid]={\"logical_id\":lid,\"kind\":str(r.get(\"kind\",\"\")).upper(),\"id\":str(r.get(\"id\")) if r.get(\"id\") is not None else None}\n\nvmrc,vmo,_=run([REGH,\"freshness-seconds\",\"VM\"],20)\naprc,apo,_=run([REGH,\"freshness-seconds\",\"APP\"],20)\nvm_sla=int(vmo.strip()) if vmrc==0 and vmo.strip().isdigit() else 72000\napp_sla=int(apo.strip()) if aprc==0 and apo.strip().isdigit() else 72000\n\nrrc,rout,_=run([\"pvesh\",\"get\",\"/cluster/resources\",\"--type\",\"vm\",\"--output-format\",\"json\"],30)\nresources={}\nif rrc==0:\n try:\n for x in json.loads(rout): resources[str(x.get(\"vmid\"))]=x\n except Exception: pass\n\nqrc,qout,qerr=run([APPQ,\"freshness-plan\"],180)\napp_dec=parse_app_plan(qout if qrc==0 else \"\")\nif qrc!=0: issues.append({\"severity\":\"CRITICAL\",\"code\":\"APP_FRESHNESS_PLAN_FAILED\",\"logical_id\":\"-\",\"root_cause\":\"APP_QUEUE_RC_\"+str(qrc)})\n\nfor lid,r in sorted(active.items()):\n kind=r[\"kind\"]; ident=r[\"id\"]; rec={\"logical_id\":lid,\"kind\":kind,\"state\":\"UNVERIFIED\"}\n if kind==\"VM\":\n typ=str(resources.get(ident,{}).get(\"type\",\"qemu\"))\n sid=(\"ct\"+ident) if typ==\"lxc\" else (\"vm\"+ident)\n v2=None\n try: v2=json.loads((STATE/(sid+\".json\")).read_text(encoding=\"utf-8\"))\n except Exception: pass\n v2_ok=False; v2_age=None; repcount=None\n if isinstance(v2,dict):\n ls=v2.get(\"last_success_epoch\")\n v2_age=max(0,int(now-ls)) if isinstance(ls,(int,float)) and ls>0 else None\n reps=v2.get(\"replicas\"); repcount=len(reps) if isinstance(reps,(list,dict)) else None\n v2_ok=(v2.get(\"result\")==\"SUCCESS\" and v2_age is not None and v2_age=3:\n issues.append({\"severity\":\"HIGH\",\"code\":\"RETRY_EXHAUSTION\",\"logical_id\":lid,\"root_cause\":\"CONSECUTIVE_FAILURES_\"+str(cf)})\n legacy=latest_legacy(lid,\"VM\")\n legacy_ok=bool(legacy and legacy[\"strict\"] and legacy[\"restore_ok\"] and legacy[\"age_seconds\"] is not None and legacy[\"age_seconds\"]0 else None\n fresh=(obj.get(\"result\")==\"SUCCESS\" and age is not None and age<86400)\n rec.update({\"state\":\"FRESH\" if fresh else \"STALE\",\"age_seconds\":age,\"generation\":obj.get(\"generation\") if isinstance(obj,dict) else None})\n if not fresh: issues.append({\"severity\":\"HIGH\",\"code\":\"RPO_BREACH\",\"logical_id\":lid,\"root_cause\":\"CONTROL_STATE_NOT_FRESH\"})\n targets.append(rec)\n\nif len(targets)!=len(active):\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"LIFECYCLE_COVERAGE_GAP\",\"logical_id\":\"-\",\"root_cause\":\"ACTIVE_TARGET_COUNT_MISMATCH\"})\n\nfor unit,code in [\n (\"homelab-backup-v2.timer\",\"V2_TIMER_BAD\"),\n (\"homelab-app-cloud-quorum-queue.timer\",\"APP_TIMER_BAD\"),\n (\"homelab-backup-retention-gc.timer\",\"RETENTION_TIMER_BAD\"),\n (\"homelab-alertmanager-backup-health.timer\",\"ALERT_TIMER_BAD\"),\n]:\n ok,a,e=unit_ok(unit)\n if not ok: issues.append({\"severity\":\"HIGH\",\"code\":code,\"logical_id\":unit,\"root_cause\":\"UNIT_\"+a+\"_\"+e})\n\nfor lock in [\"/run/homelab-backup-heavy.lock\",\"/run/lock/homelab-app-cloud-quorum-queue.lock\",\"/run/lock/homelab-vm-cloud-quorum-queue.lock\",\"/run/lock/homelab-backup-dispatcher.lock\"]:\n p=pathlib.Path(lock)\n if p.exists():\n lrc,_,_=run([\"flock\",\"-n\",str(p),\"-c\",\"true\"],5)\n if lrc!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"LOCK_BUSY\",\"logical_id\":lock,\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\"})\n\nenv=os.environ.copy(); env[\"HOMELAB_BACKUP_GFS_POLICY\"]=GFSP; env[\"HOMELAB_BACKUP_TARGET_REGISTRY\"]=str(REG)\ngrc,gout,_=run([GFSH,\"validate\"],20,env)\nif grc!=0 or \"GFS_POLICY=OK\" not in gout:\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"GFS_INVALID\",\"logical_id\":\"-\",\"root_cause\":\"GFS_VALIDATE_FAILED\"})\nrr,_,_=run([RET,\"contract\"],120)\nif rr!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_CONTRACT_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_CONTRACT_RC_\"+str(rr)})\nrhp=pathlib.Path(\"/var/lib/homelab-health/backup-retention-safe-gc.json\")\nif not rhp.is_file():\n issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_FILE_MISSING\"})\nelse:\n try:\n rh=json.loads(rhp.read_text(encoding=\"utf-8\"))\n if rh.get(\"status\")!=\"OK\": issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_NOT_OK\"})\n except Exception: issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_UNREADABLE\"})\n\ncloud={}\nfor label,remote in [(\"A\",\"pve01-mail-01-crypt:homelab-backups\"),(\"B\",\"pve01-mail-02-crypt:homelab-backups\")]:\n lrc,lo,_=run([\"rclone\",\"lsf\",remote,\"--max-depth\",\"1\",\"--dirs-only\"],60)\n arc,ao,_=run([\"rclone\",\"about\",remote,\"--json\"],60)\n rec={\"readable\":lrc==0,\"top_dirs\":len([x for x in lo.splitlines() if x.strip()])}\n if arc==0:\n try:\n j=json.loads(ao); rec.update({k:j.get(k) for k in (\"total\",\"used\",\"free\")})\n if j.get(\"total\") and j.get(\"free\") is not None:\n frac=j[\"free\"]/j[\"total\"]; rec[\"free_fraction\"]=round(frac,6)\n if frac<0.15: issues.append({\"severity\":\"HIGH\",\"code\":\"CAPACITY_LOW\",\"logical_id\":\"cloud-\"+label,\"root_cause\":\"FREE_LT_15_PERCENT\"})\n except Exception: pass\n if lrc!=0: issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_UNREADABLE\",\"logical_id\":\"cloud-\"+label,\"root_cause\":\"RCLONE_LSF_FAILED\"})\n cloud[label]=rec\n\ntry: dr=json.loads(DRSEAL.read_text(encoding=\"utf-8\"))\nexcept Exception: dr=None\nif not isinstance(dr,dict) or dr.get(\"status\")!=\"SEALED\":\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"DR_SEAL_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"SITE_DR_FINAL_SEAL_NOT_SEALED\"})\n\ncloudb_ok=False\ntry:\n import pathlib as _pathlib, json as _json, subprocess as _subprocess, hashlib as _hashlib\n _state_root=_pathlib.Path(\"/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor\")\n _restore_root=_pathlib.Path(\"/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor\")\n _states=sorted([p for p in _state_root.glob(\"*.json\") if p.is_file() and not p.is_symlink()],key=lambda p:p.stat().st_mtime,reverse=True)\n for _sp in _states:\n try: _st=_json.loads(_sp.read_text(encoding=\"utf-8\"))\n except Exception: continue\n if not (\n _st.get(\"schema\")==\"homelab-cloud-quorum-payload-v1\" and\n _st.get(\"logical_id\")==\"app-skladchik-reports-monitor\" and\n _st.get(\"status\")==\"SEALED\" and _st.get(\"cloud_quorum\")==2 and\n _st.get(\"cloud_a_verified\") is True and _st.get(\"cloud_b_verified\") is True and\n _st.get(\"part_hashes_verified\") is True and _st.get(\"seal_written\") is True and\n _st.get(\"cloud_a_seal_verified\") is True and _st.get(\"cloud_b_seal_verified\") is True and\n _st.get(\"local_payload_present\") is False and _st.get(\"local_cleanup\")==\"OK\"):\n continue\n _gen=_st.get(\"generation\"); _archive_sha=_st.get(\"archive_sha256\")\n _manifest_sha=_st.get(\"parts_manifest_sha256\"); _part_count=_st.get(\"part_count\")\n _archive_size=_st.get(\"archive_size_bytes\"); _remote_b=_st.get(\"remote_b_base\")\n if not (isinstance(_gen,str) and isinstance(_archive_sha,str) and len(_archive_sha)==64 and\n isinstance(_manifest_sha,str) and len(_manifest_sha)==64 and isinstance(_part_count,int) and\n _part_count>0 and isinstance(_archive_size,int) and _archive_size>0 and isinstance(_remote_b,str)):\n continue\n _evp=_restore_root/(_gen+\".json\")\n try: _ev=_json.loads(_evp.read_text(encoding=\"utf-8\"))\n except Exception: continue\n if not (\n _ev.get(\"schema\")==\"homelab-app-restore-validation-v1\" and\n _ev.get(\"logical_id\")==\"app-skladchik-reports-monitor\" and\n _ev.get(\"generation\")==_gen and _ev.get(\"transport_state\")==\"SEALED\" and\n _ev.get(\"restore_source\")==\"Cloud A\" and _ev.get(\"payload_sha256\")==_archive_sha and\n _ev.get(\"payload_sha256_verified\") is True and _ev.get(\"result\")==\"OK\"):\n continue\n _mp=_subprocess.run([\"rclone\",\"cat\",_remote_b.rstrip(\"/\")+\"/PARTS.tsv\"],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=60)\n if _mp.returncode!=0 or _hashlib.sha256(_mp.stdout).hexdigest()!=_manifest_sha: continue\n _rows=[]; _valid=True\n try:\n for _line in _mp.stdout.decode(\"utf-8\",\"strict\").splitlines():\n if not _line: continue\n _f=_line.split(\"\\t\")\n if len(_f)!=4: _valid=False; break\n _idx,_name,_size_s,_psha=_f\n if not (_idx.isdigit() and len(_idx)==8 and _size_s.isdigit() and len(_psha)==64):\n _valid=False; break\n _size=int(_size_s)\n _pp=_subprocess.run([\"rclone\",\"cat\",_remote_b.rstrip(\"/\")+\"/payload.parts/\"+_name],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=120)\n if _pp.returncode!=0 or len(_pp.stdout)!=_size or _hashlib.sha256(_pp.stdout).hexdigest()!=_psha:\n _valid=False; break\n _rows.append((_size,_pp.stdout))\n except Exception:\n _valid=False\n if not _valid or len(_rows)!=_part_count or sum(x[0] for x in _rows)!=_archive_size: continue\n _payload=b\"\".join(x[1] for x in _rows)\n if _hashlib.sha256(_payload).hexdigest()!=_archive_sha: continue\n cloudb_ok=True\n break\nexcept Exception:\n cloudb_ok=False\nif not cloudb_ok:\n issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK\"})\n\nsource={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\nhrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\nif hrc==0: source[\"local_head\"]=ho.strip()\nfrc,fo,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.url\"],10)\nprc,po,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.pushurl\"],10)\nsource[\"pushurl_disabled\"]=(prc==0 and po.strip()==\"DISABLED\")\nif frc==0 and fo.strip():\n lrc,lo,_=run([\"git\",\"ls-remote\",fo.strip(),\"refs/heads/\"+BRANCH],30)\n if lrc==0 and lo.strip(): source[\"remote_head\"]=lo.split()[0]\nif source[\"remote_head\"]!=source[\"local_head\"]:\n issues.append({\"severity\":\"MEDIUM\",\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\"})\n\nstatus=\"GREEN\" if not issues else \"RED\"\nfor t in targets:\n print(\"TARGET|\"+\"|\".join(f\"{k}={v}\" for k,v in t.items() if v is not None))\nfor i in issues:\n print(\"ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}\".format(**i))\nprint(f\"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}\")\nhealth={\"schema\":\"homelab-backup-audit-v3\",\"status\":status,\"generated_at_epoch\":now,\"targets\":targets,\"issues\":issues,\"cloud\":cloud,\"source\":source,\"cloud_b_restore_drill\":cloudb_ok,\"dr_proven\":bool(isinstance(dr,dict) and dr.get(\"status\")==\"SEALED\")}\nif MODE in (\"write-health\",\"scheduled\"):\n for hp in HEALTHS:\n hp.parent.mkdir(parents=True,exist_ok=True)\n tmp=hp.with_name(hp.name+\".tmp\")\n tmp.write_text(json.dumps(health,separators=(\",\",\":\"),sort_keys=True)+\"\\n\",encoding=\"utf-8\")\n os.chmod(tmp,0o644); os.replace(tmp,hp)\nsys.exit(0 if not issues else 1)\n=== BACKUP_AUDIT_EXECSTART_SOURCE_END ===\n=== BACKUP_AUDIT_TIMER_SHOW ===\nNextElapseUSecRealtime=Sun 2026-09-27 21:36:58 MSK\nLastTriggerUSec=Sun 2026-09-27 21:05:58 MSK\nResult=success\nId=homelab-backup-audit.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nFragmentPath=/etc/systemd/system/homelab-backup-audit.timer\nDropInPaths=\nUnitFileState=enabled\nBACKUP_AUDIT_TIMER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.timer\nBACKUP_AUDIT_TIMER_SHA256=987695f83ee993bd4d716b8365308211110ac4c684f0bd9b17dfdb1e523e466a\n=== BACKUP_AUDIT_TIMER_CAT ===\n# /etc/systemd/system/homelab-backup-audit.timer\n[Unit]\nDescription=Run homelab unified backup and DR audit\n\n[Timer]\nOnCalendar=*-*-* *:05,35:00\nRandomizedDelaySec=2min\nPersistent=true\nAccuracySec=1min\nUnit=homelab-backup-audit.service\n\n[Install]\nWantedBy=timers.target\n=== BACKUP_V2_JOURNAL_LAST12H ===\n2026-09-27T16:56:08.780555+03:00 pve01 homelab-backup-v2[3412388]: lxc_adapter.AdapterError: FRESH_HELPER_FAILED\n2026-09-27T16:56:08.785999+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T16:56:08.786073+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\n2026-09-27T16:56:08.786211+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-27T16:56:08.786299+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 7.475s CPU time, 159M memory peak.\n2026-09-27T17:32:01.150519+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-27T17:32:01.367425+03:00 pve01 homelab-backup-v2[3534166]: SELFTEST_REPLACEMENT486=PASS\n2026-09-27T17:32:08.781016+03:00 pve01 homelab-backup-v2[3534166]: SCHEDULE_TARGET=ct110\n2026-09-27T17:32:08.788194+03:00 pve01 homelab-backup-v2[3534166]: Traceback (most recent call last):\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/usr/local/sbin/homelab-backup-v2\", line 26, in \n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return int(e.main())\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try:return legacy.scheduled_run()\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try: backup_target(t,green_verify=False)\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return _legacy_backup_target(t,green_verify)\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return int(e.main())\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try:return legacy.scheduled_run()\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try: backup_target(t,green_verify=False)\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return _legacy_backup_target(t,green_verify)\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return int(e.main())\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try:return legacy.scheduled_run()\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try: backup_target(t,green_verify=False)\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return _legacy_backup_target(t,green_verify)\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return int(e.main())\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try:return legacy.scheduled_run()\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try: backup_target(t,green_verify=False)\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return _legacy_backup_target(t,green_verify)\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return int(e.main())\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try:return legacy.scheduled_run()\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try: backup_target(t,green_verify=False)\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return _legacy_backup_target(t,green_verify)\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return int(e.main())\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try:return legacy.scheduled_run()\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try: backup_target(t,green_verify=False)\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return _legacy_backup_target(t,green_verify)\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: _need(p.returncode==0 and not p.stderr and 0\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if __name__=='__main__': raise SystemExit(main())\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/usr/local/sbin/homelab-backup-v2\", line 25, in main\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return int(e.main())\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py\", line 186, in main\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return wrapper.main([sys.argv[0],*argv])\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 94, in main\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if cmd=='scheduled':return _scheduled_locked()\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 85, in _scheduled_locked\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try:return legacy.scheduled_run()\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~~~~^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 960, in scheduled_run\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try: backup_target(t,green_verify=False)\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py\", line 42, in backup_target\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return _legacy_backup_target(t,green_verify)\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 908, in backup_target\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py\", line 873, in capture_target\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 204, in capture_pve_guest\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes'])\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 85, in plan_only\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch))\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File \"/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py\", line 69, in _helper\n2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: _need(p.returncode==0 and not p.stderr and 01 else "all" -if MODE not in ("all","write-health","scheduled","contract"): - print("AUDIT_ERROR=USAGE_all_write-health_scheduled_contract",file=sys.stderr); sys.exit(64) -if MODE=="contract": - print("AUDIT_CONTRACT=v2-plus-sealed-quorum") - print("AUDIT_SCHEMA=homelab-backup-audit-v3") - sys.exit(0) - -REG=pathlib.Path("/etc/homelab-backup/targets.json") -STATE=pathlib.Path("/var/lib/homelab-backup-v2/state") -QROOT=pathlib.Path("/var/lib/homelab-backup/cloud-quorum") -VMREST=pathlib.Path("/var/lib/homelab-backup/vm-restore-validation") -APPREST=pathlib.Path("/var/lib/homelab-backup/app-restore-validation") -REGH="/usr/local/libexec/homelab-backup-target-registry" -APPQ="/usr/local/sbin/homelab-app-cloud-quorum-queue" -GFSH="/usr/local/libexec/homelab-backup-gfs-policy" -GFSP="/etc/homelab-backup/gfs-retention.json" -RET="/usr/local/sbin/homelab-backup-retention-gc" -WT="/root/homelab-ops-cr-2026-1005" -BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" -DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") -CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") -HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] - -def run(cmd,timeout=60,env=None): - try: - p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env) - return p.returncode,p.stdout,p.stderr - except subprocess.TimeoutExpired as e: - return 124,e.stdout or "",e.stderr or "" - -def collect(obj,out): - if isinstance(obj,dict): - if isinstance(obj.get("logical_id"),str): out.append(obj) - for v in obj.values(): collect(v,out) - elif isinstance(obj,list): - for v in obj: collect(v,out) - -def parse_epoch(obj): - for k in ("sealed_epoch","last_success_epoch","updated_epoch"): - v=obj.get(k) - if isinstance(v,(int,float)) and v>0: return int(v) - for k in ("sealed_at_utc","updated_at_utc","completed_at_utc"): - v=obj.get(k) - if isinstance(v,str) and v: - try: - return int(datetime.datetime.fromisoformat(v.replace("Z","+00:00")).timestamp()) - except Exception: pass - return None - -def latest_legacy(logical,kind): - d=QROOT/logical - best=None; best_epoch=-1 - if not d.is_dir(): return None - for p in d.glob("*.json"): - try: obj=json.loads(p.read_text(encoding="utf-8")) - except Exception: continue - epoch=parse_epoch(obj) - score=epoch if epoch is not None else int(p.stat().st_mtime) - if score>best_epoch: - best_epoch=score; best=(p,obj,score) - if not best: return None - p,obj,epoch=best - gen=obj.get("generation") - strict=(obj.get("status")=="SEALED" and obj.get("cloud_quorum")==2 and obj.get("cloud_a_verified") is True and obj.get("cloud_b_verified") is True and obj.get("part_hashes_verified") is True and obj.get("seal_written") is True and obj.get("local_payload_present") is False and obj.get("local_cleanup")=="OK") - rroot=VMREST if kind=="VM" else APPREST - rev=(rroot/logical/(str(gen)+".json")) if gen else None - restore=False - if rev and rev.is_file(): - try: - rj=json.loads(rev.read_text(encoding="utf-8")) - restore=(rj.get("result")=="OK") - except Exception: pass - age=max(0,int(time.time()-epoch)) if epoch else None - return {"generation":gen,"strict":strict,"restore_ok":restore,"age_seconds":age,"state_file":str(p),"obj":obj} - -def unit_ok(unit): - a=run(["systemctl","is-active",unit],10)[1].strip() - e=run(["systemctl","is-enabled",unit],10)[1].strip() - return a=="active" and e=="enabled",a,e - -def parse_app_plan(txt): - d={} - for line in txt.splitlines(): - if line.startswith("QUEUE_FRESHNESS_PLAN APP="): - r={} - for tok in line.split()[1:]: - if "=" in tok: - k,v=tok.split("=",1); r[k]=v - if r.get("APP"): d[r["APP"]]=r.get("DECISION") - return d - -issues=[]; targets=[]; now=int(time.time()) -try: robj=json.loads(REG.read_text(encoding="utf-8")) -except Exception: - print("AUDIT_ERROR=REGISTRY_UNREADABLE",file=sys.stderr); sys.exit(69) -rows=[]; collect(robj,rows) -active={} -for r in rows: - lid=r.get("logical_id") - if lid and str(r.get("lifecycle","")).upper()=="ACTIVE": - active[lid]={"logical_id":lid,"kind":str(r.get("kind","")).upper(),"id":str(r.get("id")) if r.get("id") is not None else None} - -vmrc,vmo,_=run([REGH,"freshness-seconds","VM"],20) -aprc,apo,_=run([REGH,"freshness-seconds","APP"],20) -vm_sla=int(vmo.strip()) if vmrc==0 and vmo.strip().isdigit() else 72000 -app_sla=int(apo.strip()) if aprc==0 and apo.strip().isdigit() else 72000 - -rrc,rout,_=run(["pvesh","get","/cluster/resources","--type","vm","--output-format","json"],30) -resources={} -if rrc==0: - try: - for x in json.loads(rout): resources[str(x.get("vmid"))]=x - except Exception: pass - -qrc,qout,qerr=run([APPQ,"freshness-plan"],180) -app_dec=parse_app_plan(qout if qrc==0 else "") -if qrc!=0: issues.append({"severity":"CRITICAL","code":"APP_FRESHNESS_PLAN_FAILED","logical_id":"-","root_cause":"APP_QUEUE_RC_"+str(qrc)}) - -for lid,r in sorted(active.items()): - kind=r["kind"]; ident=r["id"]; rec={"logical_id":lid,"kind":kind,"state":"UNVERIFIED"} - if kind=="VM": - typ=str(resources.get(ident,{}).get("type","qemu")) - sid=("ct"+ident) if typ=="lxc" else ("vm"+ident) - v2=None - try: v2=json.loads((STATE/(sid+".json")).read_text(encoding="utf-8")) - except Exception: pass - v2_ok=False; v2_age=None; repcount=None - if isinstance(v2,dict): - ls=v2.get("last_success_epoch") - v2_age=max(0,int(now-ls)) if isinstance(ls,(int,float)) and ls>0 else None - reps=v2.get("replicas"); repcount=len(reps) if isinstance(reps,(list,dict)) else None - v2_ok=(v2.get("result")=="SUCCESS" and v2_age is not None and v2_age=3: - issues.append({"severity":"HIGH","code":"RETRY_EXHAUSTION","logical_id":lid,"root_cause":"CONSECUTIVE_FAILURES_"+str(cf)}) - legacy=latest_legacy(lid,"VM") - legacy_ok=bool(legacy and legacy["strict"] and legacy["restore_ok"] and legacy["age_seconds"] is not None and legacy["age_seconds"]0 else None - fresh=(obj.get("result")=="SUCCESS" and age is not None and age<86400) - rec.update({"state":"FRESH" if fresh else "STALE","age_seconds":age,"generation":obj.get("generation") if isinstance(obj,dict) else None}) - if not fresh: issues.append({"severity":"HIGH","code":"RPO_BREACH","logical_id":lid,"root_cause":"CONTROL_STATE_NOT_FRESH"}) - targets.append(rec) - -if len(targets)!=len(active): - issues.append({"severity":"CRITICAL","code":"LIFECYCLE_COVERAGE_GAP","logical_id":"-","root_cause":"ACTIVE_TARGET_COUNT_MISMATCH"}) - -for unit,code in [ - ("homelab-backup-v2.timer","V2_TIMER_BAD"), - ("homelab-app-cloud-quorum-queue.timer","APP_TIMER_BAD"), - ("homelab-backup-retention-gc.timer","RETENTION_TIMER_BAD"), - ("homelab-alertmanager-backup-health.timer","ALERT_TIMER_BAD"), -]: - ok,a,e=unit_ok(unit) - if not ok: issues.append({"severity":"HIGH","code":code,"logical_id":unit,"root_cause":"UNIT_"+a+"_"+e}) - -for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]: - p=pathlib.Path(lock) - if p.exists(): - lrc,_,_=run(["flock","-n",str(p),"-c","true"],5) - if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"}) - -env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG) -grc,gout,_=run([GFSH,"validate"],20,env) -if grc!=0 or "GFS_POLICY=OK" not in gout: - issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"}) -rr,_,_=run([RET,"contract"],120) -if rr!=0: issues.append({"severity":"HIGH","code":"RETENTION_CONTRACT_BAD","logical_id":"-","root_cause":"RETENTION_CONTRACT_RC_"+str(rr)}) -rhp=pathlib.Path("/var/lib/homelab-health/backup-retention-safe-gc.json") -if not rhp.is_file(): - issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_MISSING","logical_id":"-","root_cause":"RETENTION_HEALTH_FILE_MISSING"}) -else: - try: - rh=json.loads(rhp.read_text(encoding="utf-8")) - if rh.get("status")!="OK": issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_NOT_OK"}) - except Exception: issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_UNREADABLE"}) - -cloud={} -for label,remote in [("A","pve01-mail-01-crypt:homelab-backups"),("B","pve01-mail-02-crypt:homelab-backups")]: - lrc,lo,_=run(["rclone","lsf",remote,"--max-depth","1","--dirs-only"],60) - arc,ao,_=run(["rclone","about",remote,"--json"],60) - rec={"readable":lrc==0,"top_dirs":len([x for x in lo.splitlines() if x.strip()])} - if arc==0: - try: - j=json.loads(ao); rec.update({k:j.get(k) for k in ("total","used","free")}) - if j.get("total") and j.get("free") is not None: - frac=j["free"]/j["total"]; rec["free_fraction"]=round(frac,6) - if frac<0.15: issues.append({"severity":"HIGH","code":"CAPACITY_LOW","logical_id":"cloud-"+label,"root_cause":"FREE_LT_15_PERCENT"}) - except Exception: pass - if lrc!=0: issues.append({"severity":"CRITICAL","code":"CLOUD_UNREADABLE","logical_id":"cloud-"+label,"root_cause":"RCLONE_LSF_FAILED"}) - cloud[label]=rec - -try: dr=json.loads(DRSEAL.read_text(encoding="utf-8")) -except Exception: dr=None -if not isinstance(dr,dict) or dr.get("status")!="SEALED": - issues.append({"severity":"CRITICAL","code":"DR_SEAL_MISSING","logical_id":"-","root_cause":"SITE_DR_FINAL_SEAL_NOT_SEALED"}) - -cloudb_ok=False -try: - import pathlib as _pathlib, json as _json, subprocess as _subprocess, hashlib as _hashlib - _state_root=_pathlib.Path("/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor") - _restore_root=_pathlib.Path("/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor") - _states=sorted([p for p in _state_root.glob("*.json") if p.is_file() and not p.is_symlink()],key=lambda p:p.stat().st_mtime,reverse=True) - for _sp in _states: - try: _st=_json.loads(_sp.read_text(encoding="utf-8")) - except Exception: continue - if not ( - _st.get("schema")=="homelab-cloud-quorum-payload-v1" and - _st.get("logical_id")=="app-skladchik-reports-monitor" and - _st.get("status")=="SEALED" and _st.get("cloud_quorum")==2 and - _st.get("cloud_a_verified") is True and _st.get("cloud_b_verified") is True and - _st.get("part_hashes_verified") is True and _st.get("seal_written") is True and - _st.get("cloud_a_seal_verified") is True and _st.get("cloud_b_seal_verified") is True and - _st.get("local_payload_present") is False and _st.get("local_cleanup")=="OK"): - continue - _gen=_st.get("generation"); _archive_sha=_st.get("archive_sha256") - _manifest_sha=_st.get("parts_manifest_sha256"); _part_count=_st.get("part_count") - _archive_size=_st.get("archive_size_bytes"); _remote_b=_st.get("remote_b_base") - if not (isinstance(_gen,str) and isinstance(_archive_sha,str) and len(_archive_sha)==64 and - isinstance(_manifest_sha,str) and len(_manifest_sha)==64 and isinstance(_part_count,int) and - _part_count>0 and isinstance(_archive_size,int) and _archive_size>0 and isinstance(_remote_b,str)): - continue - _evp=_restore_root/(_gen+".json") - try: _ev=_json.loads(_evp.read_text(encoding="utf-8")) - except Exception: continue - if not ( - _ev.get("schema")=="homelab-app-restore-validation-v1" and - _ev.get("logical_id")=="app-skladchik-reports-monitor" and - _ev.get("generation")==_gen and _ev.get("transport_state")=="SEALED" and - _ev.get("restore_source")=="Cloud A" and _ev.get("payload_sha256")==_archive_sha and - _ev.get("payload_sha256_verified") is True and _ev.get("result")=="OK"): - continue - _mp=_subprocess.run(["rclone","cat",_remote_b.rstrip("/")+"/PARTS.tsv"],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=60) - if _mp.returncode!=0 or _hashlib.sha256(_mp.stdout).hexdigest()!=_manifest_sha: continue - _rows=[]; _valid=True - try: - for _line in _mp.stdout.decode("utf-8","strict").splitlines(): - if not _line: continue - _f=_line.split("\t") - if len(_f)!=4: _valid=False; break - _idx,_name,_size_s,_psha=_f - if not (_idx.isdigit() and len(_idx)==8 and _size_s.isdigit() and len(_psha)==64): - _valid=False; break - _size=int(_size_s) - _pp=_subprocess.run(["rclone","cat",_remote_b.rstrip("/")+"/payload.parts/"+_name],stdout=_subprocess.PIPE,stderr=_subprocess.PIPE,timeout=120) - if _pp.returncode!=0 or len(_pp.stdout)!=_size or _hashlib.sha256(_pp.stdout).hexdigest()!=_psha: - _valid=False; break - _rows.append((_size,_pp.stdout)) - except Exception: - _valid=False - if not _valid or len(_rows)!=_part_count or sum(x[0] for x in _rows)!=_archive_size: continue - _payload=b"".join(x[1] for x in _rows) - if _hashlib.sha256(_payload).hexdigest()!=_archive_sha: continue - cloudb_ok=True - break -except Exception: - cloudb_ok=False -if not cloudb_ok: - issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"}) - -source={"local_head":None,"remote_head":None,"pushurl_disabled":False} -hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) -if hrc==0: source["local_head"]=ho.strip() -frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) -prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) -source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") -if frc==0 and fo.strip(): - lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) - if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] -if source["remote_head"]!=source["local_head"]: - issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) - -status="GREEN" if not issues else "RED" -for t in targets: - print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None)) -for i in issues: - print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i)) -print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}") -health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")} -if MODE in ("write-health","scheduled"): - for hp in HEALTHS: - hp.parent.mkdir(parents=True,exist_ok=True) - tmp=hp.with_name(hp.name+".tmp") - tmp.write_text(json.dumps(health,separators=(",",":"),sort_keys=True)+"\n",encoding="utf-8") - os.chmod(tmp,0o644); os.replace(tmp,hp) -sys.exit(0 if not issues else 1) -=== BACKUP_AUDIT_EXECSTART_SOURCE_END === -=== BACKUP_AUDIT_TIMER_SHOW === -NextElapseUSecRealtime=Sun 2026-09-27 21:36:58 MSK -LastTriggerUSec=Sun 2026-09-27 21:05:58 MSK -Result=success -Id=homelab-backup-audit.timer -LoadState=loaded -ActiveState=active -SubState=waiting -FragmentPath=/etc/systemd/system/homelab-backup-audit.timer -DropInPaths= -UnitFileState=enabled -BACKUP_AUDIT_TIMER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.timer -BACKUP_AUDIT_TIMER_SHA256=987695f83ee993bd4d716b8365308211110ac4c684f0bd9b17dfdb1e523e466a -=== BACKUP_AUDIT_TIMER_CAT === -# /etc/systemd/system/homelab-backup-audit.timer -[Unit] -Description=Run homelab unified backup and DR audit - -[Timer] -OnCalendar=*-*-* *:05,35:00 -RandomizedDelaySec=2min -Persistent=true -AccuracySec=1min -Unit=homelab-backup-audit.service - -[Install] -WantedBy=timers.target -=== BACKUP_V2_JOURNAL_LAST12H === -2026-09-27T16:56:08.780555+03:00 pve01 homelab-backup-v2[3412388]: lxc_adapter.AdapterError: FRESH_HELPER_FAILED -2026-09-27T16:56:08.785999+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=1/FAILURE -2026-09-27T16:56:08.786073+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. -2026-09-27T16:56:08.786211+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. -2026-09-27T16:56:08.786299+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 7.475s CPU time, 159M memory peak. -2026-09-27T17:32:01.150519+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... -2026-09-27T17:32:01.367425+03:00 pve01 homelab-backup-v2[3534166]: SELFTEST_REPLACEMENT486=PASS -2026-09-27T17:32:08.781016+03:00 pve01 homelab-backup-v2[3534166]: SCHEDULE_TARGET=ct110 -2026-09-27T17:32:08.788194+03:00 pve01 homelab-backup-v2[3534166]: Traceback (most recent call last): -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/usr/local/sbin/homelab-backup-v2", line 26, in -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if __name__=='__main__': raise SystemExit(main()) -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~^^ -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return int(e.main()) -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~^^ -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return wrapper.main([sys.argv[0],*argv]) -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if cmd=='scheduled':return _scheduled_locked() -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^ -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try:return legacy.scheduled_run() -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~~~~^^ -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: try: backup_target(t,green_verify=False) -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: return _legacy_backup_target(t,green_verify) -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) -2026-09-27T17:32:08.790489+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only -2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) -2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper -2026-09-27T17:32:08.790789+03:00 pve01 homelab-backup-v2[3534166]: _need(p.returncode==0 and not p.stderr and 0 -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if __name__=='__main__': raise SystemExit(main()) -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~^^ -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return int(e.main()) -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~^^ -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return wrapper.main([sys.argv[0],*argv]) -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if cmd=='scheduled':return _scheduled_locked() -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^ -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try:return legacy.scheduled_run() -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~~~~^^ -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: try: backup_target(t,green_verify=False) -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: return _legacy_backup_target(t,green_verify) -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) -2026-09-27T18:10:39.926992+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only -2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) -2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper -2026-09-27T18:10:39.927246+03:00 pve01 homelab-backup-v2[3649069]: _need(p.returncode==0 and not p.stderr and 0 -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if __name__=='__main__': raise SystemExit(main()) -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~^^ -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return int(e.main()) -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~^^ -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return wrapper.main([sys.argv[0],*argv]) -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if cmd=='scheduled':return _scheduled_locked() -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^ -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try:return legacy.scheduled_run() -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~~~~^^ -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: try: backup_target(t,green_verify=False) -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: return _legacy_backup_target(t,green_verify) -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) -2026-09-27T18:46:00.910101+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only -2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) -2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper -2026-09-27T18:46:00.910381+03:00 pve01 homelab-backup-v2[3761852]: _need(p.returncode==0 and not p.stderr and 0 -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if __name__=='__main__': raise SystemExit(main()) -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~^^ -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return int(e.main()) -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~^^ -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return wrapper.main([sys.argv[0],*argv]) -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if cmd=='scheduled':return _scheduled_locked() -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^ -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try:return legacy.scheduled_run() -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~~~~^^ -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: try: backup_target(t,green_verify=False) -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: return _legacy_backup_target(t,green_verify) -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) -2026-09-27T19:23:49.950794+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only -2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) -2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper -2026-09-27T19:23:49.951058+03:00 pve01 homelab-backup-v2[3884916]: _need(p.returncode==0 and not p.stderr and 0 -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if __name__=='__main__': raise SystemExit(main()) -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~^^ -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return int(e.main()) -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~^^ -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return wrapper.main([sys.argv[0],*argv]) -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if cmd=='scheduled':return _scheduled_locked() -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^ -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try:return legacy.scheduled_run() -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~~~~^^ -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: try: backup_target(t,green_verify=False) -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: return _legacy_backup_target(t,green_verify) -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) -2026-09-27T19:54:58.953354+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only -2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) -2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper -2026-09-27T19:54:58.953685+03:00 pve01 homelab-backup-v2[3986164]: _need(p.returncode==0 and not p.stderr and 0 -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if __name__=='__main__': raise SystemExit(main()) -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~^^ -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return int(e.main()) -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~^^ -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return wrapper.main([sys.argv[0],*argv]) -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if cmd=='scheduled':return _scheduled_locked() -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^ -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try:return legacy.scheduled_run() -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~~~~^^ -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: try: backup_target(t,green_verify=False) -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: return _legacy_backup_target(t,green_verify) -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) -2026-09-27T20:34:15.128859+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only -2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) -2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper -2026-09-27T20:34:15.129121+03:00 pve01 homelab-backup-v2[4110155]: _need(p.returncode==0 and not p.stderr and 0 -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if __name__=='__main__': raise SystemExit(main()) -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~^^ -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/usr/local/sbin/homelab-backup-v2", line 25, in main -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return int(e.main()) -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~^^ -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py", line 186, in main -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return wrapper.main([sys.argv[0],*argv]) -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 94, in main -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if cmd=='scheduled':return _scheduled_locked() -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^ -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 85, in _scheduled_locked -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try:return legacy.scheduled_run() -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~~~~^^ -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 960, in scheduled_run -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: try: backup_target(t,green_verify=False) -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py", line 42, in backup_target -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: return _legacy_backup_target(t,green_verify) -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 908, in backup_target -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py", line 873, in capture_target -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: if typ=='pve_guest': return capture_pve_guest(t,gen,work) -2026-09-27T21:08:59.774470+03:00 pve01 homelab-backup-v2[19346]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ -2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 204, in capture_pve_guest -2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: p=plan_only(legacy,t,scratch); local=pathlib.Path(work)/'payload.bin'; cap=int(p['job']['archive_cap_bytes']) -2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 85, in plan_only -2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: a=_helper(_live_inputs(legacy,node,vmid,scratch)); b=_helper(_live_inputs(legacy,node,vmid,scratch)) -2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: File "/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/lxc_adapter.py", line 69, in _helper -2026-09-27T21:08:59.774878+03:00 pve01 homelab-backup-v2[19346]: _need(p.returncode==0 and not p.stderr and 0