From 35c98b733b8a842f27d51d0dd73a92e451d23533 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Sun, 27 Sep 2026 20:02:07 +0000 Subject: [PATCH] runtime: publish SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1 --- ...HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1.json | 38 ++++ ...-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1.txt | 164 ++++++++++++++++ runtime/latest.json | 22 +-- runtime/latest.txt | 185 +++++++++++++----- 4 files changed, 344 insertions(+), 65 deletions(-) create mode 100644 runtime/history/SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1.json create mode 100644 runtime/history/SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1.txt diff --git a/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1.json b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1.json new file mode 100644 index 00000000..cdd2ba52 --- /dev/null +++ b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-nonrpo-lock-rca", + "started_at_utc": "2026-09-27T20:02:01Z", + "finished_at_utc": "2026-09-27T20:02:06Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "b775bb0d8b318e704bf86baa42c51f6a73c0c0f57d7358ecb9a30b5d0e8cd341", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7", + "sanitized_output_sha256": "5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "BACKUP_NONRPO_LOCK_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1\nMODE=read-only\nCOMPONENT=homelab-backup-nonrpo-lock-rca\nMUTATION_BOUNDARY=READ_ONLY_AUDIT_ALL_LOCK_OWNERSHIP_AND_AUDIT_SOURCE;NO_LOCK_ACQUIRE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1233_STATUS=OK\nSOURCE_1233_RC=0\nSOURCE_1233_CHANGES_MADE=false\nSOURCE_1233_MUTATION_OUTCOME=NO_MUTATION\nSOURCE_1233_DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT\nSOURCE_1233_NEXT_GATE=READ_EXACT_1233_AND_RECONCILE_NON_RPO_ISSUE\nSOURCE_1233_AUDIT0_ISSUE_COUNT=12\nSOURCE_1233_AUDIT0_RPO_COUNT=11\nSOURCE_1233_AUDIT0_NON_RPO_COUNT=1\n=== LIVE_IDENTITY ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\n=== LOCK_FILE_METADATA ===\nLOCK_FILE=/run/homelab-backup-heavy.lock|600|root:root|0|2026-09-27 22:58:26.543174461 +0300|inode=3474\nLOCK_FILE=/run/lock/homelab-backup-dispatcher.lock|600|root:root|0|2026-09-27 03:27:41.596907725 +0300|inode=9\nLOCK_FILE=/run/lock/homelab-vm-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 03:27:41.597907739 +0300|inode=2944\nLOCK_FILE=/run/lock/homelab-app-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 09:30:02.178976732 +0300|inode=15\n=== LSLOCKS_BEFORE_AUDIT ===\nkvm 4446 POSIX WRITE /run/qemu-server/150.pid\nbash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock\ncorosync 1463 POSIX WRITE /run/corosync.pid\ncron 3999 FLOCK WRITE /run/crond.pid\nlxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo\ncron 1464 FLOCK WRITE /run/crond.pid\nkvm 4335 POSIX WRITE /run/qemu-server/171.pid\nkvm 1100046 POSIX WRITE /run/qemu-server/100.pid\nkvm 4237 POSIX WRITE /run/qemu-server/170.pid\nkvm 342771 POSIX WRITE /run/qemu-server/210.pid\npvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock\npvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock\nspiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock\npve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock\npve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock\npvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock\npveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock\npve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock\ndmeventd 531 POSIX WRITE /run/dmeventd.pid\ncron 3538 FLOCK WRITE /run/crond.pid\nzed 952 POSIX WRITE /run/zed.pid\nzed 952 POSIX WRITE /run/zed.state\nlxcfs 963 POSIX WRITE /run/lxcfs.pid\npvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck\n=== FUSER_BEFORE_AUDIT ===\nFUSER_PATH=/run/homelab-backup-heavy.lock PIDS=\nFUSER_PATH=/run/lock/homelab-backup-dispatcher.lock PIDS=\nFUSER_PATH=/run/lock/homelab-vm-cloud-quorum-queue.lock PIDS=\nFUSER_PATH=/run/lock/homelab-app-cloud-quorum-queue.lock PIDS=\n=== AUDIT_LOCK_SOURCE ===\n183- (\"homelab-alertmanager-backup-health.timer\",\"ALERT_TIMER_BAD\"),\n184-]:\n185- ok,a,e=unit_ok(unit)\n186- if not ok: issues.append({\"severity\":\"HIGH\",\"code\":code,\"logical_id\":unit,\"root_cause\":\"UNIT_\"+a+\"_\"+e})\n187-\n188:for lock in [\"/run/homelab-backup-heavy.lock\",\"/run/lock/homelab-app-cloud-quorum-queue.lock\",\"/run/lock/homelab-vm-cloud-quorum-queue.lock\",\"/run/lock/homelab-backup-dispatcher.lock\"]:\n189- p=pathlib.Path(lock)\n190- if p.exists():\n191: lrc,_,_=run([\"flock\",\"-n\",str(p),\"-c\",\"true\"],5)\n192: if lrc!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"LOCK_BUSY\",\"logical_id\":lock,\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\"})\n193-\n194-env=os.environ.copy(); env[\"HOMELAB_BACKUP_GFS_POLICY\"]=GFSP; env[\"HOMELAB_BACKUP_TARGET_REGISTRY\"]=str(REG)\n195-grc,gout,_=run([GFSH,\"validate\"],20,env)\n196-if grc!=0 or \"GFS_POLICY=OK\" not in gout:\n197- issues.append({\"severity\":\"CRITICAL\",\"code\":\"GFS_INVALID\",\"logical_id\":\"-\",\"root_cause\":\"GFS_VALIDATE_FAILED\"})\n=== AUDIT_ALL_NO_OWNED_HEAVY_LOCK ===\nAUDIT_ALL_RC=1\nAUDIT_ALL_STDERR_BYTES=0\nAUDIT_ISSUES_BEGIN\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nAUDIT_ISSUES_END\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nAUDIT_RPO_COUNT=11\nAUDIT_NON_RPO_COUNT=0\nAUDIT_NON_RPO_ISSUES_BEGIN\nAUDIT_NON_RPO_ISSUES_END\nAUDIT_ALL_HEALTH_NO_WRITE=true\n=== LSLOCKS_AFTER_AUDIT ===\nkvm 4446 POSIX WRITE /run/qemu-server/150.pid\nbash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock\ncorosync 1463 POSIX WRITE /run/corosync.pid\ncron 3999 FLOCK WRITE /run/crond.pid\nlxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo\ncron 1464 FLOCK WRITE /run/crond.pid\nkvm 4335 POSIX WRITE /run/qemu-server/171.pid\nkvm 1100046 POSIX WRITE /run/qemu-server/100.pid\nkvm 4237 POSIX WRITE /run/qemu-server/170.pid\nkvm 342771 POSIX WRITE /run/qemu-server/210.pid\npvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock\npvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock\nspiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock\npve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock\npve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock\npvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock\npveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock\npve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock\ndmeventd 531 POSIX WRITE /run/dmeventd.pid\ncron 3538 FLOCK WRITE /run/crond.pid\nzed 952 POSIX WRITE /run/zed.pid\nzed 952 POSIX WRITE /run/zed.state\nlxcfs 963 POSIX WRITE /run/lxcfs.pid\npvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck\n=== 1233_RUNNER_RAW_LOCK_EVIDENCE ===\nRAW1233_PATH=/var/lib/homelab-chat-run/raw/SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1-20260927T195820Z.log\n29:BACKUP_HEAVY_LOCK_ACQUIRED=true\n33:=== FRESH_AUDIT_UNDER_LOCK ===\n37:AUDIT0_NON_RPO_COUNT=1\n53:DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT\nTASK_RESULT=PASS_NONRPO_LOCK_RCA_COLLECTED\nDECISION=PASS_NONRPO_LOCK_RCA_COLLECTED\nLOCK_SELF_INDUCED_PROVISIONAL=true\nNEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1.txt b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1.txt new file mode 100644 index 00000000..5533c835 --- /dev/null +++ b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1.txt @@ -0,0 +1,164 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-nonrpo-lock-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=b775bb0d8b318e704bf86baa42c51f6a73c0c0f57d7358ecb9a30b5d0e8cd341 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7 +SANITIZED_OUTPUT_SHA256=5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7 +OUTPUT_BEGIN +BACKUP_NONRPO_LOCK_RCA_BEGIN=true +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1 +MODE=read-only +COMPONENT=homelab-backup-nonrpo-lock-rca +MUTATION_BOUNDARY=READ_ONLY_AUDIT_ALL_LOCK_OWNERSHIP_AND_AUDIT_SOURCE;NO_LOCK_ACQUIRE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE +=== AUTHORITY === +HOST=pve01 +CLUSTER_QUORATE=Yes +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +SOURCE_1233_STATUS=OK +SOURCE_1233_RC=0 +SOURCE_1233_CHANGES_MADE=false +SOURCE_1233_MUTATION_OUTCOME=NO_MUTATION +SOURCE_1233_DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT +SOURCE_1233_NEXT_GATE=READ_EXACT_1233_AND_RECONCILE_NON_RPO_ISSUE +SOURCE_1233_AUDIT0_ISSUE_COUNT=12 +SOURCE_1233_AUDIT0_RPO_COUNT=11 +SOURCE_1233_AUDIT0_NON_RPO_COUNT=1 +=== LIVE_IDENTITY === +WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 +VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad +AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +=== LOCK_FILE_METADATA === +LOCK_FILE=/run/homelab-backup-heavy.lock|600|root:root|0|2026-09-27 22:58:26.543174461 +0300|inode=3474 +LOCK_FILE=/run/lock/homelab-backup-dispatcher.lock|600|root:root|0|2026-09-27 03:27:41.596907725 +0300|inode=9 +LOCK_FILE=/run/lock/homelab-vm-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 03:27:41.597907739 +0300|inode=2944 +LOCK_FILE=/run/lock/homelab-app-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 09:30:02.178976732 +0300|inode=15 +=== LSLOCKS_BEFORE_AUDIT === +kvm 4446 POSIX WRITE /run/qemu-server/150.pid +bash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock +corosync 1463 POSIX WRITE /run/corosync.pid +cron 3999 FLOCK WRITE /run/crond.pid +lxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo +cron 1464 FLOCK WRITE /run/crond.pid +kvm 4335 POSIX WRITE /run/qemu-server/171.pid +kvm 1100046 POSIX WRITE /run/qemu-server/100.pid +kvm 4237 POSIX WRITE /run/qemu-server/170.pid +kvm 342771 POSIX WRITE /run/qemu-server/210.pid +pvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock +pvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock +spiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock +pve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock +pve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock +pvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock +pveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock +pve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock +dmeventd 531 POSIX WRITE /run/dmeventd.pid +cron 3538 FLOCK WRITE /run/crond.pid +zed 952 POSIX WRITE /run/zed.pid +zed 952 POSIX WRITE /run/zed.state +lxcfs 963 POSIX WRITE /run/lxcfs.pid +pvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck +=== FUSER_BEFORE_AUDIT === +FUSER_PATH=/run/homelab-backup-heavy.lock PIDS= +FUSER_PATH=/run/lock/homelab-backup-dispatcher.lock PIDS= +FUSER_PATH=/run/lock/homelab-vm-cloud-quorum-queue.lock PIDS= +FUSER_PATH=/run/lock/homelab-app-cloud-quorum-queue.lock PIDS= +=== AUDIT_LOCK_SOURCE === +183- ("homelab-alertmanager-backup-health.timer","ALERT_TIMER_BAD"), +184-]: +185- ok,a,e=unit_ok(unit) +186- if not ok: issues.append({"severity":"HIGH","code":code,"logical_id":unit,"root_cause":"UNIT_"+a+"_"+e}) +187- +188:for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]: +189- p=pathlib.Path(lock) +190- if p.exists(): +191: lrc,_,_=run(["flock","-n",str(p),"-c","true"],5) +192: if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"}) +193- +194-env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG) +195-grc,gout,_=run([GFSH,"validate"],20,env) +196-if grc!=0 or "GFS_POLICY=OK" not in gout: +197- issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"}) +=== AUDIT_ALL_NO_OWNED_HEAVY_LOCK === +AUDIT_ALL_RC=1 +AUDIT_ALL_STDERR_BYTES=0 +AUDIT_ISSUES_BEGIN +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +AUDIT_ISSUES_END +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +AUDIT_RPO_COUNT=11 +AUDIT_NON_RPO_COUNT=0 +AUDIT_NON_RPO_ISSUES_BEGIN +AUDIT_NON_RPO_ISSUES_END +AUDIT_ALL_HEALTH_NO_WRITE=true +=== LSLOCKS_AFTER_AUDIT === +kvm 4446 POSIX WRITE /run/qemu-server/150.pid +bash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock +corosync 1463 POSIX WRITE /run/corosync.pid +cron 3999 FLOCK WRITE /run/crond.pid +lxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo +cron 1464 FLOCK WRITE /run/crond.pid +kvm 4335 POSIX WRITE /run/qemu-server/171.pid +kvm 1100046 POSIX WRITE /run/qemu-server/100.pid +kvm 4237 POSIX WRITE /run/qemu-server/170.pid +kvm 342771 POSIX WRITE /run/qemu-server/210.pid +pvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock +pvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock +spiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock +pve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock +pve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock +pvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock +pveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock +pve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock +dmeventd 531 POSIX WRITE /run/dmeventd.pid +cron 3538 FLOCK WRITE /run/crond.pid +zed 952 POSIX WRITE /run/zed.pid +zed 952 POSIX WRITE /run/zed.state +lxcfs 963 POSIX WRITE /run/lxcfs.pid +pvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck +=== 1233_RUNNER_RAW_LOCK_EVIDENCE === +RAW1233_PATH=/var/lib/homelab-chat-run/raw/SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1-20260927T195820Z.log +29:BACKUP_HEAVY_LOCK_ACQUIRED=true +33:=== FRESH_AUDIT_UNDER_LOCK === +37:AUDIT0_NON_RPO_COUNT=1 +53:DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT +TASK_RESULT=PASS_NONRPO_LOCK_RCA_COLLECTED +DECISION=PASS_NONRPO_LOCK_RCA_COLLECTED +LOCK_SELF_INDUCED_PROVISIONAL=true +NEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 57efac14..cdd2ba52 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,25 +1,25 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1", + "command_id": "SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1", "status": "OK", "rc": 0, "host": "pve01", - "mode": "change", - "component": "homelab-backup-adaptive-rpo-recovery", - "started_at_utc": "2026-09-27T19:58:20Z", - "finished_at_utc": "2026-09-27T19:58:30Z", + "mode": "read-only", + "component": "homelab-backup-nonrpo-lock-rca", + "started_at_utc": "2026-09-27T20:02:01Z", + "finished_at_utc": "2026-09-27T20:02:06Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "380add3ede2fd9137f06e7c75aad3301d45af2b01ab9cd6cc19daaa510387ece", + "command_sha256": "b775bb0d8b318e704bf86baa42c51f6a73c0c0f57d7358ecb9a30b5d0e8cd341", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, + "change_declared": false, "result_contract_valid": true, - "result_contract_status": "OK", + "result_contract_status": null, "result_contract_error": null, "command_rc": 0, "changes_made": false, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "4a832057af33457c2a74ea89cfc77fe0ee10af72ec95c10818e3396b7e293f12", - "sanitized_output_sha256": "4a832057af33457c2a74ea89cfc77fe0ee10af72ec95c10818e3396b7e293f12", + "raw_evidence_sha256": "5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7", + "sanitized_output_sha256": "5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "BACKUP_ADAPTIVE_RPO_RECOVERY_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1\nMODE=change\nCOMPONENT=homelab-backup-adaptive-rpo-recovery\nMUTATION_BOUNDARY=ONLY_CURRENT_V2_VERIFY_TARGET_FOR_FRESH_AUDIT_RPO_BREACHES;EXCLUDES_CT110_ALREADY_FRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1232_STATUS=OK\nSOURCE_1232_RC=0\nSOURCE_1232_DECISION=PASS_CT110_STATE_AUDIT_RECONCILE\nSOURCE_1232_NEXT_GATE=ADAPTIVE_RPO_RECOVERY_FROM_FRESH_AUDIT\nSOURCE_1232_STATE_ACCEPTED=true\nSOURCE_1232_SUCCESS_MAIL_POLICY_ACCEPTED=true\nSOURCE_1232_CT110_RPO_COUNT=0\n=== LIVE_IDENTITY ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nSOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_WORKTREE_CLEAN=true\n=== OWNER_WAIT ===\nOWNER_WAIT_ITERATION=1 ACTIVE=inactive PID=0\nOWNER_WAIT_ITERATION=2 ACTIVE=inactive PID=0\nBACKUP_HEAVY_LOCK_ACQUIRED=true\nACTIVE_VZDUMP_COUNT=0\nRECOVERY_START_EPOCH=1790539106\nRECOVERY_START_UTC=2026-09-27T19:58:26Z\n=== FRESH_AUDIT_UNDER_LOCK ===\nAUDIT0_RC=1\nAUDIT0_ISSUE_COUNT=12\nAUDIT0_RPO_COUNT=11\nAUDIT0_NON_RPO_COUNT=1\nAUDIT0_RPO_SET_BEGIN\ncontrol-bootstrap-v2\nvm111\nvm112\nvm113\nvm130\nvm150\nvm160\nvm170\nvm171\nvm190\nvm9130\nAUDIT0_RPO_SET_END\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nTASK_RESULT=HOLD_ADAPTIVE_RPO_RECOVERY\nDECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT\nNEXT_GATE=READ_EXACT_1233_AND_RECONCILE_NON_RPO_ISSUE\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "BACKUP_NONRPO_LOCK_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1\nMODE=read-only\nCOMPONENT=homelab-backup-nonrpo-lock-rca\nMUTATION_BOUNDARY=READ_ONLY_AUDIT_ALL_LOCK_OWNERSHIP_AND_AUDIT_SOURCE;NO_LOCK_ACQUIRE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1233_STATUS=OK\nSOURCE_1233_RC=0\nSOURCE_1233_CHANGES_MADE=false\nSOURCE_1233_MUTATION_OUTCOME=NO_MUTATION\nSOURCE_1233_DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT\nSOURCE_1233_NEXT_GATE=READ_EXACT_1233_AND_RECONCILE_NON_RPO_ISSUE\nSOURCE_1233_AUDIT0_ISSUE_COUNT=12\nSOURCE_1233_AUDIT0_RPO_COUNT=11\nSOURCE_1233_AUDIT0_NON_RPO_COUNT=1\n=== LIVE_IDENTITY ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\n=== LOCK_FILE_METADATA ===\nLOCK_FILE=/run/homelab-backup-heavy.lock|600|root:root|0|2026-09-27 22:58:26.543174461 +0300|inode=3474\nLOCK_FILE=/run/lock/homelab-backup-dispatcher.lock|600|root:root|0|2026-09-27 03:27:41.596907725 +0300|inode=9\nLOCK_FILE=/run/lock/homelab-vm-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 03:27:41.597907739 +0300|inode=2944\nLOCK_FILE=/run/lock/homelab-app-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 09:30:02.178976732 +0300|inode=15\n=== LSLOCKS_BEFORE_AUDIT ===\nkvm 4446 POSIX WRITE /run/qemu-server/150.pid\nbash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock\ncorosync 1463 POSIX WRITE /run/corosync.pid\ncron 3999 FLOCK WRITE /run/crond.pid\nlxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo\ncron 1464 FLOCK WRITE /run/crond.pid\nkvm 4335 POSIX WRITE /run/qemu-server/171.pid\nkvm 1100046 POSIX WRITE /run/qemu-server/100.pid\nkvm 4237 POSIX WRITE /run/qemu-server/170.pid\nkvm 342771 POSIX WRITE /run/qemu-server/210.pid\npvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock\npvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock\nspiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock\npve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock\npve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock\npvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock\npveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock\npve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock\ndmeventd 531 POSIX WRITE /run/dmeventd.pid\ncron 3538 FLOCK WRITE /run/crond.pid\nzed 952 POSIX WRITE /run/zed.pid\nzed 952 POSIX WRITE /run/zed.state\nlxcfs 963 POSIX WRITE /run/lxcfs.pid\npvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck\n=== FUSER_BEFORE_AUDIT ===\nFUSER_PATH=/run/homelab-backup-heavy.lock PIDS=\nFUSER_PATH=/run/lock/homelab-backup-dispatcher.lock PIDS=\nFUSER_PATH=/run/lock/homelab-vm-cloud-quorum-queue.lock PIDS=\nFUSER_PATH=/run/lock/homelab-app-cloud-quorum-queue.lock PIDS=\n=== AUDIT_LOCK_SOURCE ===\n183- (\"homelab-alertmanager-backup-health.timer\",\"ALERT_TIMER_BAD\"),\n184-]:\n185- ok,a,e=unit_ok(unit)\n186- if not ok: issues.append({\"severity\":\"HIGH\",\"code\":code,\"logical_id\":unit,\"root_cause\":\"UNIT_\"+a+\"_\"+e})\n187-\n188:for lock in [\"/run/homelab-backup-heavy.lock\",\"/run/lock/homelab-app-cloud-quorum-queue.lock\",\"/run/lock/homelab-vm-cloud-quorum-queue.lock\",\"/run/lock/homelab-backup-dispatcher.lock\"]:\n189- p=pathlib.Path(lock)\n190- if p.exists():\n191: lrc,_,_=run([\"flock\",\"-n\",str(p),\"-c\",\"true\"],5)\n192: if lrc!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"LOCK_BUSY\",\"logical_id\":lock,\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\"})\n193-\n194-env=os.environ.copy(); env[\"HOMELAB_BACKUP_GFS_POLICY\"]=GFSP; env[\"HOMELAB_BACKUP_TARGET_REGISTRY\"]=str(REG)\n195-grc,gout,_=run([GFSH,\"validate\"],20,env)\n196-if grc!=0 or \"GFS_POLICY=OK\" not in gout:\n197- issues.append({\"severity\":\"CRITICAL\",\"code\":\"GFS_INVALID\",\"logical_id\":\"-\",\"root_cause\":\"GFS_VALIDATE_FAILED\"})\n=== AUDIT_ALL_NO_OWNED_HEAVY_LOCK ===\nAUDIT_ALL_RC=1\nAUDIT_ALL_STDERR_BYTES=0\nAUDIT_ISSUES_BEGIN\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nAUDIT_ISSUES_END\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nAUDIT_RPO_COUNT=11\nAUDIT_NON_RPO_COUNT=0\nAUDIT_NON_RPO_ISSUES_BEGIN\nAUDIT_NON_RPO_ISSUES_END\nAUDIT_ALL_HEALTH_NO_WRITE=true\n=== LSLOCKS_AFTER_AUDIT ===\nkvm 4446 POSIX WRITE /run/qemu-server/150.pid\nbash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock\ncorosync 1463 POSIX WRITE /run/corosync.pid\ncron 3999 FLOCK WRITE /run/crond.pid\nlxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo\ncron 1464 FLOCK WRITE /run/crond.pid\nkvm 4335 POSIX WRITE /run/qemu-server/171.pid\nkvm 1100046 POSIX WRITE /run/qemu-server/100.pid\nkvm 4237 POSIX WRITE /run/qemu-server/170.pid\nkvm 342771 POSIX WRITE /run/qemu-server/210.pid\npvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock\npvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock\nspiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock\npve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock\npve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock\npvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock\npveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock\npve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock\ndmeventd 531 POSIX WRITE /run/dmeventd.pid\ncron 3538 FLOCK WRITE /run/crond.pid\nzed 952 POSIX WRITE /run/zed.pid\nzed 952 POSIX WRITE /run/zed.state\nlxcfs 963 POSIX WRITE /run/lxcfs.pid\npvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck\n=== 1233_RUNNER_RAW_LOCK_EVIDENCE ===\nRAW1233_PATH=/var/lib/homelab-chat-run/raw/SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1-20260927T195820Z.log\n29:BACKUP_HEAVY_LOCK_ACQUIRED=true\n33:=== FRESH_AUDIT_UNDER_LOCK ===\n37:AUDIT0_NON_RPO_COUNT=1\n53:DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT\nTASK_RESULT=PASS_NONRPO_LOCK_RCA_COLLECTED\nDECISION=PASS_NONRPO_LOCK_RCA_COLLECTED\nLOCK_SELF_INDUCED_PROVISIONAL=true\nNEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 931f4cce..5533c835 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,20 +1,20 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1 +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1 STATUS=OK RC=0 HOST=pve01 -MODE=change -COMPONENT=homelab-backup-adaptive-rpo-recovery +MODE=read-only +COMPONENT=homelab-backup-nonrpo-lock-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=380add3ede2fd9137f06e7c75aad3301d45af2b01ab9cd6cc19daaa510387ece +COMMAND_SHA256=b775bb0d8b318e704bf86baa42c51f6a73c0c0f57d7358ecb9a30b5d0e8cd341 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true +CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=OK +RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE COMMAND_RC=0 CHANGES_MADE=false @@ -24,64 +24,141 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=4a832057af33457c2a74ea89cfc77fe0ee10af72ec95c10818e3396b7e293f12 -SANITIZED_OUTPUT_SHA256=4a832057af33457c2a74ea89cfc77fe0ee10af72ec95c10818e3396b7e293f12 +RAW_EVIDENCE_SHA256=5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7 +SANITIZED_OUTPUT_SHA256=5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7 OUTPUT_BEGIN -BACKUP_ADAPTIVE_RPO_RECOVERY_BEGIN=true -COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1 -MODE=change -COMPONENT=homelab-backup-adaptive-rpo-recovery -MUTATION_BOUNDARY=ONLY_CURRENT_V2_VERIFY_TARGET_FOR_FRESH_AUDIT_RPO_BREACHES;EXCLUDES_CT110_ALREADY_FRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE +BACKUP_NONRPO_LOCK_RCA_BEGIN=true +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1 +MODE=read-only +COMPONENT=homelab-backup-nonrpo-lock-rca +MUTATION_BOUNDARY=READ_ONLY_AUDIT_ALL_LOCK_OWNERSHIP_AND_AUDIT_SOURCE;NO_LOCK_ACQUIRE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE === AUTHORITY === HOST=pve01 CLUSTER_QUORATE=Yes RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -SOURCE_1232_STATUS=OK -SOURCE_1232_RC=0 -SOURCE_1232_DECISION=PASS_CT110_STATE_AUDIT_RECONCILE -SOURCE_1232_NEXT_GATE=ADAPTIVE_RPO_RECOVERY_FROM_FRESH_AUDIT -SOURCE_1232_STATE_ACCEPTED=true -SOURCE_1232_SUCCESS_MAIL_POLICY_ACCEPTED=true -SOURCE_1232_CT110_RPO_COUNT=0 +SOURCE_1233_STATUS=OK +SOURCE_1233_RC=0 +SOURCE_1233_CHANGES_MADE=false +SOURCE_1233_MUTATION_OUTCOME=NO_MUTATION +SOURCE_1233_DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT +SOURCE_1233_NEXT_GATE=READ_EXACT_1233_AND_RECONCILE_NON_RPO_ISSUE +SOURCE_1233_AUDIT0_ISSUE_COUNT=12 +SOURCE_1233_AUDIT0_RPO_COUNT=11 +SOURCE_1233_AUDIT0_NON_RPO_COUNT=1 === LIVE_IDENTITY === WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a -SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 -SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation -SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 -SOURCE_WORKTREE_CLEAN=true -=== OWNER_WAIT === -OWNER_WAIT_ITERATION=1 ACTIVE=inactive PID=0 -OWNER_WAIT_ITERATION=2 ACTIVE=inactive PID=0 -BACKUP_HEAVY_LOCK_ACQUIRED=true -ACTIVE_VZDUMP_COUNT=0 -RECOVERY_START_EPOCH=1790539106 -RECOVERY_START_UTC=2026-09-27T19:58:26Z -=== FRESH_AUDIT_UNDER_LOCK === -AUDIT0_RC=1 -AUDIT0_ISSUE_COUNT=12 -AUDIT0_RPO_COUNT=11 -AUDIT0_NON_RPO_COUNT=1 -AUDIT0_RPO_SET_BEGIN -control-bootstrap-v2 -vm111 -vm112 -vm113 -vm130 -vm150 -vm160 -vm170 -vm171 -vm190 -vm9130 -AUDIT0_RPO_SET_END -SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -TASK_RESULT=HOLD_ADAPTIVE_RPO_RECOVERY -DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT -NEXT_GATE=READ_EXACT_1233_AND_RECONCILE_NON_RPO_ISSUE -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +=== LOCK_FILE_METADATA === +LOCK_FILE=/run/homelab-backup-heavy.lock|600|root:root|0|2026-09-27 22:58:26.543174461 +0300|inode=3474 +LOCK_FILE=/run/lock/homelab-backup-dispatcher.lock|600|root:root|0|2026-09-27 03:27:41.596907725 +0300|inode=9 +LOCK_FILE=/run/lock/homelab-vm-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 03:27:41.597907739 +0300|inode=2944 +LOCK_FILE=/run/lock/homelab-app-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 09:30:02.178976732 +0300|inode=15 +=== LSLOCKS_BEFORE_AUDIT === +kvm 4446 POSIX WRITE /run/qemu-server/150.pid +bash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock +corosync 1463 POSIX WRITE /run/corosync.pid +cron 3999 FLOCK WRITE /run/crond.pid +lxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo +cron 1464 FLOCK WRITE /run/crond.pid +kvm 4335 POSIX WRITE /run/qemu-server/171.pid +kvm 1100046 POSIX WRITE /run/qemu-server/100.pid +kvm 4237 POSIX WRITE /run/qemu-server/170.pid +kvm 342771 POSIX WRITE /run/qemu-server/210.pid +pvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock +pvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock +spiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock +pve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock +pve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock +pvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock +pveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock +pve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock +dmeventd 531 POSIX WRITE /run/dmeventd.pid +cron 3538 FLOCK WRITE /run/crond.pid +zed 952 POSIX WRITE /run/zed.pid +zed 952 POSIX WRITE /run/zed.state +lxcfs 963 POSIX WRITE /run/lxcfs.pid +pvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck +=== FUSER_BEFORE_AUDIT === +FUSER_PATH=/run/homelab-backup-heavy.lock PIDS= +FUSER_PATH=/run/lock/homelab-backup-dispatcher.lock PIDS= +FUSER_PATH=/run/lock/homelab-vm-cloud-quorum-queue.lock PIDS= +FUSER_PATH=/run/lock/homelab-app-cloud-quorum-queue.lock PIDS= +=== AUDIT_LOCK_SOURCE === +183- ("homelab-alertmanager-backup-health.timer","ALERT_TIMER_BAD"), +184-]: +185- ok,a,e=unit_ok(unit) +186- if not ok: issues.append({"severity":"HIGH","code":code,"logical_id":unit,"root_cause":"UNIT_"+a+"_"+e}) +187- +188:for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]: +189- p=pathlib.Path(lock) +190- if p.exists(): +191: lrc,_,_=run(["flock","-n",str(p),"-c","true"],5) +192: if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"}) +193- +194-env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG) +195-grc,gout,_=run([GFSH,"validate"],20,env) +196-if grc!=0 or "GFS_POLICY=OK" not in gout: +197- issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"}) +=== AUDIT_ALL_NO_OWNED_HEAVY_LOCK === +AUDIT_ALL_RC=1 +AUDIT_ALL_STDERR_BYTES=0 +AUDIT_ISSUES_BEGIN +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +AUDIT_ISSUES_END +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +AUDIT_RPO_COUNT=11 +AUDIT_NON_RPO_COUNT=0 +AUDIT_NON_RPO_ISSUES_BEGIN +AUDIT_NON_RPO_ISSUES_END +AUDIT_ALL_HEALTH_NO_WRITE=true +=== LSLOCKS_AFTER_AUDIT === +kvm 4446 POSIX WRITE /run/qemu-server/150.pid +bash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock +corosync 1463 POSIX WRITE /run/corosync.pid +cron 3999 FLOCK WRITE /run/crond.pid +lxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo +cron 1464 FLOCK WRITE /run/crond.pid +kvm 4335 POSIX WRITE /run/qemu-server/171.pid +kvm 1100046 POSIX WRITE /run/qemu-server/100.pid +kvm 4237 POSIX WRITE /run/qemu-server/170.pid +kvm 342771 POSIX WRITE /run/qemu-server/210.pid +pvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock +pvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock +spiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock +pve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock +pve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock +pvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock +pveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock +pve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock +dmeventd 531 POSIX WRITE /run/dmeventd.pid +cron 3538 FLOCK WRITE /run/crond.pid +zed 952 POSIX WRITE /run/zed.pid +zed 952 POSIX WRITE /run/zed.state +lxcfs 963 POSIX WRITE /run/lxcfs.pid +pvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck +=== 1233_RUNNER_RAW_LOCK_EVIDENCE === +RAW1233_PATH=/var/lib/homelab-chat-run/raw/SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1-20260927T195820Z.log +29:BACKUP_HEAVY_LOCK_ACQUIRED=true +33:=== FRESH_AUDIT_UNDER_LOCK === +37:AUDIT0_NON_RPO_COUNT=1 +53:DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT +TASK_RESULT=PASS_NONRPO_LOCK_RCA_COLLECTED +DECISION=PASS_NONRPO_LOCK_RCA_COLLECTED +LOCK_SELF_INDUCED_PROVISIONAL=true +NEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END