runtime: publish SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1
This commit is contained in:
homelab-runtime-publisher
File diff suppressed because one or more lines are too long
@@ -0,0 +1,164 @@
|
|||||||
|
CHAT_OUTPUT_BEGIN
|
||||||
|
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1
|
||||||
|
STATUS=OK
|
||||||
|
RC=0
|
||||||
|
HOST=pve01
|
||||||
|
MODE=read-only
|
||||||
|
COMPONENT=homelab-backup-nonrpo-lock-rca
|
||||||
|
REFERENCE_REGISTER_CHECK=OK
|
||||||
|
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||||
|
ERROR_REGISTER_CHECK=OK
|
||||||
|
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||||
|
COMMAND_SHA256=b775bb0d8b318e704bf86baa42c51f6a73c0c0f57d7358ecb9a30b5d0e8cd341
|
||||||
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||||
|
EXECUTION_STARTED=true
|
||||||
|
CHANGE_DECLARED=false
|
||||||
|
RESULT_CONTRACT_VALID=true
|
||||||
|
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||||
|
RESULT_CONTRACT_ERROR=NONE
|
||||||
|
COMMAND_RC=0
|
||||||
|
CHANGES_MADE=false
|
||||||
|
ROLLBACK_STARTED=false
|
||||||
|
ROLLBACK_RESTORED=null
|
||||||
|
MUTATION_OUTCOME=NO_MUTATION
|
||||||
|
SANITIZED=yes
|
||||||
|
SECRETS_INCLUDED=no
|
||||||
|
PRIVATE_ADDRESSES_INCLUDED=no
|
||||||
|
RAW_EVIDENCE_SHA256=5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7
|
||||||
|
SANITIZED_OUTPUT_SHA256=5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7
|
||||||
|
OUTPUT_BEGIN
|
||||||
|
BACKUP_NONRPO_LOCK_RCA_BEGIN=true
|
||||||
|
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1
|
||||||
|
MODE=read-only
|
||||||
|
COMPONENT=homelab-backup-nonrpo-lock-rca
|
||||||
|
MUTATION_BOUNDARY=READ_ONLY_AUDIT_ALL_LOCK_OWNERSHIP_AND_AUDIT_SOURCE;NO_LOCK_ACQUIRE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE
|
||||||
|
=== AUTHORITY ===
|
||||||
|
HOST=pve01
|
||||||
|
CLUSTER_QUORATE=Yes
|
||||||
|
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||||
|
SOURCE_1233_STATUS=OK
|
||||||
|
SOURCE_1233_RC=0
|
||||||
|
SOURCE_1233_CHANGES_MADE=false
|
||||||
|
SOURCE_1233_MUTATION_OUTCOME=NO_MUTATION
|
||||||
|
SOURCE_1233_DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT
|
||||||
|
SOURCE_1233_NEXT_GATE=READ_EXACT_1233_AND_RECONCILE_NON_RPO_ISSUE
|
||||||
|
SOURCE_1233_AUDIT0_ISSUE_COUNT=12
|
||||||
|
SOURCE_1233_AUDIT0_RPO_COUNT=11
|
||||||
|
SOURCE_1233_AUDIT0_NON_RPO_COUNT=1
|
||||||
|
=== LIVE_IDENTITY ===
|
||||||
|
WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
|
||||||
|
HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
|
||||||
|
VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
|
||||||
|
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
|
||||||
|
=== LOCK_FILE_METADATA ===
|
||||||
|
LOCK_FILE=/run/homelab-backup-heavy.lock|600|root:root|0|2026-09-27 22:58:26.543174461 +0300|inode=3474
|
||||||
|
LOCK_FILE=/run/lock/homelab-backup-dispatcher.lock|600|root:root|0|2026-09-27 03:27:41.596907725 +0300|inode=9
|
||||||
|
LOCK_FILE=/run/lock/homelab-vm-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 03:27:41.597907739 +0300|inode=2944
|
||||||
|
LOCK_FILE=/run/lock/homelab-app-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 09:30:02.178976732 +0300|inode=15
|
||||||
|
=== LSLOCKS_BEFORE_AUDIT ===
|
||||||
|
kvm 4446 POSIX WRITE /run/qemu-server/150.pid
|
||||||
|
bash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock
|
||||||
|
corosync 1463 POSIX WRITE /run/corosync.pid
|
||||||
|
cron 3999 FLOCK WRITE /run/crond.pid
|
||||||
|
lxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo
|
||||||
|
cron 1464 FLOCK WRITE /run/crond.pid
|
||||||
|
kvm 4335 POSIX WRITE /run/qemu-server/171.pid
|
||||||
|
kvm 1100046 POSIX WRITE /run/qemu-server/100.pid
|
||||||
|
kvm 4237 POSIX WRITE /run/qemu-server/170.pid
|
||||||
|
kvm 342771 POSIX WRITE /run/qemu-server/210.pid
|
||||||
|
pvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock
|
||||||
|
pvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock
|
||||||
|
spiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock
|
||||||
|
pve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock
|
||||||
|
pve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock
|
||||||
|
pvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock
|
||||||
|
pveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock
|
||||||
|
pve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock
|
||||||
|
dmeventd 531 POSIX WRITE /run/dmeventd.pid
|
||||||
|
cron 3538 FLOCK WRITE /run/crond.pid
|
||||||
|
zed 952 POSIX WRITE /run/zed.pid
|
||||||
|
zed 952 POSIX WRITE /run/zed.state
|
||||||
|
lxcfs 963 POSIX WRITE /run/lxcfs.pid
|
||||||
|
pvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck
|
||||||
|
=== FUSER_BEFORE_AUDIT ===
|
||||||
|
FUSER_PATH=/run/homelab-backup-heavy.lock PIDS=
|
||||||
|
FUSER_PATH=/run/lock/homelab-backup-dispatcher.lock PIDS=
|
||||||
|
FUSER_PATH=/run/lock/homelab-vm-cloud-quorum-queue.lock PIDS=
|
||||||
|
FUSER_PATH=/run/lock/homelab-app-cloud-quorum-queue.lock PIDS=
|
||||||
|
=== AUDIT_LOCK_SOURCE ===
|
||||||
|
183- ("homelab-alertmanager-backup-health.timer","ALERT_TIMER_BAD"),
|
||||||
|
184-]:
|
||||||
|
185- ok,a,e=unit_ok(unit)
|
||||||
|
186- if not ok: issues.append({"severity":"HIGH","code":code,"logical_id":unit,"root_cause":"UNIT_"+a+"_"+e})
|
||||||
|
187-
|
||||||
|
188:for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]:
|
||||||
|
189- p=pathlib.Path(lock)
|
||||||
|
190- if p.exists():
|
||||||
|
191: lrc,_,_=run(["flock","-n",str(p),"-c","true"],5)
|
||||||
|
192: if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
|
||||||
|
193-
|
||||||
|
194-env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG)
|
||||||
|
195-grc,gout,_=run([GFSH,"validate"],20,env)
|
||||||
|
196-if grc!=0 or "GFS_POLICY=OK" not in gout:
|
||||||
|
197- issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"})
|
||||||
|
=== AUDIT_ALL_NO_OWNED_HEAVY_LOCK ===
|
||||||
|
AUDIT_ALL_RC=1
|
||||||
|
AUDIT_ALL_STDERR_BYTES=0
|
||||||
|
AUDIT_ISSUES_BEGIN
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
AUDIT_ISSUES_END
|
||||||
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||||
|
AUDIT_RPO_COUNT=11
|
||||||
|
AUDIT_NON_RPO_COUNT=0
|
||||||
|
AUDIT_NON_RPO_ISSUES_BEGIN
|
||||||
|
AUDIT_NON_RPO_ISSUES_END
|
||||||
|
AUDIT_ALL_HEALTH_NO_WRITE=true
|
||||||
|
=== LSLOCKS_AFTER_AUDIT ===
|
||||||
|
kvm 4446 POSIX WRITE /run/qemu-server/150.pid
|
||||||
|
bash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock
|
||||||
|
corosync 1463 POSIX WRITE /run/corosync.pid
|
||||||
|
cron 3999 FLOCK WRITE /run/crond.pid
|
||||||
|
lxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo
|
||||||
|
cron 1464 FLOCK WRITE /run/crond.pid
|
||||||
|
kvm 4335 POSIX WRITE /run/qemu-server/171.pid
|
||||||
|
kvm 1100046 POSIX WRITE /run/qemu-server/100.pid
|
||||||
|
kvm 4237 POSIX WRITE /run/qemu-server/170.pid
|
||||||
|
kvm 342771 POSIX WRITE /run/qemu-server/210.pid
|
||||||
|
pvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock
|
||||||
|
pvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock
|
||||||
|
spiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock
|
||||||
|
pve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock
|
||||||
|
pve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock
|
||||||
|
pvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock
|
||||||
|
pveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock
|
||||||
|
pve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock
|
||||||
|
dmeventd 531 POSIX WRITE /run/dmeventd.pid
|
||||||
|
cron 3538 FLOCK WRITE /run/crond.pid
|
||||||
|
zed 952 POSIX WRITE /run/zed.pid
|
||||||
|
zed 952 POSIX WRITE /run/zed.state
|
||||||
|
lxcfs 963 POSIX WRITE /run/lxcfs.pid
|
||||||
|
pvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck
|
||||||
|
=== 1233_RUNNER_RAW_LOCK_EVIDENCE ===
|
||||||
|
RAW1233_PATH=/var/lib/homelab-chat-run/raw/SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1-20260927T195820Z.log
|
||||||
|
29:BACKUP_HEAVY_LOCK_ACQUIRED=true
|
||||||
|
33:=== FRESH_AUDIT_UNDER_LOCK ===
|
||||||
|
37:AUDIT0_NON_RPO_COUNT=1
|
||||||
|
53:DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT
|
||||||
|
TASK_RESULT=PASS_NONRPO_LOCK_RCA_COLLECTED
|
||||||
|
DECISION=PASS_NONRPO_LOCK_RCA_COLLECTED
|
||||||
|
LOCK_SELF_INDUCED_PROVISIONAL=true
|
||||||
|
NEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT
|
||||||
|
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||||
|
|
||||||
|
OUTPUT_END
|
||||||
|
CHAT_OUTPUT_END
|
||||||
+11
-11
File diff suppressed because one or more lines are too long
+131
-54
@@ -1,20 +1,20 @@
|
|||||||
CHAT_OUTPUT_BEGIN
|
CHAT_OUTPUT_BEGIN
|
||||||
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1
|
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1
|
||||||
STATUS=OK
|
STATUS=OK
|
||||||
RC=0
|
RC=0
|
||||||
HOST=pve01
|
HOST=pve01
|
||||||
MODE=change
|
MODE=read-only
|
||||||
COMPONENT=homelab-backup-adaptive-rpo-recovery
|
COMPONENT=homelab-backup-nonrpo-lock-rca
|
||||||
REFERENCE_REGISTER_CHECK=OK
|
REFERENCE_REGISTER_CHECK=OK
|
||||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||||
ERROR_REGISTER_CHECK=OK
|
ERROR_REGISTER_CHECK=OK
|
||||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||||
COMMAND_SHA256=380add3ede2fd9137f06e7c75aad3301d45af2b01ab9cd6cc19daaa510387ece
|
COMMAND_SHA256=b775bb0d8b318e704bf86baa42c51f6a73c0c0f57d7358ecb9a30b5d0e8cd341
|
||||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||||
EXECUTION_STARTED=true
|
EXECUTION_STARTED=true
|
||||||
CHANGE_DECLARED=true
|
CHANGE_DECLARED=false
|
||||||
RESULT_CONTRACT_VALID=true
|
RESULT_CONTRACT_VALID=true
|
||||||
RESULT_CONTRACT_STATUS=OK
|
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||||
RESULT_CONTRACT_ERROR=NONE
|
RESULT_CONTRACT_ERROR=NONE
|
||||||
COMMAND_RC=0
|
COMMAND_RC=0
|
||||||
CHANGES_MADE=false
|
CHANGES_MADE=false
|
||||||
@@ -24,64 +24,141 @@ MUTATION_OUTCOME=NO_MUTATION
|
|||||||
SANITIZED=yes
|
SANITIZED=yes
|
||||||
SECRETS_INCLUDED=no
|
SECRETS_INCLUDED=no
|
||||||
PRIVATE_ADDRESSES_INCLUDED=no
|
PRIVATE_ADDRESSES_INCLUDED=no
|
||||||
RAW_EVIDENCE_SHA256=4a832057af33457c2a74ea89cfc77fe0ee10af72ec95c10818e3396b7e293f12
|
RAW_EVIDENCE_SHA256=5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7
|
||||||
SANITIZED_OUTPUT_SHA256=4a832057af33457c2a74ea89cfc77fe0ee10af72ec95c10818e3396b7e293f12
|
SANITIZED_OUTPUT_SHA256=5edf9016f579b8e7598db06143582804c2a921fcc75974612f085b8e3ca649c7
|
||||||
OUTPUT_BEGIN
|
OUTPUT_BEGIN
|
||||||
BACKUP_ADAPTIVE_RPO_RECOVERY_BEGIN=true
|
BACKUP_NONRPO_LOCK_RCA_BEGIN=true
|
||||||
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1
|
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1
|
||||||
MODE=change
|
MODE=read-only
|
||||||
COMPONENT=homelab-backup-adaptive-rpo-recovery
|
COMPONENT=homelab-backup-nonrpo-lock-rca
|
||||||
MUTATION_BOUNDARY=ONLY_CURRENT_V2_VERIFY_TARGET_FOR_FRESH_AUDIT_RPO_BREACHES;EXCLUDES_CT110_ALREADY_FRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE
|
MUTATION_BOUNDARY=READ_ONLY_AUDIT_ALL_LOCK_OWNERSHIP_AND_AUDIT_SOURCE;NO_LOCK_ACQUIRE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE
|
||||||
=== AUTHORITY ===
|
=== AUTHORITY ===
|
||||||
HOST=pve01
|
HOST=pve01
|
||||||
CLUSTER_QUORATE=Yes
|
CLUSTER_QUORATE=Yes
|
||||||
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||||
SOURCE_1232_STATUS=OK
|
SOURCE_1233_STATUS=OK
|
||||||
SOURCE_1232_RC=0
|
SOURCE_1233_RC=0
|
||||||
SOURCE_1232_DECISION=PASS_CT110_STATE_AUDIT_RECONCILE
|
SOURCE_1233_CHANGES_MADE=false
|
||||||
SOURCE_1232_NEXT_GATE=ADAPTIVE_RPO_RECOVERY_FROM_FRESH_AUDIT
|
SOURCE_1233_MUTATION_OUTCOME=NO_MUTATION
|
||||||
SOURCE_1232_STATE_ACCEPTED=true
|
SOURCE_1233_DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT
|
||||||
SOURCE_1232_SUCCESS_MAIL_POLICY_ACCEPTED=true
|
SOURCE_1233_NEXT_GATE=READ_EXACT_1233_AND_RECONCILE_NON_RPO_ISSUE
|
||||||
SOURCE_1232_CT110_RPO_COUNT=0
|
SOURCE_1233_AUDIT0_ISSUE_COUNT=12
|
||||||
|
SOURCE_1233_AUDIT0_RPO_COUNT=11
|
||||||
|
SOURCE_1233_AUDIT0_NON_RPO_COUNT=1
|
||||||
=== LIVE_IDENTITY ===
|
=== LIVE_IDENTITY ===
|
||||||
WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
|
WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
|
||||||
HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
|
HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
|
||||||
VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
|
VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
|
||||||
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
|
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
|
||||||
SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
|
=== LOCK_FILE_METADATA ===
|
||||||
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
LOCK_FILE=/run/homelab-backup-heavy.lock|600|root:root|0|2026-09-27 22:58:26.543174461 +0300|inode=3474
|
||||||
SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
|
LOCK_FILE=/run/lock/homelab-backup-dispatcher.lock|600|root:root|0|2026-09-27 03:27:41.596907725 +0300|inode=9
|
||||||
SOURCE_WORKTREE_CLEAN=true
|
LOCK_FILE=/run/lock/homelab-vm-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 03:27:41.597907739 +0300|inode=2944
|
||||||
=== OWNER_WAIT ===
|
LOCK_FILE=/run/lock/homelab-app-cloud-quorum-queue.lock|600|root:root|0|2026-09-27 09:30:02.178976732 +0300|inode=15
|
||||||
OWNER_WAIT_ITERATION=1 ACTIVE=inactive PID=0
|
=== LSLOCKS_BEFORE_AUDIT ===
|
||||||
OWNER_WAIT_ITERATION=2 ACTIVE=inactive PID=0
|
kvm 4446 POSIX WRITE /run/qemu-server/150.pid
|
||||||
BACKUP_HEAVY_LOCK_ACQUIRED=true
|
bash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock
|
||||||
ACTIVE_VZDUMP_COUNT=0
|
corosync 1463 POSIX WRITE /run/corosync.pid
|
||||||
RECOVERY_START_EPOCH=1790539106
|
cron 3999 FLOCK WRITE /run/crond.pid
|
||||||
RECOVERY_START_UTC=2026-09-27T19:58:26Z
|
lxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo
|
||||||
=== FRESH_AUDIT_UNDER_LOCK ===
|
cron 1464 FLOCK WRITE /run/crond.pid
|
||||||
AUDIT0_RC=1
|
kvm 4335 POSIX WRITE /run/qemu-server/171.pid
|
||||||
AUDIT0_ISSUE_COUNT=12
|
kvm 1100046 POSIX WRITE /run/qemu-server/100.pid
|
||||||
AUDIT0_RPO_COUNT=11
|
kvm 4237 POSIX WRITE /run/qemu-server/170.pid
|
||||||
AUDIT0_NON_RPO_COUNT=1
|
kvm 342771 POSIX WRITE /run/qemu-server/210.pid
|
||||||
AUDIT0_RPO_SET_BEGIN
|
pvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock
|
||||||
control-bootstrap-v2
|
pvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock
|
||||||
vm111
|
spiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock
|
||||||
vm112
|
pve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock
|
||||||
vm113
|
pve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock
|
||||||
vm130
|
pvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock
|
||||||
vm150
|
pveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock
|
||||||
vm160
|
pve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock
|
||||||
vm170
|
dmeventd 531 POSIX WRITE /run/dmeventd.pid
|
||||||
vm171
|
cron 3538 FLOCK WRITE /run/crond.pid
|
||||||
vm190
|
zed 952 POSIX WRITE /run/zed.pid
|
||||||
vm9130
|
zed 952 POSIX WRITE /run/zed.state
|
||||||
AUDIT0_RPO_SET_END
|
lxcfs 963 POSIX WRITE /run/lxcfs.pid
|
||||||
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
pvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck
|
||||||
TASK_RESULT=HOLD_ADAPTIVE_RPO_RECOVERY
|
=== FUSER_BEFORE_AUDIT ===
|
||||||
DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT
|
FUSER_PATH=/run/homelab-backup-heavy.lock PIDS=
|
||||||
NEXT_GATE=READ_EXACT_1233_AND_RECONCILE_NON_RPO_ISSUE
|
FUSER_PATH=/run/lock/homelab-backup-dispatcher.lock PIDS=
|
||||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
FUSER_PATH=/run/lock/homelab-vm-cloud-quorum-queue.lock PIDS=
|
||||||
|
FUSER_PATH=/run/lock/homelab-app-cloud-quorum-queue.lock PIDS=
|
||||||
|
=== AUDIT_LOCK_SOURCE ===
|
||||||
|
183- ("homelab-alertmanager-backup-health.timer","ALERT_TIMER_BAD"),
|
||||||
|
184-]:
|
||||||
|
185- ok,a,e=unit_ok(unit)
|
||||||
|
186- if not ok: issues.append({"severity":"HIGH","code":code,"logical_id":unit,"root_cause":"UNIT_"+a+"_"+e})
|
||||||
|
187-
|
||||||
|
188:for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]:
|
||||||
|
189- p=pathlib.Path(lock)
|
||||||
|
190- if p.exists():
|
||||||
|
191: lrc,_,_=run(["flock","-n",str(p),"-c","true"],5)
|
||||||
|
192: if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
|
||||||
|
193-
|
||||||
|
194-env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG)
|
||||||
|
195-grc,gout,_=run([GFSH,"validate"],20,env)
|
||||||
|
196-if grc!=0 or "GFS_POLICY=OK" not in gout:
|
||||||
|
197- issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"})
|
||||||
|
=== AUDIT_ALL_NO_OWNED_HEAVY_LOCK ===
|
||||||
|
AUDIT_ALL_RC=1
|
||||||
|
AUDIT_ALL_STDERR_BYTES=0
|
||||||
|
AUDIT_ISSUES_BEGIN
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||||
|
AUDIT_ISSUES_END
|
||||||
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||||
|
AUDIT_RPO_COUNT=11
|
||||||
|
AUDIT_NON_RPO_COUNT=0
|
||||||
|
AUDIT_NON_RPO_ISSUES_BEGIN
|
||||||
|
AUDIT_NON_RPO_ISSUES_END
|
||||||
|
AUDIT_ALL_HEALTH_NO_WRITE=true
|
||||||
|
=== LSLOCKS_AFTER_AUDIT ===
|
||||||
|
kvm 4446 POSIX WRITE /run/qemu-server/150.pid
|
||||||
|
bash 380990 FLOCK WRITE /var/lib/homelab-chat-run/state/runner.lock
|
||||||
|
corosync 1463 POSIX WRITE /run/corosync.pid
|
||||||
|
cron 3999 FLOCK WRITE /run/crond.pid
|
||||||
|
lxc-monitord 1218 POSIX WRITE /run/lxc/var/lib/lxc/monitor-fifo
|
||||||
|
cron 1464 FLOCK WRITE /run/crond.pid
|
||||||
|
kvm 4335 POSIX WRITE /run/qemu-server/171.pid
|
||||||
|
kvm 1100046 POSIX WRITE /run/qemu-server/100.pid
|
||||||
|
kvm 4237 POSIX WRITE /run/qemu-server/170.pid
|
||||||
|
kvm 342771 POSIX WRITE /run/qemu-server/210.pid
|
||||||
|
pvestatd 1492 FLOCK WRITE /run/pvestatd.pid.lock
|
||||||
|
pvedaemon 1556 FLOCK WRITE /run/pvedaemon.pid.lock
|
||||||
|
spiceproxy 3243 FLOCK WRITE /run/pveproxy/spiceproxy.pid.lock
|
||||||
|
pve-firewall 1494 FLOCK WRITE /run/pve-firewall.pid.lock
|
||||||
|
pve-ha-crm 1564 FLOCK WRITE /run/pve-ha-crm.pid.lock
|
||||||
|
pvescheduler 4709 FLOCK WRITE /run/pvescheduler.pid.lock
|
||||||
|
pveproxy 3074 FLOCK WRITE /run/pveproxy/pveproxy.pid.lock
|
||||||
|
pve-ha-lrm 3297 FLOCK WRITE /run/pve-ha-lrm.pid.lock
|
||||||
|
dmeventd 531 POSIX WRITE /run/dmeventd.pid
|
||||||
|
cron 3538 FLOCK WRITE /run/crond.pid
|
||||||
|
zed 952 POSIX WRITE /run/zed.pid
|
||||||
|
zed 952 POSIX WRITE /run/zed.state
|
||||||
|
lxcfs 963 POSIX WRITE /run/lxcfs.pid
|
||||||
|
pvefw-logger 220239 FLOCK WRITE /run/lock/pvefw-logger.lck
|
||||||
|
=== 1233_RUNNER_RAW_LOCK_EVIDENCE ===
|
||||||
|
RAW1233_PATH=/var/lib/homelab-chat-run/raw/SUPPORT-260927-HOMELAB-BACKUP-ADAPTIVE-RPO-RECOVERY-1233R1-20260927T195820Z.log
|
||||||
|
29:BACKUP_HEAVY_LOCK_ACQUIRED=true
|
||||||
|
33:=== FRESH_AUDIT_UNDER_LOCK ===
|
||||||
|
37:AUDIT0_NON_RPO_COUNT=1
|
||||||
|
53:DECISION=HOLD_1233_NON_RPO_ISSUE_PRESENT
|
||||||
|
TASK_RESULT=PASS_NONRPO_LOCK_RCA_COLLECTED
|
||||||
|
DECISION=PASS_NONRPO_LOCK_RCA_COLLECTED
|
||||||
|
LOCK_SELF_INDUCED_PROVISIONAL=true
|
||||||
|
NEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT
|
||||||
|
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-NONRPO-LOCK-RCA-1234R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||||
|
|
||||||
OUTPUT_END
|
OUTPUT_END
|
||||||
CHAT_OUTPUT_END
|
CHAT_OUTPUT_END
|
||||||
|
|||||||
Reference in New Issue
Block a user