homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-09-27 19:51:15 +00:00
parent b0710da07f
commit 32e01e035d
4 changed files with 332 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1.json
+38
View File
File diff suppressed because one or more lines are too long
runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1.txt
+158
View File
@@ -0,0 +1,158 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-ct110-state-audit-reconcile
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=04f4eb8b36878aa3a82b87657a987381666519b93c3a0218c69e5cf1062cefae
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922
SANITIZED_OUTPUT_SHA256=610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922
OUTPUT_BEGIN
BACKUP_CT110_STATE_AUDIT_RECONCILE_BEGIN=true
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1
MODE=read-only
COMPONENT=homelab-backup-ct110-state-audit-reconcile
MUTATION_BOUNDARY=READ_ONLY_STATE_JOURNAL_SYSTEMD_AND_AUDIT_ALL;AUDIT_ALL_PROVEN_NO_HEALTH_WRITE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE
=== AUTHORITY ===
HOST=pve01
CLUSTER_QUORATE=Yes
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
SOURCE_1231_STATUS=OK
SOURCE_1231_RC=0
SOURCE_1231_DECISION=HOLD_1231_POST_STATE_MISMATCH
SOURCE_1231_NEXT_GATE=READ_EXACT_1231_AND_RECONCILE_CT110_STATE
SOURCE_1231_VERIFY_RESULT_ACCEPTED=true
SOURCE_1231_VERIFY_RESULT_REPLICAS=2
SOURCE_1231_ACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17
SOURCE_1231_VERIFY_START_EPOCH=1790538076
SOURCE_1231_POST_LAST_SUCCESS_EPOCH=1790538303
=== LIVE_IDENTITY ===
WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
=== CT110_STATE_SCHEMA_RECONCILE ===
STATE_EXISTS=true
STATE_SHA256=ee7999edad0bf4c2988edfdd5059b5e4d33899cd0880aaf2567f647eaeab8033
STATE_KEYS=consecutive_failures,failure_backoff_seconds,generation,green_verified,last_attempt_epoch,last_success_epoch,logical_id,manifest_sha256,payload_sha256,replicas,result,retry_after_epoch,schema,updated_at_utc
STATE_SCHEMA=1
STATE_LOGICAL_ID=ct110
STATE_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17
STATE_RESULT=SUCCESS
STATE_LAST_ATTEMPT_EPOCH=1790538303
STATE_LAST_SUCCESS_EPOCH=1790538303
STATE_CONSECUTIVE_FAILURES=0
STATE_FAILURE_BACKOFF_SECONDS=0
STATE_REPLICAS_TYPE=list
STATE_REPLICA_COUNT_DERIVED=2
STATE_REPLICA_1_SCHEMA_OK=true
STATE_REPLICA_1_PHYSICAL_SHA256=c3b0138acb547907e498f9601c9a6cbb320da8e916ed685b30a4e35b92d0a247
STATE_REPLICA_1_CRYPT_SHA256=d2d63811693130d9b521ec13bc62c80ff0f7a557f0a1b605db30541858a808f1
STATE_REPLICA_2_SCHEMA_OK=true
STATE_REPLICA_2_PHYSICAL_SHA256=cf59a41c57e69eaac77ef340b75ff146dd1cfe5da28674e06050730912385bf3
STATE_REPLICA_2_CRYPT_SHA256=7218b750518d1a6bd57398eee3cd19e397c587adeda6dfff4cbec1aeb50eb1c8
STATE_REPLICA_PHYSICAL_UNIQUE=true
STATE_REPLICA_CRYPT_UNIQUE=true
STATE_REPLICA_PAIR_UNIQUE=true
STATE_CHECK_LOGICAL_ID=true
STATE_CHECK_GENERATION=true
STATE_CHECK_RESULT=true
STATE_CHECK_REPLICAS=true
STATE_CHECK_LAST_SUCCESS=true
STATE_CHECK_LAST_ATTEMPT=true
STATE_ACCEPTED=true
STATE_RECONCILE_RC=0
=== 1231_NOTIFICATION_RUNTIME_RECONCILE ===
VERIFY_WINDOW_START_EPOCH=1790538076
VERIFY_WINDOW_END_EPOCH=1790538483
SUCCESS_BACKUP_EVENT_COUNT=1
MAIL_TO_ROOT_EVENT_COUNT=0
CT110_SUCCESS_EVIDENCE_COUNT=2
SUCCESS_MAIL_POLICY_ACCEPTED=true
=== AUDIT_MODE_SAFETY ===
AUDIT_SHA256_PRE=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
AUDIT_CONTRACT_RC=0
AUDIT_CONTRACT=v2-plus-sealed-quorum
AUDIT_SCHEMA=homelab-backup-audit-v3
AUDIT_CONTRACT_STDERR_BYTES=0
311:if MODE in ("write-health","scheduled"):
AUDIT_WRITE_GUARD_COUNT=1
HEALTH1_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2
HEALTH2_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2
HEALTH1_MTIME_PRE=1790537775
HEALTH2_MTIME_PRE=1790537775
=== FRESH_OFFICIAL_AUDIT_ALL ===
AUDIT_ALL_RC=1
AUDIT_ALL_STDERR_BYTES=0
AUDIT_OUTPUT_BEGIN
TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48057|generation=app-filebrowser-pve01-20260927T063003Z
TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47890|generation=app-hbarhub-pve01-20260927T063154Z
TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47991|generation=app-immich-pve01-20260927T063041Z
TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48038|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z
TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=STALE|age_seconds=234007|generation=control-bootstrap-v2-pve01-20260916T120057Z
TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=366|legacy_age_seconds=2038301|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17
TARGET|logical_id=vm111|kind=VM|state=STALE|source=NONE|v2_age_seconds=283415|replica_count=2|state_id=ct111
TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=266907|legacy_age_seconds=2076796|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z
TARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=264888|replica_count=2|state_id=ct113
TARGET|logical_id=vm130|kind=VM|state=STALE|source=NONE|v2_age_seconds=259498|replica_count=2|state_id=vm130
TARGET|logical_id=vm150|kind=VM|state=STALE|source=NONE|v2_age_seconds=262950|legacy_age_seconds=2038417|replica_count=2|state_id=vm150|generation=vm150-pve01-20260904T053517Z
TARGET|logical_id=vm160|kind=VM|state=STALE|source=NONE|v2_age_seconds=258251|replica_count=2|state_id=vm160
TARGET|logical_id=vm170|kind=VM|state=STALE|source=NONE|v2_age_seconds=235042|legacy_age_seconds=983038|replica_count=2|state_id=vm170|generation=vm170-pve01-20260916T103019Z
TARGET|logical_id=vm171|kind=VM|state=STALE|source=NONE|v2_age_seconds=239472|legacy_age_seconds=1012570|replica_count=2|state_id=vm171|generation=vm171-pve01-20260916T023018Z
TARGET|logical_id=vm190|kind=VM|state=STALE|source=NONE|v2_age_seconds=235640|replica_count=2|state_id=vm190
TARGET|logical_id=vm9130|kind=VM|state=STALE|source=NONE|v2_age_seconds=226897|replica_count=2|state_id=vm9130
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
AUDIT_OUTPUT_END
AUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
AUDIT_BINARY_UNCHANGED=true
HEALTH1_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2
HEALTH2_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2
HEALTH1_MTIME_POST=1790537775
HEALTH2_MTIME_POST=1790537775
AUDIT_ALL_HEALTH_NO_WRITE=true
FRESH_AUDIT_ISSUE_COUNT=11
FRESH_AUDIT_RPO_BREACH_COUNT=0
FRESH_AUDIT_CT110_RPO_COUNT=0
FRESH_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
=== CURRENT_UNIT_TIMER_STATE ===
UNIT=homelab-backup-v2.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0
UNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID=
UNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0
UNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID=
=== RECENT_SCHEDULER_AFTER_HOTFIX ===
TASK_RESULT=PASS_CT110_STATE_AUDIT_RECONCILE
DECISION=PASS_CT110_STATE_AUDIT_RECONCILE
NEXT_GATE=ADAPTIVE_RPO_RECOVERY_FROM_FRESH_AUDIT
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+13 -13
View File
File diff suppressed because one or more lines are too long
runtime/latest.txt
+123 -89
View File
@@ -1,124 +1,158 @@
CHAT_OUTPUT_BEGIN CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-TARGETED-VERIFY-1231R1 COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1
STATUS=OK STATUS=OK
RC=0 RC=0
HOST=pve01 HOST=pve01
MODE=change MODE=read-only
COMPONENT=homelab-backup-ct110-targeted-verify COMPONENT=homelab-backup-ct110-state-audit-reconcile
REFERENCE_REGISTER_CHECK=OK REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=5a48244cd686dffbe11630b2c2fbced81680daec26b2e3a8acaeaca1e83b4cb2 COMMAND_SHA256=04f4eb8b36878aa3a82b87657a987381666519b93c3a0218c69e5cf1062cefae
DUPLICATE_FAILED_COMMAND_BLOCKED=false DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true EXECUTION_STARTED=true
CHANGE_DECLARED=true CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=OK RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0 COMMAND_RC=0
CHANGES_MADE=true CHANGES_MADE=false
ROLLBACK_STARTED=false ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null ROLLBACK_RESTORED=null
MUTATION_OUTCOME=MUTATED MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes SANITIZED=yes
SECRETS_INCLUDED=no SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=8330976ae073c2483c486fdd01691dfbf2a8d74f1d1d774338bac2fca89f7705 RAW_EVIDENCE_SHA256=610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922
SANITIZED_OUTPUT_SHA256=8330976ae073c2483c486fdd01691dfbf2a8d74f1d1d774338bac2fca89f7705 SANITIZED_OUTPUT_SHA256=610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922
OUTPUT_BEGIN OUTPUT_BEGIN
BACKUP_CT110_TARGETED_VERIFY_BEGIN=true BACKUP_CT110_STATE_AUDIT_RECONCILE_BEGIN=true
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-TARGETED-VERIFY-1231R1 COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1
MODE=change MODE=read-only
COMPONENT=homelab-backup-ct110-targeted-verify COMPONENT=homelab-backup-ct110-state-audit-reconcile
MUTATION_BOUNDARY=ONE_CURRENT_V2_VERIFY_TARGET_CT110_ONLY;MAY_CREATE_NEW_CT110_BACKUP_GENERATION_AND_UPDATE_CT110_STATE;NO_OTHER_TARGET_REFRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_AUDIT_RUN;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE MUTATION_BOUNDARY=READ_ONLY_STATE_JOURNAL_SYSTEMD_AND_AUDIT_ALL;AUDIT_ALL_PROVEN_NO_HEALTH_WRITE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE
=== AUTHORITY === === AUTHORITY ===
HOST=pve01 HOST=pve01
CLUSTER_QUORATE=Yes CLUSTER_QUORATE=Yes
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
SOURCE_1230_STATUS=OK SOURCE_1231_STATUS=OK
SOURCE_1230_RC=0 SOURCE_1231_RC=0
SOURCE_1230_DECISION=PASS_BACKUP_NOTIFICATION_COMPAT_HOTFIX_DEPLOYED SOURCE_1231_DECISION=HOLD_1231_POST_STATE_MISMATCH
SOURCE_1230_NEXT_GATE=TARGETED_CT110_VERIFY_AND_ADAPTIVE_RPO_RECOVERY SOURCE_1231_NEXT_GATE=READ_EXACT_1231_AND_RECONCILE_CT110_STATE
SOURCE_1231_VERIFY_RESULT_ACCEPTED=true
SOURCE_1231_VERIFY_RESULT_REPLICAS=2
SOURCE_1231_ACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17
SOURCE_1231_VERIFY_START_EPOCH=1790538076
SOURCE_1231_POST_LAST_SUCCESS_EPOCH=1790538303
=== LIVE_IDENTITY === === LIVE_IDENTITY ===
WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
RELEASE_META=500:root:root === CT110_STATE_SCHEMA_RECONCILE ===
HELPER_META=400:root:root STATE_EXISTS=true
SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STATE_SHA256=ee7999edad0bf4c2988edfdd5059b5e4d33899cd0880aaf2567f647eaeab8033
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation STATE_KEYS=consecutive_failures,failure_backoff_seconds,generation,green_verified,last_attempt_epoch,last_success_epoch,logical_id,manifest_sha256,payload_sha256,replicas,result,retry_after_epoch,schema,updated_at_utc
SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STATE_SCHEMA=1
SOURCE_WORKTREE_CLEAN=true STATE_LOGICAL_ID=ct110
=== OWNER_WAIT === STATE_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17
OWNER_WAIT_ITERATION=1 ACTIVE=failed SUB=failed PID=0 STATE_RESULT=SUCCESS
OWNER_WAIT_ITERATION=2 ACTIVE=failed SUB=failed PID=0 STATE_LAST_ATTEMPT_EPOCH=1790538303
BACKUP_HEAVY_LOCK_ACQUIRED=true STATE_LAST_SUCCESS_EPOCH=1790538303
POSTLOCK_SERVICE_ACTIVE=failed STATE_CONSECUTIVE_FAILURES=0
POSTLOCK_SERVICE_MAINPID=0 STATE_FAILURE_BACKOFF_SECONDS=0
ACTIVE_VZDUMP_COUNT=0 STATE_REPLICAS_TYPE=list
=== PRE_STATE === STATE_REPLICA_COUNT_DERIVED=2
PRE_STATE_EXISTS=true STATE_REPLICA_1_SCHEMA_OK=true
PRE_STATE_SHA256=b3595df8d1890c26b0a93dc5e38722294b617d98431bb2693d7b05a5356ae4e8 STATE_REPLICA_1_PHYSICAL_SHA256=c3b0138acb547907e498f9601c9a6cbb320da8e916ed685b30a4e35b92d0a247
PRE_STATE_JSON_OK=true STATE_REPLICA_1_CRYPT_SHA256=d2d63811693130d9b521ec13bc62c80ff0f7a557f0a1b605db30541858a808f1
PRE_LOGICAL_ID=ct110 STATE_REPLICA_2_SCHEMA_OK=true
PRE_GENERATION=v2-ct110-20260924T123032Z-e893686ccca9 STATE_REPLICA_2_PHYSICAL_SHA256=cf59a41c57e69eaac77ef340b75ff146dd1cfe5da28674e06050730912385bf3
PRE_RESULT=SUCCESS STATE_REPLICA_2_CRYPT_SHA256=7218b750518d1a6bd57398eee3cd19e397c587adeda6dfff4cbec1aeb50eb1c8
PRE_REPLICA_COUNT=None STATE_REPLICA_PHYSICAL_UNIQUE=true
PRE_LAST_ATTEMPT_EPOCH=1790253235 STATE_REPLICA_CRYPT_UNIQUE=true
PRE_LAST_SUCCESS_EPOCH=1790253235 STATE_REPLICA_PAIR_UNIQUE=true
PRE_FAILURE_BACKOFF_SECONDS=0
PRE_CONSECUTIVE_FAILURES=0
VERIFY_START_EPOCH=1790538076
VERIFY_START_UTC=2026-09-27T19:41:16Z
=== TARGETED_VERIFY_CT110 ===
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=561
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=9c399e7d401ea4dfdcd9f610e024d5728e97b27bccfb018cd9bf12cfb2d934f8
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN
SELFTEST_REPLACEMENT486=PASS
LXC_EXECUTION_DIAG={"archive_bytes":566926006,"exception_class":null,"exception_message_sha256":null,"executor":"local-proven-pve270","helper_rc":0,"node":"pve01","producer_rc":0,"stderr_bytes_observed":1364,"stderr_bytes_retained":1364,"stderr_stream_sha256":"7e6320861d4dbb5b6ecba9fb36e611b408b8e36628004a0d266845f8f93b513a","stderr_truncated":false}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct110-20260927T194116Z-d16fe26dbd17","logical_id":"ct110","payload_bytes":566926006,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=ct110
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=566926006
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
ACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17
ACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17
=== POST_STATE ===
POST_STATE_EXISTS=true
POST_STATE_SHA256=ee7999edad0bf4c2988edfdd5059b5e4d33899cd0880aaf2567f647eaeab8033
POST_STATE_JSON_OK=true
POST_LOGICAL_ID=ct110
POST_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17
POST_RESULT=SUCCESS
POST_REPLICA_COUNT=None
POST_LAST_ATTEMPT_EPOCH=1790538303
POST_LAST_SUCCESS_EPOCH=1790538303
POST_FAILURE_BACKOFF_SECONDS=0
POST_CONSECUTIVE_FAILURES=0
STATE_CHECK_LOGICAL_ID=true STATE_CHECK_LOGICAL_ID=true
STATE_CHECK_GENERATION=true STATE_CHECK_GENERATION=true
STATE_CHECK_RESULT=true STATE_CHECK_RESULT=true
STATE_CHECK_REPLICA_COUNT=false STATE_CHECK_REPLICAS=true
STATE_CHECK_LAST_SUCCESS=true STATE_CHECK_LAST_SUCCESS=true
STATE_CHECK_LAST_ATTEMPT=true STATE_CHECK_LAST_ATTEMPT=true
STATE_ACCEPTED=false STATE_ACCEPTED=true
CHANGES_MADE=true STATE_RECONCILE_RC=0
ROLLBACK_STARTED=false === 1231_NOTIFICATION_RUNTIME_RECONCILE ===
ROLLBACK_RESTORED=null VERIFY_WINDOW_START_EPOCH=1790538076
MUTATION_OUTCOME=MUTATED_OR_ATTEMPTED_BACKUP VERIFY_WINDOW_END_EPOCH=1790538483
TASK_RESULT=HOLD_CT110_TARGETED_VERIFY_AFTER_ATTEMPT SUCCESS_BACKUP_EVENT_COUNT=1
DECISION=HOLD_1231_POST_STATE_MISMATCH MAIL_TO_ROOT_EVENT_COUNT=0
NEXT_GATE=READ_EXACT_1231_AND_RECONCILE_CT110_STATE CT110_SUCCESS_EVIDENCE_COUNT=2
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-CT110-TARGETED-VERIFY-1231R1","status":"OK","changes_made":true,"rollback_started":false,"rollback_restored":null} SUCCESS_MAIL_POLICY_ACCEPTED=true
=== AUDIT_MODE_SAFETY ===
AUDIT_SHA256_PRE=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
AUDIT_CONTRACT_RC=0
AUDIT_CONTRACT=v2-plus-sealed-quorum
AUDIT_SCHEMA=homelab-backup-audit-v3
AUDIT_CONTRACT_STDERR_BYTES=0
311:if MODE in ("write-health","scheduled"):
AUDIT_WRITE_GUARD_COUNT=1
HEALTH1_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2
HEALTH2_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2
HEALTH1_MTIME_PRE=1790537775
HEALTH2_MTIME_PRE=1790537775
=== FRESH_OFFICIAL_AUDIT_ALL ===
AUDIT_ALL_RC=1
AUDIT_ALL_STDERR_BYTES=0
AUDIT_OUTPUT_BEGIN
TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48057|generation=app-filebrowser-pve01-20260927T063003Z
TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47890|generation=app-hbarhub-pve01-20260927T063154Z
TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47991|generation=app-immich-pve01-20260927T063041Z
TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48038|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z
TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=STALE|age_seconds=234007|generation=control-bootstrap-v2-pve01-20260916T120057Z
TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=366|legacy_age_seconds=2038301|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17
TARGET|logical_id=vm111|kind=VM|state=STALE|source=NONE|v2_age_seconds=283415|replica_count=2|state_id=ct111
TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=266907|legacy_age_seconds=2076796|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z
TARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=264888|replica_count=2|state_id=ct113
TARGET|logical_id=vm130|kind=VM|state=STALE|source=NONE|v2_age_seconds=259498|replica_count=2|state_id=vm130
TARGET|logical_id=vm150|kind=VM|state=STALE|source=NONE|v2_age_seconds=262950|legacy_age_seconds=2038417|replica_count=2|state_id=vm150|generation=vm150-pve01-20260904T053517Z
TARGET|logical_id=vm160|kind=VM|state=STALE|source=NONE|v2_age_seconds=258251|replica_count=2|state_id=vm160
TARGET|logical_id=vm170|kind=VM|state=STALE|source=NONE|v2_age_seconds=235042|legacy_age_seconds=983038|replica_count=2|state_id=vm170|generation=vm170-pve01-20260916T103019Z
TARGET|logical_id=vm171|kind=VM|state=STALE|source=NONE|v2_age_seconds=239472|legacy_age_seconds=1012570|replica_count=2|state_id=vm171|generation=vm171-pve01-20260916T023018Z
TARGET|logical_id=vm190|kind=VM|state=STALE|source=NONE|v2_age_seconds=235640|replica_count=2|state_id=vm190
TARGET|logical_id=vm9130|kind=VM|state=STALE|source=NONE|v2_age_seconds=226897|replica_count=2|state_id=vm9130
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
AUDIT_OUTPUT_END
AUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
AUDIT_BINARY_UNCHANGED=true
HEALTH1_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2
HEALTH2_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2
HEALTH1_MTIME_POST=1790537775
HEALTH2_MTIME_POST=1790537775
AUDIT_ALL_HEALTH_NO_WRITE=true
FRESH_AUDIT_ISSUE_COUNT=11
FRESH_AUDIT_RPO_BREACH_COUNT=0
FRESH_AUDIT_CT110_RPO_COUNT=0
FRESH_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
=== CURRENT_UNIT_TIMER_STATE ===
UNIT=homelab-backup-v2.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0
UNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID=
UNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0
UNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID=
=== RECENT_SCHEDULER_AFTER_HOTFIX ===
TASK_RESULT=PASS_CT110_STATE_AUDIT_RECONCILE
DECISION=PASS_CT110_STATE_AUDIT_RECONCILE
NEXT_GATE=ADAPTIVE_RPO_RECOVERY_FROM_FRESH_AUDIT
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END OUTPUT_END
CHAT_OUTPUT_END CHAT_OUTPUT_END