From 32e01e035d82c330190483a135e66174f126d75b Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Sun, 27 Sep 2026 19:51:15 +0000 Subject: [PATCH] runtime: publish SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1 --- ...UP-CT110-STATE-AUDIT-RECONCILE-1232R1.json | 38 ++++ ...KUP-CT110-STATE-AUDIT-RECONCILE-1232R1.txt | 158 +++++++++++++ runtime/latest.json | 26 +-- runtime/latest.txt | 212 ++++++++++-------- 4 files changed, 332 insertions(+), 102 deletions(-) create mode 100644 runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1.json create mode 100644 runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1.txt diff --git a/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1.json b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1.json new file mode 100644 index 00000000..05b84965 --- /dev/null +++ b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-ct110-state-audit-reconcile", + "started_at_utc": "2026-09-27T19:51:08Z", + "finished_at_utc": "2026-09-27T19:51:14Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "04f4eb8b36878aa3a82b87657a987381666519b93c3a0218c69e5cf1062cefae", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922", + "sanitized_output_sha256": "610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "BACKUP_CT110_STATE_AUDIT_RECONCILE_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1\nMODE=read-only\nCOMPONENT=homelab-backup-ct110-state-audit-reconcile\nMUTATION_BOUNDARY=READ_ONLY_STATE_JOURNAL_SYSTEMD_AND_AUDIT_ALL;AUDIT_ALL_PROVEN_NO_HEALTH_WRITE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1231_STATUS=OK\nSOURCE_1231_RC=0\nSOURCE_1231_DECISION=HOLD_1231_POST_STATE_MISMATCH\nSOURCE_1231_NEXT_GATE=READ_EXACT_1231_AND_RECONCILE_CT110_STATE\nSOURCE_1231_VERIFY_RESULT_ACCEPTED=true\nSOURCE_1231_VERIFY_RESULT_REPLICAS=2\nSOURCE_1231_ACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17\nSOURCE_1231_VERIFY_START_EPOCH=1790538076\nSOURCE_1231_POST_LAST_SUCCESS_EPOCH=1790538303\n=== LIVE_IDENTITY ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\n=== CT110_STATE_SCHEMA_RECONCILE ===\nSTATE_EXISTS=true\nSTATE_SHA256=ee7999edad0bf4c2988edfdd5059b5e4d33899cd0880aaf2567f647eaeab8033\nSTATE_KEYS=consecutive_failures,failure_backoff_seconds,generation,green_verified,last_attempt_epoch,last_success_epoch,logical_id,manifest_sha256,payload_sha256,replicas,result,retry_after_epoch,schema,updated_at_utc\nSTATE_SCHEMA=1\nSTATE_LOGICAL_ID=ct110\nSTATE_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17\nSTATE_RESULT=SUCCESS\nSTATE_LAST_ATTEMPT_EPOCH=1790538303\nSTATE_LAST_SUCCESS_EPOCH=1790538303\nSTATE_CONSECUTIVE_FAILURES=0\nSTATE_FAILURE_BACKOFF_SECONDS=0\nSTATE_REPLICAS_TYPE=list\nSTATE_REPLICA_COUNT_DERIVED=2\nSTATE_REPLICA_1_SCHEMA_OK=true\nSTATE_REPLICA_1_PHYSICAL_SHA256=c3b0138acb547907e498f9601c9a6cbb320da8e916ed685b30a4e35b92d0a247\nSTATE_REPLICA_1_CRYPT_SHA256=d2d63811693130d9b521ec13bc62c80ff0f7a557f0a1b605db30541858a808f1\nSTATE_REPLICA_2_SCHEMA_OK=true\nSTATE_REPLICA_2_PHYSICAL_SHA256=cf59a41c57e69eaac77ef340b75ff146dd1cfe5da28674e06050730912385bf3\nSTATE_REPLICA_2_CRYPT_SHA256=7218b750518d1a6bd57398eee3cd19e397c587adeda6dfff4cbec1aeb50eb1c8\nSTATE_REPLICA_PHYSICAL_UNIQUE=true\nSTATE_REPLICA_CRYPT_UNIQUE=true\nSTATE_REPLICA_PAIR_UNIQUE=true\nSTATE_CHECK_LOGICAL_ID=true\nSTATE_CHECK_GENERATION=true\nSTATE_CHECK_RESULT=true\nSTATE_CHECK_REPLICAS=true\nSTATE_CHECK_LAST_SUCCESS=true\nSTATE_CHECK_LAST_ATTEMPT=true\nSTATE_ACCEPTED=true\nSTATE_RECONCILE_RC=0\n=== 1231_NOTIFICATION_RUNTIME_RECONCILE ===\nVERIFY_WINDOW_START_EPOCH=1790538076\nVERIFY_WINDOW_END_EPOCH=1790538483\nSUCCESS_BACKUP_EVENT_COUNT=1\nMAIL_TO_ROOT_EVENT_COUNT=0\nCT110_SUCCESS_EVIDENCE_COUNT=2\nSUCCESS_MAIL_POLICY_ACCEPTED=true\n=== AUDIT_MODE_SAFETY ===\nAUDIT_SHA256_PRE=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nAUDIT_CONTRACT_RC=0\nAUDIT_CONTRACT=v2-plus-sealed-quorum\nAUDIT_SCHEMA=homelab-backup-audit-v3\nAUDIT_CONTRACT_STDERR_BYTES=0\n311:if MODE in (\"write-health\",\"scheduled\"):\nAUDIT_WRITE_GUARD_COUNT=1\nHEALTH1_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2\nHEALTH2_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2\nHEALTH1_MTIME_PRE=1790537775\nHEALTH2_MTIME_PRE=1790537775\n=== FRESH_OFFICIAL_AUDIT_ALL ===\nAUDIT_ALL_RC=1\nAUDIT_ALL_STDERR_BYTES=0\nAUDIT_OUTPUT_BEGIN\nTARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48057|generation=app-filebrowser-pve01-20260927T063003Z\nTARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47890|generation=app-hbarhub-pve01-20260927T063154Z\nTARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47991|generation=app-immich-pve01-20260927T063041Z\nTARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48038|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nTARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=STALE|age_seconds=234007|generation=control-bootstrap-v2-pve01-20260916T120057Z\nTARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=366|legacy_age_seconds=2038301|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17\nTARGET|logical_id=vm111|kind=VM|state=STALE|source=NONE|v2_age_seconds=283415|replica_count=2|state_id=ct111\nTARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=266907|legacy_age_seconds=2076796|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z\nTARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=264888|replica_count=2|state_id=ct113\nTARGET|logical_id=vm130|kind=VM|state=STALE|source=NONE|v2_age_seconds=259498|replica_count=2|state_id=vm130\nTARGET|logical_id=vm150|kind=VM|state=STALE|source=NONE|v2_age_seconds=262950|legacy_age_seconds=2038417|replica_count=2|state_id=vm150|generation=vm150-pve01-20260904T053517Z\nTARGET|logical_id=vm160|kind=VM|state=STALE|source=NONE|v2_age_seconds=258251|replica_count=2|state_id=vm160\nTARGET|logical_id=vm170|kind=VM|state=STALE|source=NONE|v2_age_seconds=235042|legacy_age_seconds=983038|replica_count=2|state_id=vm170|generation=vm170-pve01-20260916T103019Z\nTARGET|logical_id=vm171|kind=VM|state=STALE|source=NONE|v2_age_seconds=239472|legacy_age_seconds=1012570|replica_count=2|state_id=vm171|generation=vm171-pve01-20260916T023018Z\nTARGET|logical_id=vm190|kind=VM|state=STALE|source=NONE|v2_age_seconds=235640|replica_count=2|state_id=vm190\nTARGET|logical_id=vm9130|kind=VM|state=STALE|source=NONE|v2_age_seconds=226897|replica_count=2|state_id=vm9130\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nAUDIT_OUTPUT_END\nAUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nAUDIT_BINARY_UNCHANGED=true\nHEALTH1_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2\nHEALTH2_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2\nHEALTH1_MTIME_POST=1790537775\nHEALTH2_MTIME_POST=1790537775\nAUDIT_ALL_HEALTH_NO_WRITE=true\nFRESH_AUDIT_ISSUE_COUNT=11\nFRESH_AUDIT_RPO_BREACH_COUNT=0\nFRESH_AUDIT_CT110_RPO_COUNT=0\nFRESH_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n=== CURRENT_UNIT_TIMER_STATE ===\nUNIT=homelab-backup-v2.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0\nUNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID=\nUNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0\nUNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID=\n=== RECENT_SCHEDULER_AFTER_HOTFIX ===\nTASK_RESULT=PASS_CT110_STATE_AUDIT_RECONCILE\nDECISION=PASS_CT110_STATE_AUDIT_RECONCILE\nNEXT_GATE=ADAPTIVE_RPO_RECOVERY_FROM_FRESH_AUDIT\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1.txt b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1.txt new file mode 100644 index 00000000..6cc548fe --- /dev/null +++ b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1.txt @@ -0,0 +1,158 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-ct110-state-audit-reconcile +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=04f4eb8b36878aa3a82b87657a987381666519b93c3a0218c69e5cf1062cefae +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922 +SANITIZED_OUTPUT_SHA256=610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922 +OUTPUT_BEGIN +BACKUP_CT110_STATE_AUDIT_RECONCILE_BEGIN=true +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1 +MODE=read-only +COMPONENT=homelab-backup-ct110-state-audit-reconcile +MUTATION_BOUNDARY=READ_ONLY_STATE_JOURNAL_SYSTEMD_AND_AUDIT_ALL;AUDIT_ALL_PROVEN_NO_HEALTH_WRITE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE +=== AUTHORITY === +HOST=pve01 +CLUSTER_QUORATE=Yes +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +SOURCE_1231_STATUS=OK +SOURCE_1231_RC=0 +SOURCE_1231_DECISION=HOLD_1231_POST_STATE_MISMATCH +SOURCE_1231_NEXT_GATE=READ_EXACT_1231_AND_RECONCILE_CT110_STATE +SOURCE_1231_VERIFY_RESULT_ACCEPTED=true +SOURCE_1231_VERIFY_RESULT_REPLICAS=2 +SOURCE_1231_ACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17 +SOURCE_1231_VERIFY_START_EPOCH=1790538076 +SOURCE_1231_POST_LAST_SUCCESS_EPOCH=1790538303 +=== LIVE_IDENTITY === +WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 +VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad +=== CT110_STATE_SCHEMA_RECONCILE === +STATE_EXISTS=true +STATE_SHA256=ee7999edad0bf4c2988edfdd5059b5e4d33899cd0880aaf2567f647eaeab8033 +STATE_KEYS=consecutive_failures,failure_backoff_seconds,generation,green_verified,last_attempt_epoch,last_success_epoch,logical_id,manifest_sha256,payload_sha256,replicas,result,retry_after_epoch,schema,updated_at_utc +STATE_SCHEMA=1 +STATE_LOGICAL_ID=ct110 +STATE_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17 +STATE_RESULT=SUCCESS +STATE_LAST_ATTEMPT_EPOCH=1790538303 +STATE_LAST_SUCCESS_EPOCH=1790538303 +STATE_CONSECUTIVE_FAILURES=0 +STATE_FAILURE_BACKOFF_SECONDS=0 +STATE_REPLICAS_TYPE=list +STATE_REPLICA_COUNT_DERIVED=2 +STATE_REPLICA_1_SCHEMA_OK=true +STATE_REPLICA_1_PHYSICAL_SHA256=c3b0138acb547907e498f9601c9a6cbb320da8e916ed685b30a4e35b92d0a247 +STATE_REPLICA_1_CRYPT_SHA256=d2d63811693130d9b521ec13bc62c80ff0f7a557f0a1b605db30541858a808f1 +STATE_REPLICA_2_SCHEMA_OK=true +STATE_REPLICA_2_PHYSICAL_SHA256=cf59a41c57e69eaac77ef340b75ff146dd1cfe5da28674e06050730912385bf3 +STATE_REPLICA_2_CRYPT_SHA256=7218b750518d1a6bd57398eee3cd19e397c587adeda6dfff4cbec1aeb50eb1c8 +STATE_REPLICA_PHYSICAL_UNIQUE=true +STATE_REPLICA_CRYPT_UNIQUE=true +STATE_REPLICA_PAIR_UNIQUE=true +STATE_CHECK_LOGICAL_ID=true +STATE_CHECK_GENERATION=true +STATE_CHECK_RESULT=true +STATE_CHECK_REPLICAS=true +STATE_CHECK_LAST_SUCCESS=true +STATE_CHECK_LAST_ATTEMPT=true +STATE_ACCEPTED=true +STATE_RECONCILE_RC=0 +=== 1231_NOTIFICATION_RUNTIME_RECONCILE === +VERIFY_WINDOW_START_EPOCH=1790538076 +VERIFY_WINDOW_END_EPOCH=1790538483 +SUCCESS_BACKUP_EVENT_COUNT=1 +MAIL_TO_ROOT_EVENT_COUNT=0 +CT110_SUCCESS_EVIDENCE_COUNT=2 +SUCCESS_MAIL_POLICY_ACCEPTED=true +=== AUDIT_MODE_SAFETY === +AUDIT_SHA256_PRE=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +AUDIT_CONTRACT_RC=0 +AUDIT_CONTRACT=v2-plus-sealed-quorum +AUDIT_SCHEMA=homelab-backup-audit-v3 +AUDIT_CONTRACT_STDERR_BYTES=0 +311:if MODE in ("write-health","scheduled"): +AUDIT_WRITE_GUARD_COUNT=1 +HEALTH1_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2 +HEALTH2_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2 +HEALTH1_MTIME_PRE=1790537775 +HEALTH2_MTIME_PRE=1790537775 +=== FRESH_OFFICIAL_AUDIT_ALL === +AUDIT_ALL_RC=1 +AUDIT_ALL_STDERR_BYTES=0 +AUDIT_OUTPUT_BEGIN +TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48057|generation=app-filebrowser-pve01-20260927T063003Z +TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47890|generation=app-hbarhub-pve01-20260927T063154Z +TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47991|generation=app-immich-pve01-20260927T063041Z +TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48038|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=STALE|age_seconds=234007|generation=control-bootstrap-v2-pve01-20260916T120057Z +TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=366|legacy_age_seconds=2038301|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17 +TARGET|logical_id=vm111|kind=VM|state=STALE|source=NONE|v2_age_seconds=283415|replica_count=2|state_id=ct111 +TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=266907|legacy_age_seconds=2076796|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z +TARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=264888|replica_count=2|state_id=ct113 +TARGET|logical_id=vm130|kind=VM|state=STALE|source=NONE|v2_age_seconds=259498|replica_count=2|state_id=vm130 +TARGET|logical_id=vm150|kind=VM|state=STALE|source=NONE|v2_age_seconds=262950|legacy_age_seconds=2038417|replica_count=2|state_id=vm150|generation=vm150-pve01-20260904T053517Z +TARGET|logical_id=vm160|kind=VM|state=STALE|source=NONE|v2_age_seconds=258251|replica_count=2|state_id=vm160 +TARGET|logical_id=vm170|kind=VM|state=STALE|source=NONE|v2_age_seconds=235042|legacy_age_seconds=983038|replica_count=2|state_id=vm170|generation=vm170-pve01-20260916T103019Z +TARGET|logical_id=vm171|kind=VM|state=STALE|source=NONE|v2_age_seconds=239472|legacy_age_seconds=1012570|replica_count=2|state_id=vm171|generation=vm171-pve01-20260916T023018Z +TARGET|logical_id=vm190|kind=VM|state=STALE|source=NONE|v2_age_seconds=235640|replica_count=2|state_id=vm190 +TARGET|logical_id=vm9130|kind=VM|state=STALE|source=NONE|v2_age_seconds=226897|replica_count=2|state_id=vm9130 +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +AUDIT_OUTPUT_END +AUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +AUDIT_BINARY_UNCHANGED=true +HEALTH1_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2 +HEALTH2_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2 +HEALTH1_MTIME_POST=1790537775 +HEALTH2_MTIME_POST=1790537775 +AUDIT_ALL_HEALTH_NO_WRITE=true +FRESH_AUDIT_ISSUE_COUNT=11 +FRESH_AUDIT_RPO_BREACH_COUNT=0 +FRESH_AUDIT_CT110_RPO_COUNT=0 +FRESH_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +=== CURRENT_UNIT_TIMER_STATE === +UNIT=homelab-backup-v2.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0 +UNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID= +UNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0 +UNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID= +=== RECENT_SCHEDULER_AFTER_HOTFIX === +TASK_RESULT=PASS_CT110_STATE_AUDIT_RECONCILE +DECISION=PASS_CT110_STATE_AUDIT_RECONCILE +NEXT_GATE=ADAPTIVE_RPO_RECOVERY_FROM_FRESH_AUDIT +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 60f08035..05b84965 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260927-HOMELAB-BACKUP-CT110-TARGETED-VERIFY-1231R1", + "command_id": "SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1", "status": "OK", "rc": 0, "host": "pve01", - "mode": "change", - "component": "homelab-backup-ct110-targeted-verify", - "started_at_utc": "2026-09-27T19:41:10Z", - "finished_at_utc": "2026-09-27T19:45:03Z", + "mode": "read-only", + "component": "homelab-backup-ct110-state-audit-reconcile", + "started_at_utc": "2026-09-27T19:51:08Z", + "finished_at_utc": "2026-09-27T19:51:14Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "5a48244cd686dffbe11630b2c2fbced81680daec26b2e3a8acaeaca1e83b4cb2", + "command_sha256": "04f4eb8b36878aa3a82b87657a987381666519b93c3a0218c69e5cf1062cefae", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, + "change_declared": false, "result_contract_valid": true, - "result_contract_status": "OK", + "result_contract_status": null, "result_contract_error": null, "command_rc": 0, - "changes_made": true, + "changes_made": false, "rollback_started": false, "rollback_restored": null, - "mutation_outcome": "MUTATED", + "mutation_outcome": "NO_MUTATION", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "8330976ae073c2483c486fdd01691dfbf2a8d74f1d1d774338bac2fca89f7705", - "sanitized_output_sha256": "8330976ae073c2483c486fdd01691dfbf2a8d74f1d1d774338bac2fca89f7705", + "raw_evidence_sha256": "610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922", + "sanitized_output_sha256": "610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "BACKUP_CT110_TARGETED_VERIFY_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-TARGETED-VERIFY-1231R1\nMODE=change\nCOMPONENT=homelab-backup-ct110-targeted-verify\nMUTATION_BOUNDARY=ONE_CURRENT_V2_VERIFY_TARGET_CT110_ONLY;MAY_CREATE_NEW_CT110_BACKUP_GENERATION_AND_UPDATE_CT110_STATE;NO_OTHER_TARGET_REFRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_AUDIT_RUN;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1230_STATUS=OK\nSOURCE_1230_RC=0\nSOURCE_1230_DECISION=PASS_BACKUP_NOTIFICATION_COMPAT_HOTFIX_DEPLOYED\nSOURCE_1230_NEXT_GATE=TARGETED_CT110_VERIFY_AND_ADAPTIVE_RPO_RECOVERY\n=== LIVE_IDENTITY ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nRELEASE_META=500:root:root\nHELPER_META=400:root:root\nSOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_WORKTREE_CLEAN=true\n=== OWNER_WAIT ===\nOWNER_WAIT_ITERATION=1 ACTIVE=failed SUB=failed PID=0\nOWNER_WAIT_ITERATION=2 ACTIVE=failed SUB=failed PID=0\nBACKUP_HEAVY_LOCK_ACQUIRED=true\nPOSTLOCK_SERVICE_ACTIVE=failed\nPOSTLOCK_SERVICE_MAINPID=0\nACTIVE_VZDUMP_COUNT=0\n=== PRE_STATE ===\nPRE_STATE_EXISTS=true\nPRE_STATE_SHA256=b3595df8d1890c26b0a93dc5e38722294b617d98431bb2693d7b05a5356ae4e8\nPRE_STATE_JSON_OK=true\nPRE_LOGICAL_ID=ct110\nPRE_GENERATION=v2-ct110-20260924T123032Z-e893686ccca9\nPRE_RESULT=SUCCESS\nPRE_REPLICA_COUNT=None\nPRE_LAST_ATTEMPT_EPOCH=1790253235\nPRE_LAST_SUCCESS_EPOCH=1790253235\nPRE_FAILURE_BACKOFF_SECONDS=0\nPRE_CONSECUTIVE_FAILURES=0\nVERIFY_START_EPOCH=1790538076\nVERIFY_START_UTC=2026-09-27T19:41:16Z\n=== TARGETED_VERIFY_CT110 ===\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=561\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=9c399e7d401ea4dfdcd9f610e024d5728e97b27bccfb018cd9bf12cfb2d934f8\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN\nSELFTEST_REPLACEMENT486=PASS\nLXC_EXECUTION_DIAG={\"archive_bytes\":566926006,\"exception_class\":null,\"exception_message_sha256\":null,\"executor\":\"local-proven-pve270\",\"helper_rc\":0,\"node\":\"pve01\",\"producer_rc\":0,\"stderr_bytes_observed\":1364,\"stderr_bytes_retained\":1364,\"stderr_stream_sha256\":\"7e6320861d4dbb5b6ecba9fb36e611b408b8e36628004a0d266845f8f93b513a\",\"stderr_truncated\":false}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-ct110-20260927T194116Z-d16fe26dbd17\",\"logical_id\":\"ct110\",\"payload_bytes\":566926006,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=ct110\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=566926006\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17\nACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17\n=== POST_STATE ===\nPOST_STATE_EXISTS=true\nPOST_STATE_SHA256=ee7999edad0bf4c2988edfdd5059b5e4d33899cd0880aaf2567f647eaeab8033\nPOST_STATE_JSON_OK=true\nPOST_LOGICAL_ID=ct110\nPOST_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17\nPOST_RESULT=SUCCESS\nPOST_REPLICA_COUNT=None\nPOST_LAST_ATTEMPT_EPOCH=1790538303\nPOST_LAST_SUCCESS_EPOCH=1790538303\nPOST_FAILURE_BACKOFF_SECONDS=0\nPOST_CONSECUTIVE_FAILURES=0\nSTATE_CHECK_LOGICAL_ID=true\nSTATE_CHECK_GENERATION=true\nSTATE_CHECK_RESULT=true\nSTATE_CHECK_REPLICA_COUNT=false\nSTATE_CHECK_LAST_SUCCESS=true\nSTATE_CHECK_LAST_ATTEMPT=true\nSTATE_ACCEPTED=false\nCHANGES_MADE=true\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=MUTATED_OR_ATTEMPTED_BACKUP\nTASK_RESULT=HOLD_CT110_TARGETED_VERIFY_AFTER_ATTEMPT\nDECISION=HOLD_1231_POST_STATE_MISMATCH\nNEXT_GATE=READ_EXACT_1231_AND_RECONCILE_CT110_STATE\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-HOMELAB-BACKUP-CT110-TARGETED-VERIFY-1231R1\",\"status\":\"OK\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "BACKUP_CT110_STATE_AUDIT_RECONCILE_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1\nMODE=read-only\nCOMPONENT=homelab-backup-ct110-state-audit-reconcile\nMUTATION_BOUNDARY=READ_ONLY_STATE_JOURNAL_SYSTEMD_AND_AUDIT_ALL;AUDIT_ALL_PROVEN_NO_HEALTH_WRITE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1231_STATUS=OK\nSOURCE_1231_RC=0\nSOURCE_1231_DECISION=HOLD_1231_POST_STATE_MISMATCH\nSOURCE_1231_NEXT_GATE=READ_EXACT_1231_AND_RECONCILE_CT110_STATE\nSOURCE_1231_VERIFY_RESULT_ACCEPTED=true\nSOURCE_1231_VERIFY_RESULT_REPLICAS=2\nSOURCE_1231_ACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17\nSOURCE_1231_VERIFY_START_EPOCH=1790538076\nSOURCE_1231_POST_LAST_SUCCESS_EPOCH=1790538303\n=== LIVE_IDENTITY ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\n=== CT110_STATE_SCHEMA_RECONCILE ===\nSTATE_EXISTS=true\nSTATE_SHA256=ee7999edad0bf4c2988edfdd5059b5e4d33899cd0880aaf2567f647eaeab8033\nSTATE_KEYS=consecutive_failures,failure_backoff_seconds,generation,green_verified,last_attempt_epoch,last_success_epoch,logical_id,manifest_sha256,payload_sha256,replicas,result,retry_after_epoch,schema,updated_at_utc\nSTATE_SCHEMA=1\nSTATE_LOGICAL_ID=ct110\nSTATE_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17\nSTATE_RESULT=SUCCESS\nSTATE_LAST_ATTEMPT_EPOCH=1790538303\nSTATE_LAST_SUCCESS_EPOCH=1790538303\nSTATE_CONSECUTIVE_FAILURES=0\nSTATE_FAILURE_BACKOFF_SECONDS=0\nSTATE_REPLICAS_TYPE=list\nSTATE_REPLICA_COUNT_DERIVED=2\nSTATE_REPLICA_1_SCHEMA_OK=true\nSTATE_REPLICA_1_PHYSICAL_SHA256=c3b0138acb547907e498f9601c9a6cbb320da8e916ed685b30a4e35b92d0a247\nSTATE_REPLICA_1_CRYPT_SHA256=d2d63811693130d9b521ec13bc62c80ff0f7a557f0a1b605db30541858a808f1\nSTATE_REPLICA_2_SCHEMA_OK=true\nSTATE_REPLICA_2_PHYSICAL_SHA256=cf59a41c57e69eaac77ef340b75ff146dd1cfe5da28674e06050730912385bf3\nSTATE_REPLICA_2_CRYPT_SHA256=7218b750518d1a6bd57398eee3cd19e397c587adeda6dfff4cbec1aeb50eb1c8\nSTATE_REPLICA_PHYSICAL_UNIQUE=true\nSTATE_REPLICA_CRYPT_UNIQUE=true\nSTATE_REPLICA_PAIR_UNIQUE=true\nSTATE_CHECK_LOGICAL_ID=true\nSTATE_CHECK_GENERATION=true\nSTATE_CHECK_RESULT=true\nSTATE_CHECK_REPLICAS=true\nSTATE_CHECK_LAST_SUCCESS=true\nSTATE_CHECK_LAST_ATTEMPT=true\nSTATE_ACCEPTED=true\nSTATE_RECONCILE_RC=0\n=== 1231_NOTIFICATION_RUNTIME_RECONCILE ===\nVERIFY_WINDOW_START_EPOCH=1790538076\nVERIFY_WINDOW_END_EPOCH=1790538483\nSUCCESS_BACKUP_EVENT_COUNT=1\nMAIL_TO_ROOT_EVENT_COUNT=0\nCT110_SUCCESS_EVIDENCE_COUNT=2\nSUCCESS_MAIL_POLICY_ACCEPTED=true\n=== AUDIT_MODE_SAFETY ===\nAUDIT_SHA256_PRE=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nAUDIT_CONTRACT_RC=0\nAUDIT_CONTRACT=v2-plus-sealed-quorum\nAUDIT_SCHEMA=homelab-backup-audit-v3\nAUDIT_CONTRACT_STDERR_BYTES=0\n311:if MODE in (\"write-health\",\"scheduled\"):\nAUDIT_WRITE_GUARD_COUNT=1\nHEALTH1_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2\nHEALTH2_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2\nHEALTH1_MTIME_PRE=1790537775\nHEALTH2_MTIME_PRE=1790537775\n=== FRESH_OFFICIAL_AUDIT_ALL ===\nAUDIT_ALL_RC=1\nAUDIT_ALL_STDERR_BYTES=0\nAUDIT_OUTPUT_BEGIN\nTARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48057|generation=app-filebrowser-pve01-20260927T063003Z\nTARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47890|generation=app-hbarhub-pve01-20260927T063154Z\nTARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47991|generation=app-immich-pve01-20260927T063041Z\nTARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48038|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nTARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=STALE|age_seconds=234007|generation=control-bootstrap-v2-pve01-20260916T120057Z\nTARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=366|legacy_age_seconds=2038301|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17\nTARGET|logical_id=vm111|kind=VM|state=STALE|source=NONE|v2_age_seconds=283415|replica_count=2|state_id=ct111\nTARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=266907|legacy_age_seconds=2076796|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z\nTARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=264888|replica_count=2|state_id=ct113\nTARGET|logical_id=vm130|kind=VM|state=STALE|source=NONE|v2_age_seconds=259498|replica_count=2|state_id=vm130\nTARGET|logical_id=vm150|kind=VM|state=STALE|source=NONE|v2_age_seconds=262950|legacy_age_seconds=2038417|replica_count=2|state_id=vm150|generation=vm150-pve01-20260904T053517Z\nTARGET|logical_id=vm160|kind=VM|state=STALE|source=NONE|v2_age_seconds=258251|replica_count=2|state_id=vm160\nTARGET|logical_id=vm170|kind=VM|state=STALE|source=NONE|v2_age_seconds=235042|legacy_age_seconds=983038|replica_count=2|state_id=vm170|generation=vm170-pve01-20260916T103019Z\nTARGET|logical_id=vm171|kind=VM|state=STALE|source=NONE|v2_age_seconds=239472|legacy_age_seconds=1012570|replica_count=2|state_id=vm171|generation=vm171-pve01-20260916T023018Z\nTARGET|logical_id=vm190|kind=VM|state=STALE|source=NONE|v2_age_seconds=235640|replica_count=2|state_id=vm190\nTARGET|logical_id=vm9130|kind=VM|state=STALE|source=NONE|v2_age_seconds=226897|replica_count=2|state_id=vm9130\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nAUDIT_OUTPUT_END\nAUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nAUDIT_BINARY_UNCHANGED=true\nHEALTH1_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2\nHEALTH2_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2\nHEALTH1_MTIME_POST=1790537775\nHEALTH2_MTIME_POST=1790537775\nAUDIT_ALL_HEALTH_NO_WRITE=true\nFRESH_AUDIT_ISSUE_COUNT=11\nFRESH_AUDIT_RPO_BREACH_COUNT=0\nFRESH_AUDIT_CT110_RPO_COUNT=0\nFRESH_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n=== CURRENT_UNIT_TIMER_STATE ===\nUNIT=homelab-backup-v2.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0\nUNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID=\nUNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0\nUNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID=\n=== RECENT_SCHEDULER_AFTER_HOTFIX ===\nTASK_RESULT=PASS_CT110_STATE_AUDIT_RECONCILE\nDECISION=PASS_CT110_STATE_AUDIT_RECONCILE\nNEXT_GATE=ADAPTIVE_RPO_RECOVERY_FROM_FRESH_AUDIT\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index e090ecfb..6cc548fe 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,124 +1,158 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-TARGETED-VERIFY-1231R1 +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1 STATUS=OK RC=0 HOST=pve01 -MODE=change -COMPONENT=homelab-backup-ct110-targeted-verify +MODE=read-only +COMPONENT=homelab-backup-ct110-state-audit-reconcile REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=5a48244cd686dffbe11630b2c2fbced81680daec26b2e3a8acaeaca1e83b4cb2 +COMMAND_SHA256=04f4eb8b36878aa3a82b87657a987381666519b93c3a0218c69e5cf1062cefae DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true +CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=OK +RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE COMMAND_RC=0 -CHANGES_MADE=true +CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=MUTATED +MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=8330976ae073c2483c486fdd01691dfbf2a8d74f1d1d774338bac2fca89f7705 -SANITIZED_OUTPUT_SHA256=8330976ae073c2483c486fdd01691dfbf2a8d74f1d1d774338bac2fca89f7705 +RAW_EVIDENCE_SHA256=610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922 +SANITIZED_OUTPUT_SHA256=610fd3fad77f1c61c0a7b88bbc6a35a398461ed28081a2a3e990b23173163922 OUTPUT_BEGIN -BACKUP_CT110_TARGETED_VERIFY_BEGIN=true -COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-TARGETED-VERIFY-1231R1 -MODE=change -COMPONENT=homelab-backup-ct110-targeted-verify -MUTATION_BOUNDARY=ONE_CURRENT_V2_VERIFY_TARGET_CT110_ONLY;MAY_CREATE_NEW_CT110_BACKUP_GENERATION_AND_UPDATE_CT110_STATE;NO_OTHER_TARGET_REFRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_AUDIT_RUN;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE +BACKUP_CT110_STATE_AUDIT_RECONCILE_BEGIN=true +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1 +MODE=read-only +COMPONENT=homelab-backup-ct110-state-audit-reconcile +MUTATION_BOUNDARY=READ_ONLY_STATE_JOURNAL_SYSTEMD_AND_AUDIT_ALL;AUDIT_ALL_PROVEN_NO_HEALTH_WRITE;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE === AUTHORITY === HOST=pve01 CLUSTER_QUORATE=Yes RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -SOURCE_1230_STATUS=OK -SOURCE_1230_RC=0 -SOURCE_1230_DECISION=PASS_BACKUP_NOTIFICATION_COMPAT_HOTFIX_DEPLOYED -SOURCE_1230_NEXT_GATE=TARGETED_CT110_VERIFY_AND_ADAPTIVE_RPO_RECOVERY +SOURCE_1231_STATUS=OK +SOURCE_1231_RC=0 +SOURCE_1231_DECISION=HOLD_1231_POST_STATE_MISMATCH +SOURCE_1231_NEXT_GATE=READ_EXACT_1231_AND_RECONCILE_CT110_STATE +SOURCE_1231_VERIFY_RESULT_ACCEPTED=true +SOURCE_1231_VERIFY_RESULT_REPLICAS=2 +SOURCE_1231_ACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17 +SOURCE_1231_VERIFY_START_EPOCH=1790538076 +SOURCE_1231_POST_LAST_SUCCESS_EPOCH=1790538303 === LIVE_IDENTITY === WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad -RELEASE_META=500:root:root -HELPER_META=400:root:root -SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 -SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation -SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 -SOURCE_WORKTREE_CLEAN=true -=== OWNER_WAIT === -OWNER_WAIT_ITERATION=1 ACTIVE=failed SUB=failed PID=0 -OWNER_WAIT_ITERATION=2 ACTIVE=failed SUB=failed PID=0 -BACKUP_HEAVY_LOCK_ACQUIRED=true -POSTLOCK_SERVICE_ACTIVE=failed -POSTLOCK_SERVICE_MAINPID=0 -ACTIVE_VZDUMP_COUNT=0 -=== PRE_STATE === -PRE_STATE_EXISTS=true -PRE_STATE_SHA256=b3595df8d1890c26b0a93dc5e38722294b617d98431bb2693d7b05a5356ae4e8 -PRE_STATE_JSON_OK=true -PRE_LOGICAL_ID=ct110 -PRE_GENERATION=v2-ct110-20260924T123032Z-e893686ccca9 -PRE_RESULT=SUCCESS -PRE_REPLICA_COUNT=None -PRE_LAST_ATTEMPT_EPOCH=1790253235 -PRE_LAST_SUCCESS_EPOCH=1790253235 -PRE_FAILURE_BACKOFF_SECONDS=0 -PRE_CONSECUTIVE_FAILURES=0 -VERIFY_START_EPOCH=1790538076 -VERIFY_START_UTC=2026-09-27T19:41:16Z -=== TARGETED_VERIFY_CT110 === -VERIFY_TARGET_RC=0 -VERIFY_STDOUT_BYTES=561 -VERIFY_STDERR_BYTES=0 -VERIFY_STDOUT_SHA256=9c399e7d401ea4dfdcd9f610e024d5728e97b27bccfb018cd9bf12cfb2d934f8 -VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -VERIFY_OUTPUT_BEGIN -SELFTEST_REPLACEMENT486=PASS -LXC_EXECUTION_DIAG={"archive_bytes":566926006,"exception_class":null,"exception_message_sha256":null,"executor":"local-proven-pve270","helper_rc":0,"node":"pve01","producer_rc":0,"stderr_bytes_observed":1364,"stderr_bytes_retained":1364,"stderr_stream_sha256":"7e6320861d4dbb5b6ecba9fb36e611b408b8e36628004a0d266845f8f93b513a","stderr_truncated":false} -VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct110-20260927T194116Z-d16fe26dbd17","logical_id":"ct110","payload_bytes":566926006,"replicas":2,"result":"SUCCESS"} -VERIFY_OUTPUT_END -VERIFY_RESULT_LINE_COUNT=1 -VERIFY_RESULT_LOGICAL_ID=ct110 -VERIFY_RESULT_RESULT=SUCCESS -VERIFY_RESULT_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17 -VERIFY_RESULT_REPLICAS=2 -VERIFY_RESULT_PAYLOAD_BYTES=566926006 -VERIFY_RESULT_ALLOCATOR_HEALTHY=13 -VERIFY_RESULT_ACCEPTED=true -ACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17 -ACCEPTED_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17 -=== POST_STATE === -POST_STATE_EXISTS=true -POST_STATE_SHA256=ee7999edad0bf4c2988edfdd5059b5e4d33899cd0880aaf2567f647eaeab8033 -POST_STATE_JSON_OK=true -POST_LOGICAL_ID=ct110 -POST_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17 -POST_RESULT=SUCCESS -POST_REPLICA_COUNT=None -POST_LAST_ATTEMPT_EPOCH=1790538303 -POST_LAST_SUCCESS_EPOCH=1790538303 -POST_FAILURE_BACKOFF_SECONDS=0 -POST_CONSECUTIVE_FAILURES=0 +=== CT110_STATE_SCHEMA_RECONCILE === +STATE_EXISTS=true +STATE_SHA256=ee7999edad0bf4c2988edfdd5059b5e4d33899cd0880aaf2567f647eaeab8033 +STATE_KEYS=consecutive_failures,failure_backoff_seconds,generation,green_verified,last_attempt_epoch,last_success_epoch,logical_id,manifest_sha256,payload_sha256,replicas,result,retry_after_epoch,schema,updated_at_utc +STATE_SCHEMA=1 +STATE_LOGICAL_ID=ct110 +STATE_GENERATION=v2-ct110-20260927T194116Z-d16fe26dbd17 +STATE_RESULT=SUCCESS +STATE_LAST_ATTEMPT_EPOCH=1790538303 +STATE_LAST_SUCCESS_EPOCH=1790538303 +STATE_CONSECUTIVE_FAILURES=0 +STATE_FAILURE_BACKOFF_SECONDS=0 +STATE_REPLICAS_TYPE=list +STATE_REPLICA_COUNT_DERIVED=2 +STATE_REPLICA_1_SCHEMA_OK=true +STATE_REPLICA_1_PHYSICAL_SHA256=c3b0138acb547907e498f9601c9a6cbb320da8e916ed685b30a4e35b92d0a247 +STATE_REPLICA_1_CRYPT_SHA256=d2d63811693130d9b521ec13bc62c80ff0f7a557f0a1b605db30541858a808f1 +STATE_REPLICA_2_SCHEMA_OK=true +STATE_REPLICA_2_PHYSICAL_SHA256=cf59a41c57e69eaac77ef340b75ff146dd1cfe5da28674e06050730912385bf3 +STATE_REPLICA_2_CRYPT_SHA256=7218b750518d1a6bd57398eee3cd19e397c587adeda6dfff4cbec1aeb50eb1c8 +STATE_REPLICA_PHYSICAL_UNIQUE=true +STATE_REPLICA_CRYPT_UNIQUE=true +STATE_REPLICA_PAIR_UNIQUE=true STATE_CHECK_LOGICAL_ID=true STATE_CHECK_GENERATION=true STATE_CHECK_RESULT=true -STATE_CHECK_REPLICA_COUNT=false +STATE_CHECK_REPLICAS=true STATE_CHECK_LAST_SUCCESS=true STATE_CHECK_LAST_ATTEMPT=true -STATE_ACCEPTED=false -CHANGES_MADE=true -ROLLBACK_STARTED=false -ROLLBACK_RESTORED=null -MUTATION_OUTCOME=MUTATED_OR_ATTEMPTED_BACKUP -TASK_RESULT=HOLD_CT110_TARGETED_VERIFY_AFTER_ATTEMPT -DECISION=HOLD_1231_POST_STATE_MISMATCH -NEXT_GATE=READ_EXACT_1231_AND_RECONCILE_CT110_STATE -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-CT110-TARGETED-VERIFY-1231R1","status":"OK","changes_made":true,"rollback_started":false,"rollback_restored":null} +STATE_ACCEPTED=true +STATE_RECONCILE_RC=0 +=== 1231_NOTIFICATION_RUNTIME_RECONCILE === +VERIFY_WINDOW_START_EPOCH=1790538076 +VERIFY_WINDOW_END_EPOCH=1790538483 +SUCCESS_BACKUP_EVENT_COUNT=1 +MAIL_TO_ROOT_EVENT_COUNT=0 +CT110_SUCCESS_EVIDENCE_COUNT=2 +SUCCESS_MAIL_POLICY_ACCEPTED=true +=== AUDIT_MODE_SAFETY === +AUDIT_SHA256_PRE=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +AUDIT_CONTRACT_RC=0 +AUDIT_CONTRACT=v2-plus-sealed-quorum +AUDIT_SCHEMA=homelab-backup-audit-v3 +AUDIT_CONTRACT_STDERR_BYTES=0 +311:if MODE in ("write-health","scheduled"): +AUDIT_WRITE_GUARD_COUNT=1 +HEALTH1_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2 +HEALTH2_SHA_PRE=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2 +HEALTH1_MTIME_PRE=1790537775 +HEALTH2_MTIME_PRE=1790537775 +=== FRESH_OFFICIAL_AUDIT_ALL === +AUDIT_ALL_RC=1 +AUDIT_ALL_STDERR_BYTES=0 +AUDIT_OUTPUT_BEGIN +TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48057|generation=app-filebrowser-pve01-20260927T063003Z +TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47890|generation=app-hbarhub-pve01-20260927T063154Z +TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=47991|generation=app-immich-pve01-20260927T063041Z +TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=48038|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=STALE|age_seconds=234007|generation=control-bootstrap-v2-pve01-20260916T120057Z +TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=366|legacy_age_seconds=2038301|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17 +TARGET|logical_id=vm111|kind=VM|state=STALE|source=NONE|v2_age_seconds=283415|replica_count=2|state_id=ct111 +TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=266907|legacy_age_seconds=2076796|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z +TARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=264888|replica_count=2|state_id=ct113 +TARGET|logical_id=vm130|kind=VM|state=STALE|source=NONE|v2_age_seconds=259498|replica_count=2|state_id=vm130 +TARGET|logical_id=vm150|kind=VM|state=STALE|source=NONE|v2_age_seconds=262950|legacy_age_seconds=2038417|replica_count=2|state_id=vm150|generation=vm150-pve01-20260904T053517Z +TARGET|logical_id=vm160|kind=VM|state=STALE|source=NONE|v2_age_seconds=258251|replica_count=2|state_id=vm160 +TARGET|logical_id=vm170|kind=VM|state=STALE|source=NONE|v2_age_seconds=235042|legacy_age_seconds=983038|replica_count=2|state_id=vm170|generation=vm170-pve01-20260916T103019Z +TARGET|logical_id=vm171|kind=VM|state=STALE|source=NONE|v2_age_seconds=239472|legacy_age_seconds=1012570|replica_count=2|state_id=vm171|generation=vm171-pve01-20260916T023018Z +TARGET|logical_id=vm190|kind=VM|state=STALE|source=NONE|v2_age_seconds=235640|replica_count=2|state_id=vm190 +TARGET|logical_id=vm9130|kind=VM|state=STALE|source=NONE|v2_age_seconds=226897|replica_count=2|state_id=vm9130 +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=control-bootstrap-v2|root_cause=CONTROL_STATE_NOT_FRESH +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +AUDIT_OUTPUT_END +AUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +AUDIT_BINARY_UNCHANGED=true +HEALTH1_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2 +HEALTH2_SHA_POST=4c415ddb1cb675155be45d84d257f86fa45ac4aea65764e0a810a0f9a85de2b2 +HEALTH1_MTIME_POST=1790537775 +HEALTH2_MTIME_POST=1790537775 +AUDIT_ALL_HEALTH_NO_WRITE=true +FRESH_AUDIT_ISSUE_COUNT=11 +FRESH_AUDIT_RPO_BREACH_COUNT=0 +FRESH_AUDIT_CT110_RPO_COUNT=0 +FRESH_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +=== CURRENT_UNIT_TIMER_STATE === +UNIT=homelab-backup-v2.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0 +UNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID= +UNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code MAINPID=0 +UNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success MAINPID= +=== RECENT_SCHEDULER_AFTER_HOTFIX === +TASK_RESULT=PASS_CT110_STATE_AUDIT_RECONCILE +DECISION=PASS_CT110_STATE_AUDIT_RECONCILE +NEXT_GATE=ADAPTIVE_RPO_RECOVERY_FROM_FRESH_AUDIT +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-CT110-STATE-AUDIT-RECONCILE-1232R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END