diff --git a/runtime/history/SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1.json b/runtime/history/SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1.json new file mode 100644 index 00000000..75416620 --- /dev/null +++ b/runtime/history/SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "pve01-failed-units-recovery-rca", + "started_at_utc": "2026-10-06T05:00:17Z", + "finished_at_utc": "2026-10-06T05:00:20Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "9d9e9b613b9c82cf1a4117d91d95d83b00dfacba793118fdf72e1a046304d055", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "6ac9ff0e3cb90e523d09b3a848a7d0d45af85de8cfc3e50079599ed55add7df6", + "sanitized_output_sha256": "6ac9ff0e3cb90e523d09b3a848a7d0d45af85de8cfc3e50079599ed55add7df6", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "PVE01_FAILED_UNITS_RECOVERY_RCA=V1\nMODE=read-only\nNO_REBOOT=true\nNO_APT_UPDATE=true\nNO_PACKAGE_MUTATION=true\nNO_SERVICE_START_STOP_RESTART=true\nNO_RESET_FAILED=true\nNO_CONFIG_WRITE=true\nNO_FILE_DELETE=true\nNO_GUEST_MUTATION=true\nNO_BACKUP_EXECUTION=true\nNO_NETWORK_MUTATION=true\nPOLICY_REPAIR_EPOCH=1791262278\nNODE_HOSTNAME=pve01\nNODE_KERNEL=7.0.12-1-pve\nSYSTEMD_SYSTEM_STATE=degraded\nFAILED_UNIT_COUNT=4\nREC|FAILED_UNIT_STATE|homelab-backup-audit.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791261418|EXIT_EPOCH=1791261422|EXIT_AFTER_99670=false\nREC|FAILED_UNIT_FRAGMENT|homelab-backup-audit.service|BASENAME=homelab-backup-audit.service|SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa\nREC|FAILED_UNIT_EXECSTART|homelab-backup-audit.service|BYTES=244|SHA256=5a28904328b20e6dbeb2f818ff18ec6005c7dbf6ed24700b96f5bd528405cd47\nREC|FAILED_UNIT_EXEC_FILE|homelab-backup-audit.service|IDX=1|BASENAME=homelab-backup-audit|BYTES=19756|SHA256=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3\nREC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-backup-audit.service|1\nREC|FAILED_UNIT_JOURNAL|homelab-backup-audit.service|LINES=180|SHA256=9bd6eea7ef8c09f5562cd04df593419e77c0732f88a14fc88b15c7c5a3164421\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|PVE_GUEST_NOT_UNIQUE|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|RPO_BREACH|27\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|BACKUP_STATUS_RED|7\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|BACKUP_STATUS_OK|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|COULD_NOT_RESOLVE|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|NAME_OR_SERVICE_NOT_KNOWN|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|CONNECTION_REFUSED|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|NO_ROUTE_TO_HOST|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TIMED_OUT|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TIMEOUT|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|FAILED|14\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|ERROR|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|STALE|27\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|1|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=14548|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|2|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm110|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|3|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|4|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|5|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|6|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|7|HOST_a0dac0c74d36: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|8|HOST_a0dac0c74d36: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|9|Failed to start HOST_a0dac0c74d36 - Homelab unified backup and DR audit.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|10|HOST_a0dac0c74d36: Consumed 2.707s CPU time, 147.5M memory peak.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|11|Starting HOST_a0dac0c74d36 - Homelab unified backup and DR audit...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|12|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=401|generation=app-filebrowser-pve01-20261006T043010Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|13|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=239|generation=app-hbarhub-pve01-20261006T043148Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|14|TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=345|generation=app-immich-pve01-20261006T043031Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|15|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=29207|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|16|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=42881|generation=control-bootstrap-v2-pve01-20260916T120057Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|17|TARGET|logical_id=vm110|kind=VM|state=STALE|source=NONE|v2_age_seconds=83946|legacy_age_seconds=2761050|replica_count=2|state_id=ct110|generation=vm110-pve01-20260904T053821Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|18|TARGET|logical_id=vm111|kind=VM|state=STALE|source=NONE|v2_age_seconds=82037|replica_count=2|state_id=ct111\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|19|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=67812|legacy_age_seconds=2799545|replica_count=2|state_id=ct112|generation=v2-ct112-20261005T094532Z-ee250335d44e\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|20|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=53076|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|21|TARGET|logical_id=vm130|kind=VM|state=STALE|source=NONE|v2_age_seconds=87489|replica_count=2|state_id=vm130\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|22|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=50972|legacy_age_seconds=69392|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|23|TARGET|logical_id=vm160|kind=VM|state=STALE|source=NONE|v2_age_seconds=94473|replica_count=2|state_id=vm160\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|24|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=37543|legacy_age_seconds=1705787|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|25|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=33778|legacy_age_seconds=1735319|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|26|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=48781|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|27|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=16454|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|28|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm110|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|29|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|30|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|31|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|32|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|33|HOST_a0dac0c74d36: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|34|HOST_a0dac0c74d36: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|35|Failed to start HOST_a0dac0c74d36 - Homelab unified backup and DR audit.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|36|HOST_a0dac0c74d36: Consumed 2.761s CPU time, 148.4M memory peak.\nREC|FAILED_UNIT_STATE|homelab-backup-v2.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=2|START_EPOCH=1791262452|EXIT_EPOCH=1791262453|EXIT_AFTER_99670=true\nREC|FAILED_UNIT_FRAGMENT|homelab-backup-v2.service|BASENAME=homelab-backup-v2.service|SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565\nREC|FAILED_UNIT_EXECSTART|homelab-backup-v2.service|BYTES=238|SHA256=9560de727725d1afd2a98e905f0aad5c2b1fbb1b3348ea1fdc4aa13e5038bdae\nREC|FAILED_UNIT_EXEC_FILE|homelab-backup-v2.service|IDX=1|BASENAME=homelab-backup-v2|BYTES=1656|SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nREC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-backup-v2.service|1\nREC|FAILED_UNIT_JOURNAL|homelab-backup-v2.service|LINES=180|SHA256=d9f31b02b241f88591dbad318d411219659eff0bbb7feb44dd913e38461a507e\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|PVE_GUEST_NOT_UNIQUE|8\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|RPO_BREACH|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|BACKUP_STATUS_RED|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|BACKUP_STATUS_OK|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|COULD_NOT_RESOLVE|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|NAME_OR_SERVICE_NOT_KNOWN|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|CONNECTION_REFUSED|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|NO_ROUTE_TO_HOST|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TIMED_OUT|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TIMEOUT|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|FAILED|14\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|ERROR|8\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|STALE|0\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|1| ~~~~~~~~~~~~~~^^^^^^^^^^^^\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|2| File \"PATH_REDACTED, line 873, in capture_target\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|3| if typ=='pve_guest': return capture_pve_guest(t,gen,work)\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|4| ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|5| File \"PATH_REDACTED, line 192, in capture_pve_guest\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|6| if HOST_4c1ea9873d5f('guest_type')!='lxc': return original(t,gen,work)\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|7| ~~~~~~~~^^^^^^^^^^^^\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|8| File \"PATH_REDACTED, line 44, in capture_pve_guest\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|9| _need(len(rows)==1,'PVE_GUEST_NOT_UNIQUE')\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|10| ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|11| File \"PATH_REDACTED, line 9, in _need\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|12| if not ok: raise QemuAdapterError(code)\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|13| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|14|qemu_adapter.QemuAdapterError: PVE_GUEST_NOT_UNIQUE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|15|HOST_cf2099570f34: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|16|HOST_cf2099570f34: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|17|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|18|HOST_cf2099570f34: Consumed 665ms CPU time, 169.5M memory peak.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|19|Starting HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|20|SELFTEST_REPLACEMENT486=PASS\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|21|SCHEDULE_TARGET=guest-pve01-lxc-112\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|22|SCHEDULE_RESULT=FAIL\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|23|SCHEDULE_REASON=CT112_PIPELINE_FAILED\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|24|HOST_cf2099570f34: Main process exited, code=exited, status=2/INVALIDARGUMENT\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|25|HOST_cf2099570f34: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|26|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|27|HOST_cf2099570f34: Consumed 5.092s CPU time, 173.3M memory peak.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|28|Starting HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|29|SELFTEST_REPLACEMENT486=PASS\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|30|SCHEDULE_TARGET=xf-hkmods\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|31|SCHEDULE_RESULT=FAIL\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|32|SCHEDULE_REASON=COMMAND_FAILED\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|33|HOST_cf2099570f34: Main process exited, code=exited, status=2/INVALIDARGUMENT\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|34|HOST_cf2099570f34: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|35|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|36|HOST_cf2099570f34: Consumed 224ms CPU time, 135.4M memory peak.\nREC|FAILED_UNIT_STATE|homelab-private-vpn-hosts-health.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791262774|EXIT_EPOCH=1791262776|EXIT_AFTER_99670=true\nREC|FAILED_UNIT_FRAGMENT|homelab-private-vpn-hosts-health.service|BASENAME=homelab-private-vpn-hosts-health.service|SHA256=fdab069b1e8ed92423070a8c564bfd86347de3e578a728745e9d4567031b8cad\nREC|FAILED_UNIT_EXECSTART|homelab-private-vpn-hosts-health.service|BYTES=466|SHA256=3fdf138482d43ae2496ff06e578fd2c142d6e555c269b94c7a640b824ad89f71\nREC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=1|BASENAME=bash|BYTES=1298416|SHA256=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f\nREC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=2|BASENAME=homelab-check-private-vpn-hosts|BYTES=1411|SHA256=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82\nREC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=3|BASENAME=private-vpn-hosts.txt|BYTES=522|SHA256=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d\nREC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-private-vpn-hosts-health.service|3\nREC|FAILED_UNIT_JOURNAL|homelab-private-vpn-hosts-health.service|LINES=180|SHA256=ad53b326bbc251d100f99699a216e83051b3e061aa97cff85fd44f941aab9785\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|PVE_GUEST_NOT_UNIQUE|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|RPO_BREACH|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|BACKUP_STATUS_RED|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|BACKUP_STATUS_OK|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|COULD_NOT_RESOLVE|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|NAME_OR_SERVICE_NOT_KNOWN|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|CONNECTION_REFUSED|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|NO_ROUTE_TO_HOST|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TIMED_OUT|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TIMEOUT|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|FAILED|90\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|ERROR|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|STALE|0\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|1|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|2|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|3|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|4|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|5|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|6|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|7|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|8|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|9|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|10|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|11|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|12|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|13|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|14|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|15|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|16|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|17|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|18|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|19|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|20|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|21|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|22|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|23|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|24|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|25|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|26|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|27|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|28|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|29|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|30|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|31|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|32|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|33|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|34|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|35|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|36|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_STATE|skladchik-reports-monitor-supervisor.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791262362|EXIT_EPOCH=1791262365|EXIT_AFTER_99670=true\nREC|FAILED_UNIT_FRAGMENT|skladchik-reports-monitor-supervisor.service|BASENAME=skladchik-reports-monitor-supervisor.service|SHA256=e874fb03573b76181f3eb812c7285848e60705923c172a2f102504a7b9100ef7\nREC|FAILED_UNIT_EXECSTART|skladchik-reports-monitor-supervisor.service|BYTES=272|SHA256=b3cf723f4bbe0d2362700a0c43d5de45b84332256e771a0c6b5844bbe30663e7\nREC|FAILED_UNIT_EXEC_FILE|skladchik-reports-monitor-supervisor.service|IDX=1|BASENAME=skladchik-reports-monitor-supervisor|BYTES=16903|SHA256=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2\nREC|FAILED_UNIT_EXEC_FILE_COUNT|skladchik-reports-monitor-supervisor.service|1\nREC|FAILED_UNIT_JOURNAL|skladchik-reports-monitor-supervisor.service|LINES=180|SHA256=4e6ee35c336a9b77948519e1115141083538d03dec02a4f46f30dd837d1d4d66\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|PVE_GUEST_NOT_UNIQUE|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|RPO_BREACH|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|BACKUP_STATUS_RED|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|BACKUP_STATUS_OK|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|COULD_NOT_RESOLVE|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|NAME_OR_SERVICE_NOT_KNOWN|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|CONNECTION_REFUSED|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|NO_ROUTE_TO_HOST|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TIMED_OUT|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TIMEOUT|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|FAILED|3\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|ERROR|1\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|STALE|0\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|1|HOST_f87c67064af3: Deactivated successfully.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|2|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|3|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor...\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|4|SUPERVISOR_STATUS=OK\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|5|SUPERVISOR_MAIL_SENT=no\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|6|SUPERVISOR_PROBLEMS=none\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|7|MONITOR=OK reports=19 open=11 resolved=8\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|8|CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T203005Z\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|9|REGISTRY_SOURCE=IP_REDACTED\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|10|WORKER_TARGET=debian@IP_REDACTED\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|11|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|12|HOST_f87c67064af3: Deactivated successfully.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|13|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|14|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor...\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|15|SUPERVISOR_STATUS=OK\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|16|SUPERVISOR_MAIL_SENT=no\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|17|SUPERVISOR_PROBLEMS=none\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|18|MONITOR=OK reports=19 open=11 resolved=8\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|19|CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T203005Z\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|20|REGISTRY_SOURCE=IP_REDACTED\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|21|WORKER_TARGET=debian@IP_REDACTED\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|22|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|23|HOST_f87c67064af3: Deactivated successfully.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|24|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|25|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor...\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|26|SUPERVISOR_STATUS=FAIL\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|27|SUPERVISOR_MAIL_SENT=yes\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|28|SUPERVISOR_PROBLEMS=registry_source_not_core_apps,canonical_backup_probe_failed,canonical_backup_probe_status_not_ok,canonical_backup_app_identity_bad,canonical_backup_not_sealed_proven,canonical_backup_generation_bad\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|29|MONITOR=OK reports=19 open=11 resolved=8\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|30|CANONICAL_BACKUP= generation=\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|31|REGISTRY_SOURCE=TARGET_REGISTRY_ERROR=REGISTRY_SCHEMA_INVALID\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|32|WORKER_TARGET=debian@IP_REDACTED\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|33|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|34|HOST_f87c67064af3: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|35|HOST_f87c67064af3: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|36|Failed to start HOST_f87c67064af3 - Skladchik reports monitor full supervisor.\nREC|ROLE_TIMER|homelab-backup-v2.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262452|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false\nREC|ROLE_TIMER|homelab-backup-audit.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791261418|LAST_AFTER_99670=false|NEXT_EPOCH=1791263168|NEXT_PRESENT=true\nREC|ROLE_TIMER|homelab-private-vpn-hosts-health.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262774|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false\nREC|ROLE_TIMER|skladchik-reports-monitor-supervisor.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262362|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false\nTARGETS_SHA_NOW=21b1d1cc675e0472f05234f3cc160a43f603427a7ae6a511040ef7fb38722e5d\nTARGET_COUNT_NOW=20\nVM160_TARGET_COUNT_NOW=0\nREC|BACKUP_TARGET_NOW|vm110|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm111|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm112|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm113|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm130|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm150|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm170|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm171|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm190|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm9130|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm180|KIND=VM|LIFECYCLE=RETIRED_PRESERVE\nREC|BACKUP_TARGET_NOW|vm200|KIND=VM|LIFECYCLE=RETIRED_PRESERVE\nBACKUP_V2_WRAPPER_SHA=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nBACKUP_V2_AST_OK=true\nREC|BACKUP_V2_SAFE_STRING|fresh_lxc_helper.py\nREC|BACKUP_V2_SAFE_STRING|producer_exec_helper.py\nBACKUP_AUDIT_LATEST_TARGET_COUNT=16\nBACKUP_AUDIT_RPO_BREACH_COUNT=6\nREC|BACKUP_RPO_BREACH|vm110|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=83946\nREC|BACKUP_RPO_BREACH|vm111|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=82037\nREC|BACKUP_RPO_BREACH|vm130|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=87489\nREC|BACKUP_RPO_BREACH|vm160|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=94473\nREC|BACKUP_RPO_BREACH|vm171|STATE=FRESH|SOURCE=V2|REPLICA_COUNT=2|V2_AGE_SECONDS=33778\nREC|BACKUP_RPO_BREACH|vm9130|STATE=FRESH|SOURCE=V2|REPLICA_COUNT=2|V2_AGE_SECONDS=16454\nBACKUP_AUDIT_SUMMARY_STATUS=OK\nBACKUP_AUDIT_SUMMARY_BACKUP_STATUS=RED\nBACKUP_AUDIT_SUMMARY_ISSUE_COUNT=4\nREC|CLUSTER_GUEST_NOW|100|TYPE=qemu|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|101|TYPE=qemu|NODE=pve01|STATUS=stopped\nREC|CLUSTER_GUEST_NOW|110|TYPE=lxc|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|111|TYPE=lxc|NODE=pve02|STATUS=running\nREC|CLUSTER_GUEST_NOW|112|TYPE=lxc|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|113|TYPE=lxc|NODE=pve02|STATUS=running\nREC|CLUSTER_GUEST_NOW|130|TYPE=qemu|NODE=pve03|STATUS=running\nREC|CLUSTER_GUEST_NOW|150|TYPE=qemu|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|170|TYPE=qemu|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|171|TYPE=qemu|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|190|TYPE=qemu|NODE=pve03|STATUS=running\nREC|CLUSTER_GUEST_NOW|210|TYPE=qemu|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|9130|TYPE=qemu|NODE=pve02|STATUS=stopped\nCEPH_ENTERPRISE_DISABLED=true\nPVEENT_ENTERPRISE_DISABLED=true\nPVE_NO_SUBSCRIPTION_ACTIVE=true\nAPT_FULL_UPGRADE_SIM_RC=0\nAPT_FULL_UPGRADE_SIM_UPGRADE_COUNT=184\nAPT_FULL_UPGRADE_SIM_REMOVE_COUNT=0\nDECISION=PASS_PVE01_FAILED_UNITS_RECOVERY_RCA\nNEXT_GATE=PVE01_FAILED_UNIT_TARGETED_RECOVERY\nTASK_RESULT=PVE01_FAILED_UNITS_RECOVERY_RCA\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1.txt b/runtime/history/SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1.txt new file mode 100644 index 00000000..e2142740 --- /dev/null +++ b/runtime/history/SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1.txt @@ -0,0 +1,336 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=pve01-failed-units-recovery-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=9d9e9b613b9c82cf1a4117d91d95d83b00dfacba793118fdf72e1a046304d055 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=6ac9ff0e3cb90e523d09b3a848a7d0d45af85de8cfc3e50079599ed55add7df6 +SANITIZED_OUTPUT_SHA256=6ac9ff0e3cb90e523d09b3a848a7d0d45af85de8cfc3e50079599ed55add7df6 +OUTPUT_BEGIN +PVE01_FAILED_UNITS_RECOVERY_RCA=V1 +MODE=read-only +NO_REBOOT=true +NO_APT_UPDATE=true +NO_PACKAGE_MUTATION=true +NO_SERVICE_START_STOP_RESTART=true +NO_RESET_FAILED=true +NO_CONFIG_WRITE=true +NO_FILE_DELETE=true +NO_GUEST_MUTATION=true +NO_BACKUP_EXECUTION=true +NO_NETWORK_MUTATION=true +POLICY_REPAIR_EPOCH=1791262278 +NODE_HOSTNAME=pve01 +NODE_KERNEL=7.0.12-1-pve +SYSTEMD_SYSTEM_STATE=degraded +FAILED_UNIT_COUNT=4 +REC|FAILED_UNIT_STATE|homelab-backup-audit.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791261418|EXIT_EPOCH=1791261422|EXIT_AFTER_99670=false +REC|FAILED_UNIT_FRAGMENT|homelab-backup-audit.service|BASENAME=homelab-backup-audit.service|SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa +REC|FAILED_UNIT_EXECSTART|homelab-backup-audit.service|BYTES=244|SHA256=5a28904328b20e6dbeb2f818ff18ec6005c7dbf6ed24700b96f5bd528405cd47 +REC|FAILED_UNIT_EXEC_FILE|homelab-backup-audit.service|IDX=1|BASENAME=homelab-backup-audit|BYTES=19756|SHA256=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3 +REC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-backup-audit.service|1 +REC|FAILED_UNIT_JOURNAL|homelab-backup-audit.service|LINES=180|SHA256=9bd6eea7ef8c09f5562cd04df593419e77c0732f88a14fc88b15c7c5a3164421 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|PVE_GUEST_NOT_UNIQUE|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|RPO_BREACH|27 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|BACKUP_STATUS_RED|7 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|BACKUP_STATUS_OK|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|COULD_NOT_RESOLVE|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|NAME_OR_SERVICE_NOT_KNOWN|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|CONNECTION_REFUSED|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|NO_ROUTE_TO_HOST|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TIMED_OUT|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TIMEOUT|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|FAILED|14 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|ERROR|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|STALE|27 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|1|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=14548|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|2|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm110|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|3|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|4|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|5|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|6|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|7|HOST_a0dac0c74d36: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|8|HOST_a0dac0c74d36: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|9|Failed to start HOST_a0dac0c74d36 - Homelab unified backup and DR audit. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|10|HOST_a0dac0c74d36: Consumed 2.707s CPU time, 147.5M memory peak. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|11|Starting HOST_a0dac0c74d36 - Homelab unified backup and DR audit... +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|12|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=401|generation=app-filebrowser-pve01-20261006T043010Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|13|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=239|generation=app-hbarhub-pve01-20261006T043148Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|14|TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=345|generation=app-immich-pve01-20261006T043031Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|15|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=29207|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|16|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=42881|generation=control-bootstrap-v2-pve01-20260916T120057Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|17|TARGET|logical_id=vm110|kind=VM|state=STALE|source=NONE|v2_age_seconds=83946|legacy_age_seconds=2761050|replica_count=2|state_id=ct110|generation=vm110-pve01-20260904T053821Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|18|TARGET|logical_id=vm111|kind=VM|state=STALE|source=NONE|v2_age_seconds=82037|replica_count=2|state_id=ct111 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|19|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=67812|legacy_age_seconds=2799545|replica_count=2|state_id=ct112|generation=v2-ct112-20261005T094532Z-ee250335d44e +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|20|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=53076|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|21|TARGET|logical_id=vm130|kind=VM|state=STALE|source=NONE|v2_age_seconds=87489|replica_count=2|state_id=vm130 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|22|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=50972|legacy_age_seconds=69392|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|23|TARGET|logical_id=vm160|kind=VM|state=STALE|source=NONE|v2_age_seconds=94473|replica_count=2|state_id=vm160 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|24|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=37543|legacy_age_seconds=1705787|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|25|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=33778|legacy_age_seconds=1735319|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|26|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=48781|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|27|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=16454|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|28|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm110|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|29|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|30|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|31|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|32|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|33|HOST_a0dac0c74d36: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|34|HOST_a0dac0c74d36: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|35|Failed to start HOST_a0dac0c74d36 - Homelab unified backup and DR audit. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|36|HOST_a0dac0c74d36: Consumed 2.761s CPU time, 148.4M memory peak. +REC|FAILED_UNIT_STATE|homelab-backup-v2.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=2|START_EPOCH=1791262452|EXIT_EPOCH=1791262453|EXIT_AFTER_99670=true +REC|FAILED_UNIT_FRAGMENT|homelab-backup-v2.service|BASENAME=homelab-backup-v2.service|SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565 +REC|FAILED_UNIT_EXECSTART|homelab-backup-v2.service|BYTES=238|SHA256=9560de727725d1afd2a98e905f0aad5c2b1fbb1b3348ea1fdc4aa13e5038bdae +REC|FAILED_UNIT_EXEC_FILE|homelab-backup-v2.service|IDX=1|BASENAME=homelab-backup-v2|BYTES=1656|SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +REC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-backup-v2.service|1 +REC|FAILED_UNIT_JOURNAL|homelab-backup-v2.service|LINES=180|SHA256=d9f31b02b241f88591dbad318d411219659eff0bbb7feb44dd913e38461a507e +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|PVE_GUEST_NOT_UNIQUE|8 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|RPO_BREACH|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|BACKUP_STATUS_RED|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|BACKUP_STATUS_OK|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|COULD_NOT_RESOLVE|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|NAME_OR_SERVICE_NOT_KNOWN|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|CONNECTION_REFUSED|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|NO_ROUTE_TO_HOST|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TIMED_OUT|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TIMEOUT|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|FAILED|14 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|ERROR|8 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|STALE|0 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|1| ~~~~~~~~~~~~~~^^^^^^^^^^^^ +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|2| File "PATH_REDACTED, line 873, in capture_target +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|3| if typ=='pve_guest': return capture_pve_guest(t,gen,work) +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|4| ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|5| File "PATH_REDACTED, line 192, in capture_pve_guest +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|6| if HOST_4c1ea9873d5f('guest_type')!='lxc': return original(t,gen,work) +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|7| ~~~~~~~~^^^^^^^^^^^^ +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|8| File "PATH_REDACTED, line 44, in capture_pve_guest +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|9| _need(len(rows)==1,'PVE_GUEST_NOT_UNIQUE') +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|10| ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|11| File "PATH_REDACTED, line 9, in _need +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|12| if not ok: raise QemuAdapterError(code) +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|13| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|14|qemu_adapter.QemuAdapterError: PVE_GUEST_NOT_UNIQUE +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|15|HOST_cf2099570f34: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|16|HOST_cf2099570f34: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|17|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|18|HOST_cf2099570f34: Consumed 665ms CPU time, 169.5M memory peak. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|19|Starting HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler... +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|20|SELFTEST_REPLACEMENT486=PASS +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|21|SCHEDULE_TARGET=guest-pve01-lxc-112 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|22|SCHEDULE_RESULT=FAIL +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|23|SCHEDULE_REASON=CT112_PIPELINE_FAILED +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|24|HOST_cf2099570f34: Main process exited, code=exited, status=2/INVALIDARGUMENT +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|25|HOST_cf2099570f34: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|26|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|27|HOST_cf2099570f34: Consumed 5.092s CPU time, 173.3M memory peak. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|28|Starting HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler... +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|29|SELFTEST_REPLACEMENT486=PASS +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|30|SCHEDULE_TARGET=xf-hkmods +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|31|SCHEDULE_RESULT=FAIL +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|32|SCHEDULE_REASON=COMMAND_FAILED +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|33|HOST_cf2099570f34: Main process exited, code=exited, status=2/INVALIDARGUMENT +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|34|HOST_cf2099570f34: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|35|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|36|HOST_cf2099570f34: Consumed 224ms CPU time, 135.4M memory peak. +REC|FAILED_UNIT_STATE|homelab-private-vpn-hosts-health.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791262774|EXIT_EPOCH=1791262776|EXIT_AFTER_99670=true +REC|FAILED_UNIT_FRAGMENT|homelab-private-vpn-hosts-health.service|BASENAME=homelab-private-vpn-hosts-health.service|SHA256=fdab069b1e8ed92423070a8c564bfd86347de3e578a728745e9d4567031b8cad +REC|FAILED_UNIT_EXECSTART|homelab-private-vpn-hosts-health.service|BYTES=466|SHA256=3fdf138482d43ae2496ff06e578fd2c142d6e555c269b94c7a640b824ad89f71 +REC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=1|BASENAME=bash|BYTES=1298416|SHA256=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f +REC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=2|BASENAME=homelab-check-private-vpn-hosts|BYTES=1411|SHA256=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82 +REC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=3|BASENAME=private-vpn-hosts.txt|BYTES=522|SHA256=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d +REC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-private-vpn-hosts-health.service|3 +REC|FAILED_UNIT_JOURNAL|homelab-private-vpn-hosts-health.service|LINES=180|SHA256=ad53b326bbc251d100f99699a216e83051b3e061aa97cff85fd44f941aab9785 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|PVE_GUEST_NOT_UNIQUE|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|RPO_BREACH|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|BACKUP_STATUS_RED|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|BACKUP_STATUS_OK|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|COULD_NOT_RESOLVE|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|NAME_OR_SERVICE_NOT_KNOWN|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|CONNECTION_REFUSED|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|NO_ROUTE_TO_HOST|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TIMED_OUT|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TIMEOUT|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|FAILED|90 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|ERROR|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|STALE|0 +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|1|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|2|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|3|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|4|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|5|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|6|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|7|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|8|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|9|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|10|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|11|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|12|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|13|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|14|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|15|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|16|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|17|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|18|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|19|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|20|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|21|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|22|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|23|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|24|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|25|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|26|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|27|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|28|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|29|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|30|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|31|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|32|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|33|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|34|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|35|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|36|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_STATE|skladchik-reports-monitor-supervisor.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791262362|EXIT_EPOCH=1791262365|EXIT_AFTER_99670=true +REC|FAILED_UNIT_FRAGMENT|skladchik-reports-monitor-supervisor.service|BASENAME=skladchik-reports-monitor-supervisor.service|SHA256=e874fb03573b76181f3eb812c7285848e60705923c172a2f102504a7b9100ef7 +REC|FAILED_UNIT_EXECSTART|skladchik-reports-monitor-supervisor.service|BYTES=272|SHA256=b3cf723f4bbe0d2362700a0c43d5de45b84332256e771a0c6b5844bbe30663e7 +REC|FAILED_UNIT_EXEC_FILE|skladchik-reports-monitor-supervisor.service|IDX=1|BASENAME=skladchik-reports-monitor-supervisor|BYTES=16903|SHA256=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2 +REC|FAILED_UNIT_EXEC_FILE_COUNT|skladchik-reports-monitor-supervisor.service|1 +REC|FAILED_UNIT_JOURNAL|skladchik-reports-monitor-supervisor.service|LINES=180|SHA256=4e6ee35c336a9b77948519e1115141083538d03dec02a4f46f30dd837d1d4d66 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|PVE_GUEST_NOT_UNIQUE|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|RPO_BREACH|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|BACKUP_STATUS_RED|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|BACKUP_STATUS_OK|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|COULD_NOT_RESOLVE|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|NAME_OR_SERVICE_NOT_KNOWN|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|CONNECTION_REFUSED|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|NO_ROUTE_TO_HOST|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TIMED_OUT|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TIMEOUT|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|FAILED|3 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|ERROR|1 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|STALE|0 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|1|HOST_f87c67064af3: Deactivated successfully. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|2|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|3|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor... +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|4|SUPERVISOR_STATUS=OK +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|5|SUPERVISOR_MAIL_SENT=no +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|6|SUPERVISOR_PROBLEMS=none +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|7|MONITOR=OK reports=19 open=11 resolved=8 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|8|CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T203005Z +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|9|REGISTRY_SOURCE=IP_REDACTED +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|10|WORKER_TARGET=debian@IP_REDACTED +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|11|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|12|HOST_f87c67064af3: Deactivated successfully. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|13|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|14|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor... +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|15|SUPERVISOR_STATUS=OK +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|16|SUPERVISOR_MAIL_SENT=no +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|17|SUPERVISOR_PROBLEMS=none +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|18|MONITOR=OK reports=19 open=11 resolved=8 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|19|CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T203005Z +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|20|REGISTRY_SOURCE=IP_REDACTED +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|21|WORKER_TARGET=debian@IP_REDACTED +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|22|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|23|HOST_f87c67064af3: Deactivated successfully. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|24|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|25|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor... +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|26|SUPERVISOR_STATUS=FAIL +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|27|SUPERVISOR_MAIL_SENT=yes +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|28|SUPERVISOR_PROBLEMS=registry_source_not_core_apps,canonical_backup_probe_failed,canonical_backup_probe_status_not_ok,canonical_backup_app_identity_bad,canonical_backup_not_sealed_proven,canonical_backup_generation_bad +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|29|MONITOR=OK reports=19 open=11 resolved=8 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|30|CANONICAL_BACKUP= generation= +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|31|REGISTRY_SOURCE=TARGET_REGISTRY_ERROR=REGISTRY_SCHEMA_INVALID +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|32|WORKER_TARGET=debian@IP_REDACTED +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|33|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|34|HOST_f87c67064af3: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|35|HOST_f87c67064af3: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|36|Failed to start HOST_f87c67064af3 - Skladchik reports monitor full supervisor. +REC|ROLE_TIMER|homelab-backup-v2.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262452|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false +REC|ROLE_TIMER|homelab-backup-audit.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791261418|LAST_AFTER_99670=false|NEXT_EPOCH=1791263168|NEXT_PRESENT=true +REC|ROLE_TIMER|homelab-private-vpn-hosts-health.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262774|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false +REC|ROLE_TIMER|skladchik-reports-monitor-supervisor.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262362|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false +TARGETS_SHA_NOW=21b1d1cc675e0472f05234f3cc160a43f603427a7ae6a511040ef7fb38722e5d +TARGET_COUNT_NOW=20 +VM160_TARGET_COUNT_NOW=0 +REC|BACKUP_TARGET_NOW|vm110|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm111|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm112|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm113|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm130|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm150|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm170|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm171|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm190|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm9130|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm180|KIND=VM|LIFECYCLE=RETIRED_PRESERVE +REC|BACKUP_TARGET_NOW|vm200|KIND=VM|LIFECYCLE=RETIRED_PRESERVE +BACKUP_V2_WRAPPER_SHA=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +BACKUP_V2_AST_OK=true +REC|BACKUP_V2_SAFE_STRING|fresh_lxc_helper.py +REC|BACKUP_V2_SAFE_STRING|producer_exec_helper.py +BACKUP_AUDIT_LATEST_TARGET_COUNT=16 +BACKUP_AUDIT_RPO_BREACH_COUNT=6 +REC|BACKUP_RPO_BREACH|vm110|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=83946 +REC|BACKUP_RPO_BREACH|vm111|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=82037 +REC|BACKUP_RPO_BREACH|vm130|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=87489 +REC|BACKUP_RPO_BREACH|vm160|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=94473 +REC|BACKUP_RPO_BREACH|vm171|STATE=FRESH|SOURCE=V2|REPLICA_COUNT=2|V2_AGE_SECONDS=33778 +REC|BACKUP_RPO_BREACH|vm9130|STATE=FRESH|SOURCE=V2|REPLICA_COUNT=2|V2_AGE_SECONDS=16454 +BACKUP_AUDIT_SUMMARY_STATUS=OK +BACKUP_AUDIT_SUMMARY_BACKUP_STATUS=RED +BACKUP_AUDIT_SUMMARY_ISSUE_COUNT=4 +REC|CLUSTER_GUEST_NOW|100|TYPE=qemu|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|101|TYPE=qemu|NODE=pve01|STATUS=stopped +REC|CLUSTER_GUEST_NOW|110|TYPE=lxc|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|111|TYPE=lxc|NODE=pve02|STATUS=running +REC|CLUSTER_GUEST_NOW|112|TYPE=lxc|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|113|TYPE=lxc|NODE=pve02|STATUS=running +REC|CLUSTER_GUEST_NOW|130|TYPE=qemu|NODE=pve03|STATUS=running +REC|CLUSTER_GUEST_NOW|150|TYPE=qemu|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|170|TYPE=qemu|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|171|TYPE=qemu|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|190|TYPE=qemu|NODE=pve03|STATUS=running +REC|CLUSTER_GUEST_NOW|210|TYPE=qemu|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|9130|TYPE=qemu|NODE=pve02|STATUS=stopped +CEPH_ENTERPRISE_DISABLED=true +PVEENT_ENTERPRISE_DISABLED=true +PVE_NO_SUBSCRIPTION_ACTIVE=true +APT_FULL_UPGRADE_SIM_RC=0 +APT_FULL_UPGRADE_SIM_UPGRADE_COUNT=184 +APT_FULL_UPGRADE_SIM_REMOVE_COUNT=0 +DECISION=PASS_PVE01_FAILED_UNITS_RECOVERY_RCA +NEXT_GATE=PVE01_FAILED_UNIT_TARGETED_RECOVERY +TASK_RESULT=PVE01_FAILED_UNITS_RECOVERY_RCA +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index c6d0e2f1..75416620 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,27 +1,27 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-261006-PVE03-FINAL-READINESS-STALE-KERNEL-CLEANUP-99671R1", - "status": "FAIL", - "rc": 60, + "command_id": "SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "pve03-final-readiness-stale-kernel-cleanup", - "started_at_utc": "2026-10-06T04:56:23Z", - "finished_at_utc": "2026-10-06T04:56:48Z", + "mode": "read-only", + "component": "pve01-failed-units-recovery-rca", + "started_at_utc": "2026-10-06T05:00:17Z", + "finished_at_utc": "2026-10-06T05:00:20Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "24fefa15fbc73c97d999c10a09f178f009aeaef989493be4a750de0f7b7637dd", + "command_sha256": "9d9e9b613b9c82cf1a4117d91d95d83b00dfacba793118fdf72e1a046304d055", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, + "change_declared": false, "result_contract_valid": true, - "result_contract_status": "FAIL", + "result_contract_status": null, "result_contract_error": null, - "command_rc": 60, + "command_rc": 0, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "296d9d71531385bf7a2e78264040a0ec7099683aa27040e1cf6efa453ef2bdc4", - "sanitized_output_sha256": "296d9d71531385bf7a2e78264040a0ec7099683aa27040e1cf6efa453ef2bdc4", + "raw_evidence_sha256": "6ac9ff0e3cb90e523d09b3a848a7d0d45af85de8cfc3e50079599ed55add7df6", + "sanitized_output_sha256": "6ac9ff0e3cb90e523d09b3a848a7d0d45af85de8cfc3e50079599ed55add7df6", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "PVE03_FINAL_READINESS_STALE_KERNEL_CLEANUP=V1\nMODE=change\nNO_REBOOT=true\nNO_AUTOREMOVE=true\nNO_GUEST_MUTATION=true\nNO_NETWORK_MUTATION=true\nNO_DNS_MUTATION=true\nNO_VM211_RESTORE=true\nNO_BACKUP_DELETE=true\nCURRENT_KERNEL_MUST_REMAIN=7.0.14-20-pve\nFALLBACK_KERNEL_MUST_REMAIN=7.0.12-1-pve\nPRE_CLUSTER_RC=0\nPRE_QUORATE=true\nPRE_PVE01_ONLINE=true\nPRE_PVE02_ONLINE=true\nPRE_PVE03_ONLINE=true\nPRE_VM130_RUNNING=true\nPRE_VM190_RUNNING=true\nPRE_VM9130_STOPPED=true\nPRE_VM211_ABSENT=true\nPRE_VM130_ONBOOT=1\nPRE_VM190_ONBOOT=1\nPRE_VM9130_ONBOOT=0\nPRE_VM9130_PROTECTION=1\nRUNNING_KERNEL_PRE=7.0.14-20-pve\nSIGNED_KERNEL_QUERY_RC_PRE=0\nSIGNED_KERNEL_COUNT_PRE=4\nREC|SIGNED_KERNEL_PRE|proxmox-kernel-7.0.12-1-pve-signed|7.0.12-1\nREC|SIGNED_KERNEL_PRE|proxmox-kernel-7.0.14-20-pve-signed|7.0.14-20\nREC|SIGNED_KERNEL_PRE|proxmox-kernel-7.0.2-6-pve-signed|7.0.2-6\nREC|SIGNED_KERNEL_PRE|proxmox-kernel-7.0.6-2-pve-signed|7.0.6-2\nBOOT_TOOL_RC_PRE=0\nBOOT_TOOL_HAS_CURRENT_PRE=true\nBOOT_TOOL_HAS_FALLBACK_PRE=true\nBOOT_PIN_PRESENT_PRE=false\nNEXT_BOOT_PIN_PRESENT_PRE=false\nAPT_SIM_RC_PRE=0\nAPT_INST_COUNT_PRE=0\nAPT_REMV_COUNT_PRE=0\nFAILED_UNIT_COUNT_PRE=0\nSERVICE_PVEPROXY_PRE=active\nSERVICE_PVEDAEMON_PRE=active\nSERVICE_PVESTATD_PRE=active\nSERVICE_PVE_CLUSTER_PRE=active\nSERVICE_COROSYNC_PRE=active\nSERVICE_NETBIRD_PRE=active\nHEALTH_SCRIPT_SHA_PRE=a01b5c63e1cfa5bc2eae4b12031de517345541c343f1b359abe832cd41e71b55\nHEALTH_NEWFI_PRE=0\nHEALTH_OTHER_FORUM_PRE=12\nHEALTH_SYNTAX_PRE=0\nHEALTH_TIMER_ACTIVE_PRE=active\nHEALTH_TIMER_ENABLED_PRE=enabled\nVM211_BACKUP_READY_PRE=true\nVM211_BACKUP_SHA_MATCH_PRE=true\nVM211_BACKUP_ZSTD_RC_PRE=0\nVM211_BACKUP_VMA_RC_PRE=0\nVM130_QGA_PING_RC=0\nVM130_QGA_NETWORK_RC=0\nVM130_DOCKER_TRANSPORT_RC=0\nVM130_DOCKER_EXITCODE=0\nVM130_NPMPLUS_TRANSPORT_RC=0\nVM130_NPMPLUS_EXITCODE=0\nVM130_ATOMIC_EDGE_PROOF=true\nREC|PUBLIC_EDGE_PRE|codevipe.ru|HTTP=200|PROC_RC=0|OK=true\nREC|PUBLIC_EDGE_PRE|hkmods.ru|HTTP=200|PROC_RC=0|OK=true\nREC|PUBLIC_EDGE_PRE|dsmods.ru|HTTP=200|PROC_RC=0|OK=true\nREC|PUBLIC_EDGE_PRE|gamevipe.ru|HTTP=200|PROC_RC=0|OK=true\nREC|PUBLIC_EDGE_PRE|kingofwolk.ru|HTTP=200|PROC_RC=0|OK=true\nREC|PUBLIC_EDGE_PRE|zakrutim.ru|HTTP=200|PROC_RC=0|OK=true\nPUBLIC_EDGE_PRE_GOOD_COUNT=6\nPUBLIC_EDGE_PRE_ALL_OK=true\nSTALE_PURGE_SIM_RC=0\nSTALE_PURGE_SIM_INST_COUNT=0\nSTALE_PURGE_SIM_REMV_COUNT=0\nSTALE_PURGE_SIM_INST_SET_SHA=NONE\nSTALE_PURGE_SIM_REMV_SET_SHA=NONE\nSTALE_PURGE_EXACT_SAFE=false\nFINAL_CLEANUP_PREFLIGHT=false\nDECISION=HOLD_PVE03_FINAL_CLEANUP_PREFLIGHT\nNEXT_GATE=PVE03_FINAL_CLEANUP_RECONCILE\nCOMMANDS_REMAINING_ESTIMATE=1\nTASK_RESULT=PVE03_FINAL_READINESS_STALE_KERNEL_CLEANUP\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261006-PVE03-FINAL-READINESS-STALE-KERNEL-CLEANUP-99671R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "PVE01_FAILED_UNITS_RECOVERY_RCA=V1\nMODE=read-only\nNO_REBOOT=true\nNO_APT_UPDATE=true\nNO_PACKAGE_MUTATION=true\nNO_SERVICE_START_STOP_RESTART=true\nNO_RESET_FAILED=true\nNO_CONFIG_WRITE=true\nNO_FILE_DELETE=true\nNO_GUEST_MUTATION=true\nNO_BACKUP_EXECUTION=true\nNO_NETWORK_MUTATION=true\nPOLICY_REPAIR_EPOCH=1791262278\nNODE_HOSTNAME=pve01\nNODE_KERNEL=7.0.12-1-pve\nSYSTEMD_SYSTEM_STATE=degraded\nFAILED_UNIT_COUNT=4\nREC|FAILED_UNIT_STATE|homelab-backup-audit.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791261418|EXIT_EPOCH=1791261422|EXIT_AFTER_99670=false\nREC|FAILED_UNIT_FRAGMENT|homelab-backup-audit.service|BASENAME=homelab-backup-audit.service|SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa\nREC|FAILED_UNIT_EXECSTART|homelab-backup-audit.service|BYTES=244|SHA256=5a28904328b20e6dbeb2f818ff18ec6005c7dbf6ed24700b96f5bd528405cd47\nREC|FAILED_UNIT_EXEC_FILE|homelab-backup-audit.service|IDX=1|BASENAME=homelab-backup-audit|BYTES=19756|SHA256=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3\nREC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-backup-audit.service|1\nREC|FAILED_UNIT_JOURNAL|homelab-backup-audit.service|LINES=180|SHA256=9bd6eea7ef8c09f5562cd04df593419e77c0732f88a14fc88b15c7c5a3164421\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|PVE_GUEST_NOT_UNIQUE|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|RPO_BREACH|27\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|BACKUP_STATUS_RED|7\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|BACKUP_STATUS_OK|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|COULD_NOT_RESOLVE|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|NAME_OR_SERVICE_NOT_KNOWN|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|CONNECTION_REFUSED|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|NO_ROUTE_TO_HOST|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TIMED_OUT|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TIMEOUT|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|FAILED|14\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|ERROR|0\nREC|FAILED_UNIT_CLASS|homelab-backup-audit.service|STALE|27\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|1|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=14548|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|2|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm110|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|3|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|4|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|5|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|6|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|7|HOST_a0dac0c74d36: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|8|HOST_a0dac0c74d36: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|9|Failed to start HOST_a0dac0c74d36 - Homelab unified backup and DR audit.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|10|HOST_a0dac0c74d36: Consumed 2.707s CPU time, 147.5M memory peak.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|11|Starting HOST_a0dac0c74d36 - Homelab unified backup and DR audit...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|12|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=401|generation=app-filebrowser-pve01-20261006T043010Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|13|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=239|generation=app-hbarhub-pve01-20261006T043148Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|14|TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=345|generation=app-immich-pve01-20261006T043031Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|15|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=29207|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|16|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=42881|generation=control-bootstrap-v2-pve01-20260916T120057Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|17|TARGET|logical_id=vm110|kind=VM|state=STALE|source=NONE|v2_age_seconds=83946|legacy_age_seconds=2761050|replica_count=2|state_id=ct110|generation=vm110-pve01-20260904T053821Z\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|18|TARGET|logical_id=vm111|kind=VM|state=STALE|source=NONE|v2_age_seconds=82037|replica_count=2|state_id=ct111\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|19|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=67812|legacy_age_seconds=2799545|replica_count=2|state_id=ct112|generation=v2-ct112-20261005T094532Z-ee250335d44e\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|20|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=53076|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|21|TARGET|logical_id=vm130|kind=VM|state=STALE|source=NONE|v2_age_seconds=87489|replica_count=2|state_id=vm130\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|22|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=50972|legacy_age_seconds=69392|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|23|TARGET|logical_id=vm160|kind=VM|state=STALE|source=NONE|v2_age_seconds=94473|replica_count=2|state_id=vm160\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|24|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=37543|legacy_age_seconds=1705787|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|25|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=33778|legacy_age_seconds=1735319|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|26|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=48781|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|27|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=16454|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|28|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm110|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|29|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|30|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|31|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|32|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|33|HOST_a0dac0c74d36: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|34|HOST_a0dac0c74d36: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|35|Failed to start HOST_a0dac0c74d36 - Homelab unified backup and DR audit.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|36|HOST_a0dac0c74d36: Consumed 2.761s CPU time, 148.4M memory peak.\nREC|FAILED_UNIT_STATE|homelab-backup-v2.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=2|START_EPOCH=1791262452|EXIT_EPOCH=1791262453|EXIT_AFTER_99670=true\nREC|FAILED_UNIT_FRAGMENT|homelab-backup-v2.service|BASENAME=homelab-backup-v2.service|SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565\nREC|FAILED_UNIT_EXECSTART|homelab-backup-v2.service|BYTES=238|SHA256=9560de727725d1afd2a98e905f0aad5c2b1fbb1b3348ea1fdc4aa13e5038bdae\nREC|FAILED_UNIT_EXEC_FILE|homelab-backup-v2.service|IDX=1|BASENAME=homelab-backup-v2|BYTES=1656|SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nREC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-backup-v2.service|1\nREC|FAILED_UNIT_JOURNAL|homelab-backup-v2.service|LINES=180|SHA256=d9f31b02b241f88591dbad318d411219659eff0bbb7feb44dd913e38461a507e\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|PVE_GUEST_NOT_UNIQUE|8\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|RPO_BREACH|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|BACKUP_STATUS_RED|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|BACKUP_STATUS_OK|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|COULD_NOT_RESOLVE|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|NAME_OR_SERVICE_NOT_KNOWN|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|CONNECTION_REFUSED|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|NO_ROUTE_TO_HOST|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TIMED_OUT|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TIMEOUT|0\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|FAILED|14\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|ERROR|8\nREC|FAILED_UNIT_CLASS|homelab-backup-v2.service|STALE|0\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|1| ~~~~~~~~~~~~~~^^^^^^^^^^^^\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|2| File \"PATH_REDACTED, line 873, in capture_target\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|3| if typ=='pve_guest': return capture_pve_guest(t,gen,work)\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|4| ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|5| File \"PATH_REDACTED, line 192, in capture_pve_guest\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|6| if HOST_4c1ea9873d5f('guest_type')!='lxc': return original(t,gen,work)\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|7| ~~~~~~~~^^^^^^^^^^^^\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|8| File \"PATH_REDACTED, line 44, in capture_pve_guest\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|9| _need(len(rows)==1,'PVE_GUEST_NOT_UNIQUE')\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|10| ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|11| File \"PATH_REDACTED, line 9, in _need\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|12| if not ok: raise QemuAdapterError(code)\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|13| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|14|qemu_adapter.QemuAdapterError: PVE_GUEST_NOT_UNIQUE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|15|HOST_cf2099570f34: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|16|HOST_cf2099570f34: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|17|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|18|HOST_cf2099570f34: Consumed 665ms CPU time, 169.5M memory peak.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|19|Starting HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|20|SELFTEST_REPLACEMENT486=PASS\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|21|SCHEDULE_TARGET=guest-pve01-lxc-112\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|22|SCHEDULE_RESULT=FAIL\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|23|SCHEDULE_REASON=CT112_PIPELINE_FAILED\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|24|HOST_cf2099570f34: Main process exited, code=exited, status=2/INVALIDARGUMENT\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|25|HOST_cf2099570f34: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|26|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|27|HOST_cf2099570f34: Consumed 5.092s CPU time, 173.3M memory peak.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|28|Starting HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|29|SELFTEST_REPLACEMENT486=PASS\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|30|SCHEDULE_TARGET=xf-hkmods\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|31|SCHEDULE_RESULT=FAIL\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|32|SCHEDULE_REASON=COMMAND_FAILED\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|33|HOST_cf2099570f34: Main process exited, code=exited, status=2/INVALIDARGUMENT\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|34|HOST_cf2099570f34: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|35|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|36|HOST_cf2099570f34: Consumed 224ms CPU time, 135.4M memory peak.\nREC|FAILED_UNIT_STATE|homelab-private-vpn-hosts-health.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791262774|EXIT_EPOCH=1791262776|EXIT_AFTER_99670=true\nREC|FAILED_UNIT_FRAGMENT|homelab-private-vpn-hosts-health.service|BASENAME=homelab-private-vpn-hosts-health.service|SHA256=fdab069b1e8ed92423070a8c564bfd86347de3e578a728745e9d4567031b8cad\nREC|FAILED_UNIT_EXECSTART|homelab-private-vpn-hosts-health.service|BYTES=466|SHA256=3fdf138482d43ae2496ff06e578fd2c142d6e555c269b94c7a640b824ad89f71\nREC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=1|BASENAME=bash|BYTES=1298416|SHA256=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f\nREC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=2|BASENAME=homelab-check-private-vpn-hosts|BYTES=1411|SHA256=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82\nREC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=3|BASENAME=private-vpn-hosts.txt|BYTES=522|SHA256=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d\nREC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-private-vpn-hosts-health.service|3\nREC|FAILED_UNIT_JOURNAL|homelab-private-vpn-hosts-health.service|LINES=180|SHA256=ad53b326bbc251d100f99699a216e83051b3e061aa97cff85fd44f941aab9785\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|PVE_GUEST_NOT_UNIQUE|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|RPO_BREACH|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|BACKUP_STATUS_RED|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|BACKUP_STATUS_OK|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|COULD_NOT_RESOLVE|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|NAME_OR_SERVICE_NOT_KNOWN|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|CONNECTION_REFUSED|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|NO_ROUTE_TO_HOST|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TIMED_OUT|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TIMEOUT|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|FAILED|90\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|ERROR|0\nREC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|STALE|0\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|1|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|2|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|3|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|4|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|5|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|6|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|7|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|8|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|9|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|10|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|11|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|12|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|13|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|14|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|15|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|16|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|17|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|18|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|19|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|20|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|21|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|22|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|23|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|24|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|25|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|26|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|27|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|28|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|29|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|30|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|31|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|32|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|33|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check...\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|34|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|35|HOST_bf82fe335fe6: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|36|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check.\nREC|FAILED_UNIT_STATE|skladchik-reports-monitor-supervisor.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791262362|EXIT_EPOCH=1791262365|EXIT_AFTER_99670=true\nREC|FAILED_UNIT_FRAGMENT|skladchik-reports-monitor-supervisor.service|BASENAME=skladchik-reports-monitor-supervisor.service|SHA256=e874fb03573b76181f3eb812c7285848e60705923c172a2f102504a7b9100ef7\nREC|FAILED_UNIT_EXECSTART|skladchik-reports-monitor-supervisor.service|BYTES=272|SHA256=b3cf723f4bbe0d2362700a0c43d5de45b84332256e771a0c6b5844bbe30663e7\nREC|FAILED_UNIT_EXEC_FILE|skladchik-reports-monitor-supervisor.service|IDX=1|BASENAME=skladchik-reports-monitor-supervisor|BYTES=16903|SHA256=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2\nREC|FAILED_UNIT_EXEC_FILE_COUNT|skladchik-reports-monitor-supervisor.service|1\nREC|FAILED_UNIT_JOURNAL|skladchik-reports-monitor-supervisor.service|LINES=180|SHA256=4e6ee35c336a9b77948519e1115141083538d03dec02a4f46f30dd837d1d4d66\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|PVE_GUEST_NOT_UNIQUE|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|RPO_BREACH|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|BACKUP_STATUS_RED|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|BACKUP_STATUS_OK|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|COULD_NOT_RESOLVE|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|NAME_OR_SERVICE_NOT_KNOWN|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|CONNECTION_REFUSED|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|NO_ROUTE_TO_HOST|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TIMED_OUT|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TIMEOUT|0\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|FAILED|3\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|ERROR|1\nREC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|STALE|0\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|1|HOST_f87c67064af3: Deactivated successfully.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|2|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|3|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor...\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|4|SUPERVISOR_STATUS=OK\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|5|SUPERVISOR_MAIL_SENT=no\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|6|SUPERVISOR_PROBLEMS=none\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|7|MONITOR=OK reports=19 open=11 resolved=8\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|8|CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T203005Z\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|9|REGISTRY_SOURCE=IP_REDACTED\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|10|WORKER_TARGET=debian@IP_REDACTED\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|11|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|12|HOST_f87c67064af3: Deactivated successfully.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|13|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|14|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor...\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|15|SUPERVISOR_STATUS=OK\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|16|SUPERVISOR_MAIL_SENT=no\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|17|SUPERVISOR_PROBLEMS=none\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|18|MONITOR=OK reports=19 open=11 resolved=8\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|19|CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T203005Z\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|20|REGISTRY_SOURCE=IP_REDACTED\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|21|WORKER_TARGET=debian@IP_REDACTED\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|22|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|23|HOST_f87c67064af3: Deactivated successfully.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|24|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|25|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor...\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|26|SUPERVISOR_STATUS=FAIL\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|27|SUPERVISOR_MAIL_SENT=yes\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|28|SUPERVISOR_PROBLEMS=registry_source_not_core_apps,canonical_backup_probe_failed,canonical_backup_probe_status_not_ok,canonical_backup_app_identity_bad,canonical_backup_not_sealed_proven,canonical_backup_generation_bad\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|29|MONITOR=OK reports=19 open=11 resolved=8\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|30|CANONICAL_BACKUP= generation=\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|31|REGISTRY_SOURCE=TARGET_REGISTRY_ERROR=REGISTRY_SCHEMA_INVALID\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|32|WORKER_TARGET=debian@IP_REDACTED\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|33|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|34|HOST_f87c67064af3: Main process exited, code=exited, status=1/FAILURE\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|35|HOST_f87c67064af3: Failed with result 'exit-code'.\nREC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|36|Failed to start HOST_f87c67064af3 - Skladchik reports monitor full supervisor.\nREC|ROLE_TIMER|homelab-backup-v2.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262452|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false\nREC|ROLE_TIMER|homelab-backup-audit.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791261418|LAST_AFTER_99670=false|NEXT_EPOCH=1791263168|NEXT_PRESENT=true\nREC|ROLE_TIMER|homelab-private-vpn-hosts-health.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262774|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false\nREC|ROLE_TIMER|skladchik-reports-monitor-supervisor.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262362|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false\nTARGETS_SHA_NOW=21b1d1cc675e0472f05234f3cc160a43f603427a7ae6a511040ef7fb38722e5d\nTARGET_COUNT_NOW=20\nVM160_TARGET_COUNT_NOW=0\nREC|BACKUP_TARGET_NOW|vm110|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm111|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm112|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm113|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm130|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm150|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm170|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm171|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm190|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm9130|KIND=VM|LIFECYCLE=ACTIVE\nREC|BACKUP_TARGET_NOW|vm180|KIND=VM|LIFECYCLE=RETIRED_PRESERVE\nREC|BACKUP_TARGET_NOW|vm200|KIND=VM|LIFECYCLE=RETIRED_PRESERVE\nBACKUP_V2_WRAPPER_SHA=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nBACKUP_V2_AST_OK=true\nREC|BACKUP_V2_SAFE_STRING|fresh_lxc_helper.py\nREC|BACKUP_V2_SAFE_STRING|producer_exec_helper.py\nBACKUP_AUDIT_LATEST_TARGET_COUNT=16\nBACKUP_AUDIT_RPO_BREACH_COUNT=6\nREC|BACKUP_RPO_BREACH|vm110|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=83946\nREC|BACKUP_RPO_BREACH|vm111|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=82037\nREC|BACKUP_RPO_BREACH|vm130|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=87489\nREC|BACKUP_RPO_BREACH|vm160|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=94473\nREC|BACKUP_RPO_BREACH|vm171|STATE=FRESH|SOURCE=V2|REPLICA_COUNT=2|V2_AGE_SECONDS=33778\nREC|BACKUP_RPO_BREACH|vm9130|STATE=FRESH|SOURCE=V2|REPLICA_COUNT=2|V2_AGE_SECONDS=16454\nBACKUP_AUDIT_SUMMARY_STATUS=OK\nBACKUP_AUDIT_SUMMARY_BACKUP_STATUS=RED\nBACKUP_AUDIT_SUMMARY_ISSUE_COUNT=4\nREC|CLUSTER_GUEST_NOW|100|TYPE=qemu|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|101|TYPE=qemu|NODE=pve01|STATUS=stopped\nREC|CLUSTER_GUEST_NOW|110|TYPE=lxc|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|111|TYPE=lxc|NODE=pve02|STATUS=running\nREC|CLUSTER_GUEST_NOW|112|TYPE=lxc|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|113|TYPE=lxc|NODE=pve02|STATUS=running\nREC|CLUSTER_GUEST_NOW|130|TYPE=qemu|NODE=pve03|STATUS=running\nREC|CLUSTER_GUEST_NOW|150|TYPE=qemu|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|170|TYPE=qemu|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|171|TYPE=qemu|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|190|TYPE=qemu|NODE=pve03|STATUS=running\nREC|CLUSTER_GUEST_NOW|210|TYPE=qemu|NODE=pve01|STATUS=running\nREC|CLUSTER_GUEST_NOW|9130|TYPE=qemu|NODE=pve02|STATUS=stopped\nCEPH_ENTERPRISE_DISABLED=true\nPVEENT_ENTERPRISE_DISABLED=true\nPVE_NO_SUBSCRIPTION_ACTIVE=true\nAPT_FULL_UPGRADE_SIM_RC=0\nAPT_FULL_UPGRADE_SIM_UPGRADE_COUNT=184\nAPT_FULL_UPGRADE_SIM_REMOVE_COUNT=0\nDECISION=PASS_PVE01_FAILED_UNITS_RECOVERY_RCA\nNEXT_GATE=PVE01_FAILED_UNIT_TARGETED_RECOVERY\nTASK_RESULT=PVE01_FAILED_UNITS_RECOVERY_RCA\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 64dc92b6..e2142740 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-261006-PVE03-FINAL-READINESS-STALE-KERNEL-CLEANUP-99671R1 -STATUS=FAIL -RC=60 +COMMAND_ID=SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=pve03-final-readiness-stale-kernel-cleanup +MODE=read-only +COMPONENT=pve01-failed-units-recovery-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=24fefa15fbc73c97d999c10a09f178f009aeaef989493be4a750de0f7b7637dd +COMMAND_SHA256=9d9e9b613b9c82cf1a4117d91d95d83b00dfacba793118fdf72e1a046304d055 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true +CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=60 +COMMAND_RC=0 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,96 +24,313 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=296d9d71531385bf7a2e78264040a0ec7099683aa27040e1cf6efa453ef2bdc4 -SANITIZED_OUTPUT_SHA256=296d9d71531385bf7a2e78264040a0ec7099683aa27040e1cf6efa453ef2bdc4 +RAW_EVIDENCE_SHA256=6ac9ff0e3cb90e523d09b3a848a7d0d45af85de8cfc3e50079599ed55add7df6 +SANITIZED_OUTPUT_SHA256=6ac9ff0e3cb90e523d09b3a848a7d0d45af85de8cfc3e50079599ed55add7df6 OUTPUT_BEGIN -PVE03_FINAL_READINESS_STALE_KERNEL_CLEANUP=V1 -MODE=change +PVE01_FAILED_UNITS_RECOVERY_RCA=V1 +MODE=read-only NO_REBOOT=true -NO_AUTOREMOVE=true +NO_APT_UPDATE=true +NO_PACKAGE_MUTATION=true +NO_SERVICE_START_STOP_RESTART=true +NO_RESET_FAILED=true +NO_CONFIG_WRITE=true +NO_FILE_DELETE=true NO_GUEST_MUTATION=true +NO_BACKUP_EXECUTION=true NO_NETWORK_MUTATION=true -NO_DNS_MUTATION=true -NO_VM211_RESTORE=true -NO_BACKUP_DELETE=true -CURRENT_KERNEL_MUST_REMAIN=7.0.14-20-pve -FALLBACK_KERNEL_MUST_REMAIN=7.0.12-1-pve -PRE_CLUSTER_RC=0 -PRE_QUORATE=true -PRE_PVE01_ONLINE=true -PRE_PVE02_ONLINE=true -PRE_PVE03_ONLINE=true -PRE_VM130_RUNNING=true -PRE_VM190_RUNNING=true -PRE_VM9130_STOPPED=true -PRE_VM211_ABSENT=true -PRE_VM130_ONBOOT=1 -PRE_VM190_ONBOOT=1 -PRE_VM9130_ONBOOT=0 -PRE_VM9130_PROTECTION=1 -RUNNING_KERNEL_PRE=7.0.14-20-pve -SIGNED_KERNEL_QUERY_RC_PRE=0 -SIGNED_KERNEL_COUNT_PRE=4 -REC|SIGNED_KERNEL_PRE|proxmox-kernel-7.0.12-1-pve-signed|7.0.12-1 -REC|SIGNED_KERNEL_PRE|proxmox-kernel-7.0.14-20-pve-signed|7.0.14-20 -REC|SIGNED_KERNEL_PRE|proxmox-kernel-7.0.2-6-pve-signed|7.0.2-6 -REC|SIGNED_KERNEL_PRE|proxmox-kernel-7.0.6-2-pve-signed|7.0.6-2 -BOOT_TOOL_RC_PRE=0 -BOOT_TOOL_HAS_CURRENT_PRE=true -BOOT_TOOL_HAS_FALLBACK_PRE=true -BOOT_PIN_PRESENT_PRE=false -NEXT_BOOT_PIN_PRESENT_PRE=false -APT_SIM_RC_PRE=0 -APT_INST_COUNT_PRE=0 -APT_REMV_COUNT_PRE=0 -FAILED_UNIT_COUNT_PRE=0 -SERVICE_PVEPROXY_PRE=active -SERVICE_PVEDAEMON_PRE=active -SERVICE_PVESTATD_PRE=active -SERVICE_PVE_CLUSTER_PRE=active -SERVICE_COROSYNC_PRE=active -SERVICE_NETBIRD_PRE=active -HEALTH_SCRIPT_SHA_PRE=a01b5c63e1cfa5bc2eae4b12031de517345541c343f1b359abe832cd41e71b55 -HEALTH_NEWFI_PRE=0 -HEALTH_OTHER_FORUM_PRE=12 -HEALTH_SYNTAX_PRE=0 -HEALTH_TIMER_ACTIVE_PRE=active -HEALTH_TIMER_ENABLED_PRE=enabled -VM211_BACKUP_READY_PRE=true -VM211_BACKUP_SHA_MATCH_PRE=true -VM211_BACKUP_ZSTD_RC_PRE=0 -VM211_BACKUP_VMA_RC_PRE=0 -VM130_QGA_PING_RC=0 -VM130_QGA_NETWORK_RC=0 -VM130_DOCKER_TRANSPORT_RC=0 -VM130_DOCKER_EXITCODE=0 -VM130_NPMPLUS_TRANSPORT_RC=0 -VM130_NPMPLUS_EXITCODE=0 -VM130_ATOMIC_EDGE_PROOF=true -REC|PUBLIC_EDGE_PRE|codevipe.ru|HTTP=200|PROC_RC=0|OK=true -REC|PUBLIC_EDGE_PRE|hkmods.ru|HTTP=200|PROC_RC=0|OK=true -REC|PUBLIC_EDGE_PRE|dsmods.ru|HTTP=200|PROC_RC=0|OK=true -REC|PUBLIC_EDGE_PRE|gamevipe.ru|HTTP=200|PROC_RC=0|OK=true -REC|PUBLIC_EDGE_PRE|kingofwolk.ru|HTTP=200|PROC_RC=0|OK=true -REC|PUBLIC_EDGE_PRE|zakrutim.ru|HTTP=200|PROC_RC=0|OK=true -PUBLIC_EDGE_PRE_GOOD_COUNT=6 -PUBLIC_EDGE_PRE_ALL_OK=true -STALE_PURGE_SIM_RC=0 -STALE_PURGE_SIM_INST_COUNT=0 -STALE_PURGE_SIM_REMV_COUNT=0 -STALE_PURGE_SIM_INST_SET_SHA=NONE -STALE_PURGE_SIM_REMV_SET_SHA=NONE -STALE_PURGE_EXACT_SAFE=false -FINAL_CLEANUP_PREFLIGHT=false -DECISION=HOLD_PVE03_FINAL_CLEANUP_PREFLIGHT -NEXT_GATE=PVE03_FINAL_CLEANUP_RECONCILE -COMMANDS_REMAINING_ESTIMATE=1 -TASK_RESULT=PVE03_FINAL_READINESS_STALE_KERNEL_CLEANUP +POLICY_REPAIR_EPOCH=1791262278 +NODE_HOSTNAME=pve01 +NODE_KERNEL=7.0.12-1-pve +SYSTEMD_SYSTEM_STATE=degraded +FAILED_UNIT_COUNT=4 +REC|FAILED_UNIT_STATE|homelab-backup-audit.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791261418|EXIT_EPOCH=1791261422|EXIT_AFTER_99670=false +REC|FAILED_UNIT_FRAGMENT|homelab-backup-audit.service|BASENAME=homelab-backup-audit.service|SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa +REC|FAILED_UNIT_EXECSTART|homelab-backup-audit.service|BYTES=244|SHA256=5a28904328b20e6dbeb2f818ff18ec6005c7dbf6ed24700b96f5bd528405cd47 +REC|FAILED_UNIT_EXEC_FILE|homelab-backup-audit.service|IDX=1|BASENAME=homelab-backup-audit|BYTES=19756|SHA256=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3 +REC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-backup-audit.service|1 +REC|FAILED_UNIT_JOURNAL|homelab-backup-audit.service|LINES=180|SHA256=9bd6eea7ef8c09f5562cd04df593419e77c0732f88a14fc88b15c7c5a3164421 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|PVE_GUEST_NOT_UNIQUE|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|RPO_BREACH|27 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|BACKUP_STATUS_RED|7 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|BACKUP_STATUS_OK|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|COULD_NOT_RESOLVE|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|NAME_OR_SERVICE_NOT_KNOWN|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|CONNECTION_REFUSED|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|NO_ROUTE_TO_HOST|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TIMED_OUT|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|TIMEOUT|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|FAILED|14 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|ERROR|0 +REC|FAILED_UNIT_CLASS|homelab-backup-audit.service|STALE|27 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|1|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=14548|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|2|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm110|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|3|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|4|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|5|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|6|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|7|HOST_a0dac0c74d36: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|8|HOST_a0dac0c74d36: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|9|Failed to start HOST_a0dac0c74d36 - Homelab unified backup and DR audit. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|10|HOST_a0dac0c74d36: Consumed 2.707s CPU time, 147.5M memory peak. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|11|Starting HOST_a0dac0c74d36 - Homelab unified backup and DR audit... +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|12|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=401|generation=app-filebrowser-pve01-20261006T043010Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|13|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=239|generation=app-hbarhub-pve01-20261006T043148Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|14|TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=345|generation=app-immich-pve01-20261006T043031Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|15|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=29207|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|16|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=42881|generation=control-bootstrap-v2-pve01-20260916T120057Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|17|TARGET|logical_id=vm110|kind=VM|state=STALE|source=NONE|v2_age_seconds=83946|legacy_age_seconds=2761050|replica_count=2|state_id=ct110|generation=vm110-pve01-20260904T053821Z +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|18|TARGET|logical_id=vm111|kind=VM|state=STALE|source=NONE|v2_age_seconds=82037|replica_count=2|state_id=ct111 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|19|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=67812|legacy_age_seconds=2799545|replica_count=2|state_id=ct112|generation=v2-ct112-20261005T094532Z-ee250335d44e +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|20|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=53076|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|21|TARGET|logical_id=vm130|kind=VM|state=STALE|source=NONE|v2_age_seconds=87489|replica_count=2|state_id=vm130 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|22|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=50972|legacy_age_seconds=69392|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|23|TARGET|logical_id=vm160|kind=VM|state=STALE|source=NONE|v2_age_seconds=94473|replica_count=2|state_id=vm160 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|24|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=37543|legacy_age_seconds=1705787|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|25|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=33778|legacy_age_seconds=1735319|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|26|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=48781|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|27|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=16454|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|28|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm110|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|29|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm111|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|30|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|31|ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|32|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|33|HOST_a0dac0c74d36: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|34|HOST_a0dac0c74d36: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|35|Failed to start HOST_a0dac0c74d36 - Homelab unified backup and DR audit. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-audit.service|36|HOST_a0dac0c74d36: Consumed 2.761s CPU time, 148.4M memory peak. +REC|FAILED_UNIT_STATE|homelab-backup-v2.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=2|START_EPOCH=1791262452|EXIT_EPOCH=1791262453|EXIT_AFTER_99670=true +REC|FAILED_UNIT_FRAGMENT|homelab-backup-v2.service|BASENAME=homelab-backup-v2.service|SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565 +REC|FAILED_UNIT_EXECSTART|homelab-backup-v2.service|BYTES=238|SHA256=9560de727725d1afd2a98e905f0aad5c2b1fbb1b3348ea1fdc4aa13e5038bdae +REC|FAILED_UNIT_EXEC_FILE|homelab-backup-v2.service|IDX=1|BASENAME=homelab-backup-v2|BYTES=1656|SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +REC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-backup-v2.service|1 +REC|FAILED_UNIT_JOURNAL|homelab-backup-v2.service|LINES=180|SHA256=d9f31b02b241f88591dbad318d411219659eff0bbb7feb44dd913e38461a507e +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|PVE_GUEST_NOT_UNIQUE|8 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|RPO_BREACH|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|BACKUP_STATUS_RED|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|BACKUP_STATUS_OK|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|COULD_NOT_RESOLVE|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|NAME_OR_SERVICE_NOT_KNOWN|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|CONNECTION_REFUSED|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|NO_ROUTE_TO_HOST|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TIMED_OUT|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|TIMEOUT|0 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|FAILED|14 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|ERROR|8 +REC|FAILED_UNIT_CLASS|homelab-backup-v2.service|STALE|0 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|1| ~~~~~~~~~~~~~~^^^^^^^^^^^^ +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|2| File "PATH_REDACTED, line 873, in capture_target +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|3| if typ=='pve_guest': return capture_pve_guest(t,gen,work) +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|4| ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|5| File "PATH_REDACTED, line 192, in capture_pve_guest +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|6| if HOST_4c1ea9873d5f('guest_type')!='lxc': return original(t,gen,work) +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|7| ~~~~~~~~^^^^^^^^^^^^ +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|8| File "PATH_REDACTED, line 44, in capture_pve_guest +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|9| _need(len(rows)==1,'PVE_GUEST_NOT_UNIQUE') +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|10| ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|11| File "PATH_REDACTED, line 9, in _need +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|12| if not ok: raise QemuAdapterError(code) +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|13| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|14|qemu_adapter.QemuAdapterError: PVE_GUEST_NOT_UNIQUE +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|15|HOST_cf2099570f34: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|16|HOST_cf2099570f34: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|17|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|18|HOST_cf2099570f34: Consumed 665ms CPU time, 169.5M memory peak. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|19|Starting HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler... +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|20|SELFTEST_REPLACEMENT486=PASS +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|21|SCHEDULE_TARGET=guest-pve01-lxc-112 +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|22|SCHEDULE_RESULT=FAIL +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|23|SCHEDULE_REASON=CT112_PIPELINE_FAILED +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|24|HOST_cf2099570f34: Main process exited, code=exited, status=2/INVALIDARGUMENT +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|25|HOST_cf2099570f34: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|26|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|27|HOST_cf2099570f34: Consumed 5.092s CPU time, 173.3M memory peak. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|28|Starting HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler... +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|29|SELFTEST_REPLACEMENT486=PASS +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|30|SCHEDULE_TARGET=xf-hkmods +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|31|SCHEDULE_RESULT=FAIL +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|32|SCHEDULE_REASON=COMMAND_FAILED +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|33|HOST_cf2099570f34: Main process exited, code=exited, status=2/INVALIDARGUMENT +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|34|HOST_cf2099570f34: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|35|Failed to start HOST_cf2099570f34 - Homelab Backup V2 manifest-driven scheduler. +REC|FAILED_UNIT_SAFE_TAIL|homelab-backup-v2.service|36|HOST_cf2099570f34: Consumed 224ms CPU time, 135.4M memory peak. +REC|FAILED_UNIT_STATE|homelab-private-vpn-hosts-health.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791262774|EXIT_EPOCH=1791262776|EXIT_AFTER_99670=true +REC|FAILED_UNIT_FRAGMENT|homelab-private-vpn-hosts-health.service|BASENAME=homelab-private-vpn-hosts-health.service|SHA256=fdab069b1e8ed92423070a8c564bfd86347de3e578a728745e9d4567031b8cad +REC|FAILED_UNIT_EXECSTART|homelab-private-vpn-hosts-health.service|BYTES=466|SHA256=3fdf138482d43ae2496ff06e578fd2c142d6e555c269b94c7a640b824ad89f71 +REC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=1|BASENAME=bash|BYTES=1298416|SHA256=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f +REC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=2|BASENAME=homelab-check-private-vpn-hosts|BYTES=1411|SHA256=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82 +REC|FAILED_UNIT_EXEC_FILE|homelab-private-vpn-hosts-health.service|IDX=3|BASENAME=private-vpn-hosts.txt|BYTES=522|SHA256=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d +REC|FAILED_UNIT_EXEC_FILE_COUNT|homelab-private-vpn-hosts-health.service|3 +REC|FAILED_UNIT_JOURNAL|homelab-private-vpn-hosts-health.service|LINES=180|SHA256=ad53b326bbc251d100f99699a216e83051b3e061aa97cff85fd44f941aab9785 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|PVE_GUEST_NOT_UNIQUE|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|RPO_BREACH|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|BACKUP_STATUS_RED|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|BACKUP_STATUS_OK|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|COULD_NOT_RESOLVE|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|NAME_OR_SERVICE_NOT_KNOWN|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|CONNECTION_REFUSED|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|NO_ROUTE_TO_HOST|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TIMED_OUT|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|TIMEOUT|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|FAILED|90 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|ERROR|0 +REC|FAILED_UNIT_CLASS|homelab-private-vpn-hosts-health.service|STALE|0 +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|1|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|2|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|3|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|4|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|5|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|6|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|7|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|8|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|9|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|10|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|11|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|12|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|13|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|14|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|15|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|16|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|17|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|18|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|19|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|20|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|21|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|22|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|23|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|24|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|25|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|26|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|27|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|28|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|29|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|30|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|31|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|32|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|33|Starting HOST_bf82fe335fe6 - Homelab private VPN hosts health check... +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|34|HOST_bf82fe335fe6: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|35|HOST_bf82fe335fe6: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|homelab-private-vpn-hosts-health.service|36|Failed to start HOST_bf82fe335fe6 - Homelab private VPN hosts health check. +REC|FAILED_UNIT_STATE|skladchik-reports-monitor-supervisor.service|ACTIVE=failed|ENABLED=static|RESULT=exit-code|EXEC_STATUS=1|START_EPOCH=1791262362|EXIT_EPOCH=1791262365|EXIT_AFTER_99670=true +REC|FAILED_UNIT_FRAGMENT|skladchik-reports-monitor-supervisor.service|BASENAME=skladchik-reports-monitor-supervisor.service|SHA256=e874fb03573b76181f3eb812c7285848e60705923c172a2f102504a7b9100ef7 +REC|FAILED_UNIT_EXECSTART|skladchik-reports-monitor-supervisor.service|BYTES=272|SHA256=b3cf723f4bbe0d2362700a0c43d5de45b84332256e771a0c6b5844bbe30663e7 +REC|FAILED_UNIT_EXEC_FILE|skladchik-reports-monitor-supervisor.service|IDX=1|BASENAME=skladchik-reports-monitor-supervisor|BYTES=16903|SHA256=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2 +REC|FAILED_UNIT_EXEC_FILE_COUNT|skladchik-reports-monitor-supervisor.service|1 +REC|FAILED_UNIT_JOURNAL|skladchik-reports-monitor-supervisor.service|LINES=180|SHA256=4e6ee35c336a9b77948519e1115141083538d03dec02a4f46f30dd837d1d4d66 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|PVE_GUEST_NOT_UNIQUE|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|RPO_BREACH|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|BACKUP_STATUS_RED|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|BACKUP_STATUS_OK|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|COULD_NOT_RESOLVE|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|NAME_OR_SERVICE_NOT_KNOWN|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TEMPORARY_FAILURE_IN_NAME_RESOLUTION|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|CONNECTION_REFUSED|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|NO_ROUTE_TO_HOST|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TIMED_OUT|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|TIMEOUT|0 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|FAILED|3 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|ERROR|1 +REC|FAILED_UNIT_CLASS|skladchik-reports-monitor-supervisor.service|STALE|0 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|1|HOST_f87c67064af3: Deactivated successfully. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|2|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|3|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor... +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|4|SUPERVISOR_STATUS=OK +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|5|SUPERVISOR_MAIL_SENT=no +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|6|SUPERVISOR_PROBLEMS=none +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|7|MONITOR=OK reports=19 open=11 resolved=8 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|8|CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T203005Z +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|9|REGISTRY_SOURCE=IP_REDACTED +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|10|WORKER_TARGET=debian@IP_REDACTED +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|11|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|12|HOST_f87c67064af3: Deactivated successfully. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|13|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|14|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor... +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|15|SUPERVISOR_STATUS=OK +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|16|SUPERVISOR_MAIL_SENT=no +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|17|SUPERVISOR_PROBLEMS=none +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|18|MONITOR=OK reports=19 open=11 resolved=8 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|19|CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T203005Z +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|20|REGISTRY_SOURCE=IP_REDACTED +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|21|WORKER_TARGET=debian@IP_REDACTED +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|22|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|23|HOST_f87c67064af3: Deactivated successfully. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|24|Finished HOST_f87c67064af3 - Skladchik reports monitor full supervisor. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|25|Starting HOST_f87c67064af3 - Skladchik reports monitor full supervisor... +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|26|SUPERVISOR_STATUS=FAIL +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|27|SUPERVISOR_MAIL_SENT=yes +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|28|SUPERVISOR_PROBLEMS=registry_source_not_core_apps,canonical_backup_probe_failed,canonical_backup_probe_status_not_ok,canonical_backup_app_identity_bad,canonical_backup_not_sealed_proven,canonical_backup_generation_bad +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|29|MONITOR=OK reports=19 open=11 resolved=8 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|30|CANONICAL_BACKUP= generation= +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|31|REGISTRY_SOURCE=TARGET_REGISTRY_ERROR=REGISTRY_SCHEMA_INVALID +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|32|WORKER_TARGET=debian@IP_REDACTED +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|33|NETBIRD_ROUTE=IP_REDACTED dev wt0 table 7120 src IP_REDACTED uid 0 +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|34|HOST_f87c67064af3: Main process exited, code=exited, status=1/FAILURE +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|35|HOST_f87c67064af3: Failed with result 'exit-code'. +REC|FAILED_UNIT_SAFE_TAIL|skladchik-reports-monitor-supervisor.service|36|Failed to start HOST_f87c67064af3 - Skladchik reports monitor full supervisor. +REC|ROLE_TIMER|homelab-backup-v2.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262452|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false +REC|ROLE_TIMER|homelab-backup-audit.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791261418|LAST_AFTER_99670=false|NEXT_EPOCH=1791263168|NEXT_PRESENT=true +REC|ROLE_TIMER|homelab-private-vpn-hosts-health.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262774|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false +REC|ROLE_TIMER|skladchik-reports-monitor-supervisor.timer|ACTIVE=active|ENABLED=enabled|LAST_EPOCH=1791262362|LAST_AFTER_99670=true|NEXT_EPOCH=0|NEXT_PRESENT=false +TARGETS_SHA_NOW=21b1d1cc675e0472f05234f3cc160a43f603427a7ae6a511040ef7fb38722e5d +TARGET_COUNT_NOW=20 +VM160_TARGET_COUNT_NOW=0 +REC|BACKUP_TARGET_NOW|vm110|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm111|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm112|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm113|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm130|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm150|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm170|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm171|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm190|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm9130|KIND=VM|LIFECYCLE=ACTIVE +REC|BACKUP_TARGET_NOW|vm180|KIND=VM|LIFECYCLE=RETIRED_PRESERVE +REC|BACKUP_TARGET_NOW|vm200|KIND=VM|LIFECYCLE=RETIRED_PRESERVE +BACKUP_V2_WRAPPER_SHA=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +BACKUP_V2_AST_OK=true +REC|BACKUP_V2_SAFE_STRING|fresh_lxc_helper.py +REC|BACKUP_V2_SAFE_STRING|producer_exec_helper.py +BACKUP_AUDIT_LATEST_TARGET_COUNT=16 +BACKUP_AUDIT_RPO_BREACH_COUNT=6 +REC|BACKUP_RPO_BREACH|vm110|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=83946 +REC|BACKUP_RPO_BREACH|vm111|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=82037 +REC|BACKUP_RPO_BREACH|vm130|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=87489 +REC|BACKUP_RPO_BREACH|vm160|STATE=STALE|SOURCE=NONE|REPLICA_COUNT=2|V2_AGE_SECONDS=94473 +REC|BACKUP_RPO_BREACH|vm171|STATE=FRESH|SOURCE=V2|REPLICA_COUNT=2|V2_AGE_SECONDS=33778 +REC|BACKUP_RPO_BREACH|vm9130|STATE=FRESH|SOURCE=V2|REPLICA_COUNT=2|V2_AGE_SECONDS=16454 +BACKUP_AUDIT_SUMMARY_STATUS=OK +BACKUP_AUDIT_SUMMARY_BACKUP_STATUS=RED +BACKUP_AUDIT_SUMMARY_ISSUE_COUNT=4 +REC|CLUSTER_GUEST_NOW|100|TYPE=qemu|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|101|TYPE=qemu|NODE=pve01|STATUS=stopped +REC|CLUSTER_GUEST_NOW|110|TYPE=lxc|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|111|TYPE=lxc|NODE=pve02|STATUS=running +REC|CLUSTER_GUEST_NOW|112|TYPE=lxc|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|113|TYPE=lxc|NODE=pve02|STATUS=running +REC|CLUSTER_GUEST_NOW|130|TYPE=qemu|NODE=pve03|STATUS=running +REC|CLUSTER_GUEST_NOW|150|TYPE=qemu|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|170|TYPE=qemu|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|171|TYPE=qemu|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|190|TYPE=qemu|NODE=pve03|STATUS=running +REC|CLUSTER_GUEST_NOW|210|TYPE=qemu|NODE=pve01|STATUS=running +REC|CLUSTER_GUEST_NOW|9130|TYPE=qemu|NODE=pve02|STATUS=stopped +CEPH_ENTERPRISE_DISABLED=true +PVEENT_ENTERPRISE_DISABLED=true +PVE_NO_SUBSCRIPTION_ACTIVE=true +APT_FULL_UPGRADE_SIM_RC=0 +APT_FULL_UPGRADE_SIM_UPGRADE_COUNT=184 +APT_FULL_UPGRADE_SIM_REMOVE_COUNT=0 +DECISION=PASS_PVE01_FAILED_UNITS_RECOVERY_RCA +NEXT_GATE=PVE01_FAILED_UNIT_TARGETED_RECOVERY +TASK_RESULT=PVE01_FAILED_UNITS_RECOVERY_RCA CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null MUTATION_OUTCOME=NO_MUTATION -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE03-FINAL-READINESS-STALE-KERNEL-CLEANUP-99671R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-FAILED-UNITS-RECOVERY-RCA-99672R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END