CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260919-HOME-GRAM1-MONITOR-ENDPOINT-RCA-READONLY-571R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=home-gram1-monitor-endpoint-rca-readonly
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=631729ddd35585e56884f9d6c15bd3ff397edff0237661e6e89bbd4f0e719a28
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=0920ac33de8c85e92932b6b344333cc0698ed97758d7828911a9777da25c477d
SANITIZED_OUTPUT_SHA256=05c509364c75d70ce72bf4fc3a2a5fd441d0eed5181946007e18512e955513e7
OUTPUT_BEGIN
ERROR_REGISTER_CHECK=OK
REFERENCE_CHECK=OK
AUTHORITY_CHECK_SCOPE=READABILITY_ONLY_INCIDENTS_NOT_CLOSED
PVE1_HEALTH_FILE=actual-backup.txt|STATUS=OK TS=2026-07-09T12:53:43Z TYPE=actual-budget-backup JOB=homelab-pve01-app-actual-budget-20260709T051713Z SOURCE=latest-appbackup SIZE=2011 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-actual-budget.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=actual-budget-backup.txt|STATUS=OK TS=2026-07-09T12:53:43Z TYPE=actual-budget-backup JOB=homelab-pve01-app-actual-budget-20260709T051713Z SOURCE=latest-appbackup SIZE=2011 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-actual-budget.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=alertmanager-backup.txt|STATUS=OK TS=2026-07-09T07:50:16Z TYPE=alertmanager-backup JOB=homelab-pve01-app-alertmanager-20260709T075016Z SOURCE=alertmanager_config SIZE=1842 SHA256=58fb1b3467a962acff44bf75cb274e3a17c06afdbd3a25314a904f75c8048728 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=alertmanager-two-cloud-backup.txt|STATUS=OK TS=2026-07-10T18:46:36Z TYPE=alertmanager-two-cloud-backup BAD=0 JOB=homelab-pve01-app-alertmanager-20260710T184605Z ARCHIVE_SIZE_BYTES=1107 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK
PVE1_HEALTH_FILE=app-stacks-batch-backup.txt|STATUS=OK TS=2026-07-09T07:19:28Z TYPE=app-stacks-batch-backup SOURCE=backup-sla STATUS_SOURCE=OK NOTE=legacy-health-refresh-for-prometheus-readiness
PVE1_HEALTH_FILE=appbackupctl-routing.txt|STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z
PVE1_HEALTH_FILE=audiobookshelf-backup.txt|STATUS=OK TS=2026-07-09T18:22:24Z TYPE=audiobookshelf-backup BAD=0 APP=audiobookshelf JOB=homelab-pve01-app-audiobookshelf-20260709T051713Z SIZE=23094 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-audiobookshelf.json
PVE1_HEALTH_FILE=authentik-backup.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=backup-boot-drill.json|{ ||   "drill_executed": false, ||   "reason": "CURRENT_BACKUP_AUDIT_NOT_DR_READY",
PVE1_HEALTH_FILE=backup-coverage-matrix.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=backup-framework.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=backup-last-run.log|===== HOMELAB BACKUP HEALTHCHECK ===== || 2026-07-24T07:04:28+03:00 || 
PVE1_HEALTH_FILE=backup-lock-fd-hardening.json|{ ||   "status": "OK", ||   "root_cause_count": 0,
PVE1_HEALTH_FILE=backup-matrix-authoritative.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=unified-backup-matrix
PVE1_HEALTH_FILE=backup-matrix-cluster-desired-state-sync.txt|STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
PVE1_HEALTH_FILE=backup-matrix-cluster-final-seal.txt|STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
PVE1_HEALTH_FILE=backup-matrix-cluster-preseal-convergence.txt|STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z
PVE1_HEALTH_FILE=backup-matrix-desired-state-persistence.txt|STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z
PVE1_HEALTH_FILE=backup-matrix-vm-health-feedback-fix.txt|STATUS=OK TS=2026-07-13T17:29:02Z TYPE=backup-matrix-vm-health-feedback-fix BAD=0 VM_BACKUP_HEALTH_TIMER=homelab-edge-vm-offhost-freshness.timer OLD_TIMER_REENABLED=NO MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 HEAVY_BACKUP_RUN=NO LIVE_SCRIPT_SHA256=a795ad5cacd48ac72e85c3a7908ad456d1e7c563ef4399e6957725146463faf2 MATRIX_CONFIG_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=75f8c749e25df7842ea3be2679119b922c94c2f5ea99700b56c823cfbaedd539 ROLLBACK=/var/lib/homelab-incidents/vm-backup-health-matrix-feedback-fix-20260713T172901Z
PVE1_HEALTH_FILE=backup-observability-history.jsonl|{"clouds":[{"free_bytes":804737538995,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":294774088781},{"free_bytes":804737475136,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":294774152640}],"observed_at_epoch":1788255408,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":463952405,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":364803996,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223633529,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221489783,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10632715648,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":893057662,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3313312139,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7396575587,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2062368863,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1188660025,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1525731924,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10983689236,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":383767916,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3186,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":150356,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193527,"id":"immich
PVE1_HEALTH_FILE=backup-restore-dashboard.txt|STATUS=OK || TS=20260702T214530Z || TYPE=backup-restore-dashboard-rebuilt
PVE1_HEALTH_FILE=backup-retention-safe-gc.json|{ ||   "schema": "homelab-backup-retention-v1", ||   "status": "OK",
PVE1_HEALTH_FILE=backup-scheduler-fairness-granularity.json|{ ||   "status": "OK", ||   "root_cause_count": 0,
PVE1_HEALTH_FILE=backup-scheduler-rpo-hardening.json|{ ||   "status": "OK", ||   "root_cause_count": 0,
PVE1_HEALTH_FILE=backup-sla.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=backup-unified-audit.json|{ ||   "schema": "homelab-backup-unified-audit-v1", ||   "task_id": "backup-unified-monitoring-v1",
PVE1_HEALTH_FILE=backup.json|{ ||   "status": "OK", ||   "reason": "latest backup exists, fresh, complete, and sha256 verified",
PVE1_HEALTH_FILE=backup.txt|STATUS=OK || REASON=latest backup exists, fresh, complete, and sha256 verified || LATEST=/mnt/staging/vzdump/auto-full-backup-20260719T004537Z
PVE1_HEALTH_FILE=beszel-backup.txt|STATUS=OK TS=2026-07-09T08:38:10Z TYPE=beszel-backup JOB=homelab-pve01-app-beszel-20260709T083810Z SOURCE=beszel_stack_data SIZE=20030 SHA256=a3d7069dce01b7e7b336268cfe383dc1b0842a974bd5dc9f455442ed86c912e4 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=beszel-two-cloud-backup.txt|STATUS=OK TS=2026-07-10T19:05:25Z TYPE=beszel-two-cloud-backup BAD=0 JOB=homelab-pve01-app-beszel-20260710T190453Z ARCHIVE_SIZE_BYTES=20068 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK
PVE1_HEALTH_FILE=blackbox-exporter-backup.txt|STATUS=OK TS=2026-07-09T08:04:14Z TYPE=blackbox-exporter-backup JOB=homelab-pve01-app-blackbox-exporter-20260709T080414Z SOURCE=blackbox_exporter_config SIZE=512 SHA256=31d9f2a61d820629c0e173c0e1a24cd65e24d1ef55ba0670d7fda06225cb8146 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=blackbox-exporter-two-cloud-backup.txt|STATUS=OK TS=2026-07-10T19:20:26Z TYPE=blackbox-exporter-two-cloud-backup BAD=0 JOB=homelab-pve01-app-blackbox-exporter-20260710T191958Z ARCHIVE_SIZE_BYTES=592 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK
PVE1_HEALTH_FILE=bookstack-backup.txt|STATUS=OK TS=2026-07-09T17:58:30Z TYPE=bookstack-backup BAD=0 APP=bookstack JOB=homelab-pve01-app-bookstack-20260709T024714Z SIZE=18108 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-bookstack.json
PVE1_HEALTH_FILE=cadvisor-backup.txt|STATUS=OK TS=2026-07-09T08:25:56Z TYPE=cadvisor-backup JOB=homelab-pve01-app-cadvisor-20260709T082556Z SOURCE=cadvisor_stack_proxy SIZE=1384 SHA256=33e419912291daab19f8039dfe8971f277be0fe577eb3d0e538263bdf918eac8 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=calibre-web-backup.txt|STATUS=OK TS=2026-07-09T18:46:20Z TYPE=calibre-web-backup BAD=0 APP=calibre-web JOB=homelab-pve01-app-calibre-web-20260709T051713Z SIZE=6630 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-calibre-web.json
PVE1_HEALTH_FILE=cluster-admin-backup-webpanel-final-seal.txt|STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[PRIVATE_IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md
PVE1_HEALTH_FILE=cluster-admin-incident-engine-backup.txt|STATUS=OK TS=2026-07-10T23:30:49Z TYPE=cluster-admin-incident-engine-backup BAD=0 JOB=homelab-pve01-app-cluster-admin-incident-engine-20260710T232958Z ARCHIVE_SIZE_BYTES=157380 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK POST_FIX_CODE=included
PVE1_HEALTH_FILE=cluster-admin-stage1-preapply-backup.txt|STATUS=OK TS=2026-07-13T00:10:57Z TYPE=cluster-admin-stage1-preapply-backup BAD=0 PRODUCTION_CHANGED=BACKUP_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO ROLLBACK_POINT=VERIFIED DIRECT_DB_SNAPSHOT=VERIFIED DIRECT_DB_DUMP_SIZE_BYTES=1710825 DIRECT_DB_DUMP_SHA256=c0f13cec235ab3242eb8da4cad12962df1a2e37b25792a8ce9944c1d7de4312a APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T000543Z APPBACKUP_STATUS=success APPBACKUP_RESTORE_CHECK=OK MIGRATION_DRY_RUN=OK IDEMPOTENCE_TEST=OK ROLLBACK_TEST=OK COLLECTOR_COMPATIBILITY_TRIGGER=OK PRODUCTION_SCHEMA_UNCHANGED=YES READY_FOR_PRODUCTION_APPLY=YES DESIRED_STATE_HEAD=1379b4fb4e809e56e9bc16f0e0b19b0a509192d5 PREAPPLY_ROOT=/var/lib/homelab-stage1-preapply/20260713T000041Z REMOTE_SNAPSHOT_DIR=/var/lib/cluster-admin-preapply/20260713T000041Z EVIDENCE_MANIFEST=/var/lib/homelab-stage1-preapply/20260713T000041Z/SHA256SUMS
PVE1_HEALTH_FILE=cluster-admin-stage2-preapply-backup.txt|STATUS=OK TS=2026-07-13T01:23:18Z TYPE=cluster-admin-stage2-preapply-backup BAD=0 PREAPPLY_GENERATION=V2_DYNAMIC PRODUCTION_CHANGED=BACKUP_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO INCIDENT_ENGINE_RESTARTED=NO COLLECTOR_ONESHOT_RUN=NO CLOUD_WRITE_RUN=YES_CONTROLLED_BACKUP_ONLY ROLLBACK_POINT=VERIFIED DIRECT_DB_SNAPSHOT=VERIFIED DIRECT_DB_RESTORE_TEST=OK DIRECT_DB_DUMP_SIZE_BYTES=1818083 DIRECT_DB_DUMP_SHA256=c0fe2161bb6e61b423bc72d809bdd579ac29dc4f7bac79ba4c493bfa9f8b1d5e PRECHANGE_COLLECTOR_SHA256=a83abde2290d8392b1c631122671bcb680f9271d0b576fafacb8876d3c8034f1 CANDIDATE_COLLECTOR_SHA256=5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe FOUNDATION_COUNTS=84|21|1|227|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0 RETIRED_PLACEHOLDERS_ALLOWED=YES FIXED_TOTAL_ENTITIES_REQUIRED=NO FIXED_TOTAL_ALIASES_REQUIRED=NO APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T012320Z APPBACKUP_STATUS=success APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING_DOCKGE_UNCHANGED=YES MIGRATION_PATCH_REQUIRED=NO COLLECTOR_PATCH_REQUIRED=NO APPLY_VERIFIER_PATCH_REQUIRED=YES PREFLIGHT_TRANSACTION_ROLLBACK_REQUIRED=YES ROLLBACK_AFTER_COMMITTED_SYNC=PRESERVE_APPLIED_STATE_AND_DIAGNOSE PRODUCTION_UNCHANGED=YES OVERALL=OK DUTY_ADMIN=OK READY_FOR_DYNAMIC_PRODUCTION_APPLY=YES DESIRED_STATE_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 PREAPPLY_ROOT=/var/lib/homelab-stage2-preapply-v2/20260713T012318Z REMOTE_SNAPSHOT_DIR=/var/lib/cluster-admin-stage2-preapply-v2/20260713T012318Z CONTRACT=/var/lib/homelab-change-sets/stage2-dynamic-redryrun/20260
PVE1_HEALTH_FILE=cluster-admin-stage3-postapply-backup.txt|STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z
PVE1_HEALTH_FILE=cluster-admin-stage3-preapply-backup.txt|STATUS=OK TS=2026-07-13T03:15:54Z TYPE=cluster-admin-stage3-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=EVIDENCE_ONLY LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_EXISTING_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ROLLBACK_CHECKPOINT=VERIFIED REMOTE_CHECKPOINT_DIR=/var/lib/cluster-admin-stage3-preapply/20260713T030540Z CHECKPOINT_MANIFEST_SHA256=ede8c349a3456384f11460477d7cabc1a2aa0d18189ef3aa772471f2ebb26c77 CHECKPOINT_VALIDATION_WORKDIR=CORRECTED APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T031116Z APPBACKUP_SIZE_BYTES=1767199 APPBACKUP_RUN_LOG_EMPTY=ACCEPTED APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a PREAPPLY_ROOT=/var/lib/homelab-stage3-preapply-appbackup/20260713T031116Z READY_FOR_STAGE3_PRODUCTION_APPLY=YES
PVE1_HEALTH_FILE=cluster-admin-stage4f-preapply-backup.txt|STATUS=OK TS=2026-07-14T07:54:50Z TYPE=cluster-admin-stage4f-preapply-backup-final-seal BAD=0 STAGE4F_CLOSED=YES NEXT_STAGE=4G_PRODUCTION_APPLY_READ_ONLY_ONLY PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO APPLICATION_CHANGED=NO BACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T064327Z ARCHIVE_SIZE_BYTES=2651545 DATABASE_DUMP_SIZE_BYTES=2814735 CURRENT_JOB_VERIFY_MARKERS=2 RESTORE_LIST_CHECK=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=69be75caeecaa242567d2d06c735ac5db3a9c048 DESIRED_STATE_CLEAN=YES REPORT=/var/lib/homelab-cluster-admin-stage4/stage4f-preapply-backup-final-seal-v1/report.txt REPORT_SHA256=9ee244085b9d191370371560690ea87e7bbfe2f14226a06a4e8f6720b3a7439b
PVE1_HEALTH_FILE=cluster-admin-vm-mail-cloud-backup.txt|STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md
PVE1_HEALTH_FILE=cluster-mail-cloud-cleanup.txt|STATUS=OK TS=2026-07-11T03:23:40Z TYPE=cluster-mail-cloud-cleanup BAD=0 LOCAL_DELETED=41 LOCAL_BYTES_REMOVED=94929922668 CLOUD_A_JOBS_REMOVED=2375 CLOUD_B_JOBS_REMOVED=2375 CLOUD_A_BYTES_REMOVED=1680590103 CLOUD_B_BYTES_REMOVED=1680590103 STAGING_BYTES_REMOVED=53094954 KEEP_LOCAL=2 KEEP_APPS=7 KEEP_CRITICAL=14 RESTORE=OK ACTUAL_ALIAS=REPAIRED CLOUD_PARITY=OK
PVE1_HEALTH_FILE=control-backup.json|{ ||   "schema": "homelab-control-backup-health-v1", ||   "status": "OK",
PVE1_HEALTH_FILE=crowdsec-capi-netbird-final.txt|STATUS=OK || TS=20260702T193048Z || CHECK=crowdsec-capi-netbird-final
PVE1_HEALTH_FILE=crypto-portfolio-backup.txt|STATUS=OK TS=2026-07-13T08:12:13Z TYPE=crypto-portfolio-backup APP=crypto-portfolio JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z PATH_REL=hb/a/cp-260713-075801 ARCHIVE_SIZE_BYTES=137496 FILE_COUNT=19 ARCHIVE_SHA256=d6d3f6a397c5e73b1df0ceaeeef0f313db557b163bc6b36f32e8a084e54e975a DATABASE_INTEGRITY=ok TWO_CLOUDS=OK LOCAL_STAGING_DELETED=YES
PVE1_HEALTH_FILE=crypto-portfolio-remove-grt-prechange-backup.txt|STATUS=OK TS=2026-07-13T05:10:13Z TYPE=crypto-portfolio-remove-grt-prechange-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=ROLLBACK_SNAPSHOT_AND_BACKUP_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND SERVICE_RESTARTED=NO PRODUCTION_CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_PRECHANGE_BACKUP RETENTION_RUN=NO PRODUCTION_APPLY=NO TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 CANDIDATE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-candidate-v3/20260713T042046Z INPUT_APP_SHA256=684ad884119554503db0ed36252aeb017c0c04b54729f30c51e66a7ab293f8bd INPUT_WORKER_SHA256=40a1c81111d7097120125b140683a8604abcb4051db5073ab8af708a11854038 INPUT_INDEX_SHA256=2b1ab459cabf3050af1db4a7b293fff7c2a9ac18bf4a14faa1efc70a3a3f63e3 EXACT_LOCAL_ROLLBACK=OK ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z ROLLBACK_MANIFEST_SHA256=6cc505f7e13a02b0ddfc9fd3a9700903d7aaf70d8c8ae3f630a4fea69aeb4b1f ROLLBACK_DATABASE_INTEGRITY=ok ROLLBACK_GRT_ROWS=1|1|729 TWO_CLOUD_BACKUP=OK BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z BACKUP_ARCHIVE_SIZE_BYTES=172776 BACKUP_FILE_COUNT=19 BACKUP_DATABASE_INTEGRITY=ok BACKUP_LOCAL_STAGING_DELETED=YES RESTORE_CHECK=OK_BOTH_CLOUDS PRODUCTION_SOURCE_HASHES_UNCHANGED=YES PRODUCTION_CONTAINER_IDS_AND_RESTARTS_UNCHANGED=YES PRODUCTION_DATABASE_INTEGRITY=ok PRODUCTION_GRT_STATE_PRESERVED=YES PUBLIC_PRODUCTION_STATE_PRESERVED=YES READY_FOR_GRT_PRODUCTION_APPLY=YES PRECHANGE_ROOT=/var/lib/homelab-change-sets/crypto-portfoli
PVE1_HEALTH_FILE=crypto-portfolio-ui-preapply-backup.txt|STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_
PVE1_HEALTH_FILE=deep-service-backup-audit.txt|STATUS=OK TS=2026-07-09T21:12:44Z TYPE=deep-service-backup-audit BAD=0 SERVICE_READINESS_OK_COUNT=47 NON_OK=0 APPBACKUP_FLAG_GAPS=0 CONTAINER_GAPS=0 PROM_FIRING=0 BACKUP_SLA=OK ALERTING=OK EXTERNAL_LINKS_CLASSIFIED=OK SERVICES_REMAINING_TOTAL=0 BACKUP_GAPS_TOTAL=0 COMMANDS_REMAINING_TOTAL=0
PVE1_HEALTH_FILE=dockge-backup.txt|STATUS=OK TS=2026-07-09T10:12:44Z TYPE=dockge-backup JOB=homelab-pve01-app-dockge-stacks-20260709T072218Z SOURCE=appbackup-dockge-stacks SIZE=134913 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-dockge.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=dockge-stacks-backup.txt|STATUS=OK TS=2026-07-09T10:12:44Z TYPE=dockge-stacks-backup JOB=homelab-pve01-app-dockge-stacks-20260709T072218Z SOURCE=latest-appbackup SIZE=134913 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-dockge-stacks.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=dozzle-backup.txt|STATUS=OK TS=2026-07-09T08:32:09Z TYPE=dozzle-backup JOB=homelab-pve01-app-dozzle-20260709T083209Z SOURCE=dozzle_stack_config SIZE=1328 SHA256=a571ee6f6012b25c16cdf15693dba29d8dd3acc3ce872b52f5c34edb81aa1ec8 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=edge-vm-vzdump-backup.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=extended-appbackup.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=filebrowser-backup.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=gitea-backup.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=gotify-backup.txt|STATUS=OK TS=2026-07-09T20:39:29Z TYPE=gotify-backup BAD=0 APP=gotify JOB=homelab-pve01-app-gotify-20260709T072202Z SIZE=44431 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-gotify.json
PVE1_HEALTH_FILE=grafana-backup.txt|STATUS=OK TS=2026-07-09T07:46:00Z TYPE=grafana-backup JOB=homelab-pve01-app-grafana-20260709T074600Z SOURCE=grafana_sqlite_backup_plus_provisioning SIZE=129929 SHA256=582537912f306666f04d9e855491f7275bdb31b39b4c9b0156444a297f8c1641 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=hapusya-site-backup-path-mapping.txt|STATUS=OK TS=2026-07-13T12:10:58Z TYPE=hapusya-site-backup-path-mapping BAD=0 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 SHORT=hps LEGACY_STATE_PATH=pve01/apps/hapusya-site/homelab-pve01-app-hapusya-site-20260713T115259Z DESIRED_STATE_SNAPSHOT=/srv/homelab-desired-state/config/homelab-backup-paths.map RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK PRODUCTION_CHANGED=YES_COMMIT_POINT_PRESERVED
PVE1_HEALTH_FILE=hapusya-site-backup.txt|STATUS=OK TS=2026-07-13T12:10:58Z TYPE=hapusya-site-backup JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 ARCHIVE_SIZE_BYTES=7895834 LOCAL_BACKUP_DELETED=true TWO_CLOUDS=OK ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb
PVE1_HEALTH_FILE=hapusya-site-preapply-backup.txt|STATUS=OK TS=2026-07-13T11:03:06Z TYPE=hapusya-site-preapply-backup BAD=0 NPMPLUS_JOB=homelab-pve01-app-npmplus-edge-20260713T110238Z TWO_CLOUDS=OK RESTORE_CHECK=OK ROLLBACK=/var/lib/homelab-incidents/hapusya-site-preapply-20260713T110305Z ROLLBACK_MANIFEST_SHA256=6b4020425c5d6c581c48a5dc95b20d54fd7906468e4ba8d27c0f0325e2df81a3 CANDIDATE_BROWSER=OK BROWSER_RUNS=4 PRODUCTION_CHANGED=NO
PVE1_HEALTH_FILE=hapusya-site-v2-preapply-backup.txt|STATUS=OK TS=2026-07-13T11:34:59Z TYPE=hapusya-site-v2-preapply-backup BAD=0 NPMPLUS_JOB=homelab-pve01-app-npmplus-edge-20260713T113435Z TWO_CLOUDS=OK RESTORE_CHECK=OK ROLLBACK=/var/lib/homelab-incidents/hapusya-site-v2-preapply-20260713T113458Z ROLLBACK_MANIFEST_SHA256=144506c766fe9fd3152e4edf94a8867895b3dc54f29faf1e329773922b8c5640 CANDIDATE_TREE_SHA256=0eb098f91ea706c884d36a8ce806968e82e715ded66bb96f61c2ed8106faca93 CANDIDATE_BROWSER=OK BROWSER_RUNS=5 VIDEO_DESKTOP=OK VIDEO_MOBILE=OK STATIC_FALLBACK=OK PRODUCTION_CHANGED=NO
PVE1_HEALTH_FILE=healthchecks-backup.txt|STATUS=OK TS=2026-07-09T08:51:29Z TYPE=healthchecks-backup JOB=homelab-pve01-app-healthchecks-20260709T051713Z SOURCE=latest-appbackup SIZE=687 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-healthchecks.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=homeassistant-backup.txt|STATUS=OK TS=2026-07-09T19:04:00Z TYPE=homeassistant-backup BAD=0 APP=homeassistant JOB=homelab-pve01-app-homeassistant-20260709T051713Z SIZE=1855722 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-homeassistant.json
PVE1_HEALTH_FILE=homebox-backup.txt|STATUS=OK TS=2026-07-09T16:01:19Z TYPE=homebox-backup JOB=homelab-pve01-app-homebox-20260709T051713Z SOURCE=latest-appbackup SIZE=8873 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-homebox.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=homelab-backup-baseline.txt|STATUS=OK || SERVICE=homelab-backup-baseline || TIMESTAMP_UTC=20260617T200213Z
PVE1_HEALTH_FILE=homelab-mailru-architecture-smoke.txt|STATUS=OK || TS=20260702T085400Z || CHECK=homelab-mailru-architecture-smoke
PVE1_HEALTH_FILE=immich-backup.txt|STATUS=OK TS=2026-07-09T20:09:12Z TYPE=immich-backup BAD=0 APP=immich JOB=homelab-pve01-app-immich-20260709T072002Z SIZE=328392286 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-immich.json MEDIA_BACKUP_HEALTH=/var/lib/homelab-health/immich-media-backup.txt
PVE1_HEALTH_FILE=immich-media-backup.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=it-tools-backup.txt|STATUS=OK TS=2026-07-09T19:13:00Z TYPE=it-tools-backup BAD=0 APP=it-tools JOB=homelab-pve01-app-it-tools-20260709T191300Z SIZE=689 LOCAL_BACKUP_DELETED=true ENGINE=files JSON=/var/lib/homelab-backup/apps/latest-it-tools.json
PVE1_HEALTH_FILE=jellyfin-backup.txt|STATUS=OK TS=2026-07-09T18:16:16Z TYPE=jellyfin-backup BAD=0 APP=jellyfin JOB=homelab-pve01-app-jellyfin-20260709T051713Z SIZE=25920 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-jellyfin.json
PVE1_HEALTH_FILE=karakeep-backup.txt|STATUS=OK TS=2026-07-09T19:28:34Z TYPE=karakeep-backup BAD=0 APP=karakeep JOB=homelab-pve01-app-karakeep-20260709T051713Z SIZE=23140 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-karakeep.json
PVE1_HEALTH_FILE=legacy-monitoring-backups-migrated.txt|STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK
PVE1_HEALTH_FILE=linkding-backup.txt|STATUS=OK TS=2026-07-09T17:00:18Z TYPE=linkding-backup BAD=0 APP=linkding JOB=homelab-pve01-app-linkding-20260709T051713Z SIZE=40380 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-linkding.json
PVE1_HEALTH_FILE=mail-cloud-canary.txt|STATUS=OK TS=20260618T174444Z TYPE=mail-cloud-canary REMOTE_TOTAL=13 REMOTE_OK=13 REMOTE_FAIL=0 DEST=restore-test/p1-canary/20260618T174444Z SHA256=a6bf6a1493a2556b92c8e6547163500345d4e0e32688937370b942b881c47182 SECRET_PRINTED=no NOTES=none
PVE1_HEALTH_FILE=mail-cloud-critical.txt|STATUS=OK TS=20260724T034138Z TYPE=mail-cloud-critical MODE=recurring REMOTES=2 REMOTE_OK=2 REMOTE_FAIL=0 DEST=app/p0-critical-baseline/20260724T034138Z BUNDLE=p0-critical-baseline-20260724T034138Z.tar.gz SIZE=973272634 SHA256=91745ef4a64996ed2c2d6ab7523b336d552b4177f76fa93be2eabefa8d58f8b4 RETENTION_KEEP=14 SECRET_PRINTED=no NOTES=none
PVE1_HEALTH_FILE=mail-cloud-edge-vm-controlled-no-retention-mode.txt|STATUS=OK TS=2026-07-13T18:27:22Z TYPE=mail-cloud-edge-vm-controlled-no-retention-mode BAD=0 MODE=--upload-no-retention RETENTION_ENABLED=no PURGE_GUARDED=YES WHITESPACE_SAFE_HEALTH_PATCH=YES LOCAL_INVALID_MODE_TEST=OK LOCAL_TRANSPORT_GATE_TEST=OK EDGE_UPLOAD_RUN=NO CLOUD_READ_RUN=NO CLOUD_WRITE_RUN=NO CLOUD_PURGE_RUN=NO RETENTION_RUN=NO MATRIX_COVERAGE_ADDED=NO DESIRED_STATE_COMMIT=ae28be231f154fd42f9c39de6eebbb4a570fb84f SCRIPT_SHA256=f22848d83a7ce13ca331ce9181523d0a8fadcf34f01d0c95186784c6b1f0875f ROLLBACK_ROOT=/var/lib/homelab-change-sets/mail-cloud-edge-vm-controlled-no-retention-mode-whitespace-safe/20260713T182722Z
PVE1_HEALTH_FILE=mail-cloud-edge-vm-controlled-upload.txt|STATUS=OK TS=2026-07-13T18:39:22Z TYPE=mail-cloud-edge-vm-controlled-upload BAD=0 VMID=130 SOURCE=/mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-2026_07_13-03_30_01.vma.zst SIZE=10942350973 SHA256=6979558850813d99881b20c559600ec1c8748cc4f59f18ef9875eee6e40c6757 REMOTE=pve03-mail-03-crypt TARGET_BASE=full/edge-vm-vzdump FOUND_DIR=20260713T183122Z REQUEST_MODE=--upload-no-retention RETENTION_MODE=disabled RETENTION_DELETE_COUNT=0 SPLIT_OK=YES UPLOAD_OK=YES DOWNLOAD_VERIFY=YES SHA_MATCH=YES SIZE_MATCH=YES PART_COUNT_MATCH=YES CLOUD_WRITE_RUN=YES CLOUD_PURGE_RUN=NO RETENTION_RUN=NO SOURCE_DELETE_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/mail-cloud-edge-vm-controlled-upload/20260713T183056Z
PVE1_HEALTH_FILE=mail-cloud-edge-vm-final-seal.txt|STATUS=OK TS=2026-07-14T01:02:58Z TYPE=mail-cloud-edge-vm-final-seal BAD=0 WARN=0 VMID=130 SOURCE=/mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-2026_07_13-03_30_01.vma.zst SIZE=10942350973 SHA256=6979558850813d99881b20c559600ec1c8748cc4f59f18ef9875eee6e40c6757 CLOUD_FOUND_DIR=20260713T183122Z MATRIX_JOB_ID=homelab_mail_cloud_edge_vm_upload MATRIX_STATUS=OK JOBS_TOTAL=36 JOBS_OK=32 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 UPLOAD_VERIFIED=YES DOWNLOAD_VERIFY=YES SHA_MATCH=YES SIZE_MATCH=YES PART_COUNT_MATCH=YES CONTROLLED_UPLOAD_MODE=--upload-no-retention RETENTION_MODE=disabled RETENTION_RUN=NO RETENTION_DELETE_COUNT=0 CLOUD_PURGE_RUN=NO SOURCE_DELETE_RUN=NO EDGE_SERVICE_RESULT=success EDGE_EXEC_MAIN_STATUS=0 EDGE_TIMER_ENABLED=enabled EDGE_TIMER_ACTIVE=active CLUSTER_ADMIN_STATUS=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK WEBPANEL_SYNC=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=98dfaeb35683e18cd4e986d0121bde60daac1e7c CURRENT_EDGE_VM_CLOUD_SUBSTAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/mail-cloud-edge-vm-final-seal-idle-safe/20260714T010226Z
PVE1_HEALTH_FILE=mail-cloud-edge-vm-matrix-registration.txt|STATUS=OK TS=2026-07-13T18:56:30Z TYPE=mail-cloud-edge-vm-matrix-registration BAD=0 MATRIX_JOB_ID=homelab_mail_cloud_edge_vm_upload CRITICAL=YES EVIDENCE_MAPPING=INGESTED_FROM_ORIGINAL_HEALTH MATRIX_STATUS=OK JOBS_TOTAL=36 JOBS_OK=32 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 TIMER_ENABLED=YES TIMER_ACTIVE=YES SERVICE_RESULT=success EXEC_MAIN_STATUS=0 SOURCE_STATUS=OK FRESHNESS=FRESH CONTROLLED_UPLOAD=VERIFIED RETENTION_RUN=NO CLOUD_PURGE_RUN=NO DESIRED_STATE_HEAD=98dfaeb35683e18cd4e986d0121bde60daac1e7c MATRIX_CONFIG_SHA256=f40be069d560f7c1e1dcf16575215d77fea86530bbd7732d4c1b3feb562d11d6 MATRIX_STATE_SHA256=524a6fac53ec39d82257ba78cff639bd0ac2f04eddbc3c5208a2f4609e0a3298 MAIN_HEALTH_SHA256=114707f6becdb26e84bde36a2e2ab6e9c36bd0955f6d1b1cf7851544fdd7ff87 ROLLBACK_ROOT=/var/lib/homelab-change-sets/mail-cloud-edge-vm-matrix-registration-count-gate-fixed/20260713T185629Z
PVE1_HEALTH_FILE=mail-cloud-edge-vm-readonly-probe.txt|STATUS=OK TS=2026-07-13T18:15:43Z TYPE=mail-cloud-edge-vm-readonly-probe BAD=0 RESULT=OK VMID=130 SOURCE=/mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-2026_07_13-03_30_01.vma.zst SIZE=10942350973 SHA256=6979558850813d99881b20c559600ec1c8748cc4f59f18ef9875eee6e40c6757 REMOTE=pve03-mail-03-crypt TARGET_BASE=full/edge-vm-vzdump FOUND_DIR=none CLOUD_STATE=CURRENT_ARCHIVE_NOT_PRESENT CLOUD_READ_RUN=YES CLOUD_WRITE_RUN=NO CLOUD_MKDIR_RUN=NO CLOUD_COPY_RUN=NO CLOUD_PURGE_RUN=NO RETENTION_RUN=NO MAIN_HEALTH_CHANGED=NO MATRIX_CHANGED=NO DESIRED_STATE_CHANGED=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/mail-cloud-edge-vm-readonly-probe/20260713T181517Z
PVE1_HEALTH_FILE=mail-cloud-edge-vm-runtime-contract-fix.txt|STATUS=OK TS=2026-07-13T18:13:01Z TYPE=mail-cloud-edge-vm-runtime-contract-fix BAD=0 LIVE_DESIRED_IDENTICAL=YES REMOTE_ENV_EXISTING_CONTRACT=OK EARLY_HEALTH_WRITES=YES TRANSPORT_GATE=YES SOURCE_CONTRACT=OK EDGE_UPLOAD_RUN=NO CLOUD_PROBE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO MATRIX_COVERAGE_ADDED=NO DESIRED_STATE_COMMIT=f7dc97b342077214753b3a227a9969db64c1f65e SCRIPT_SHA256=90f67fbee2a512fd1d6e381a32db54879f594fb85084a27994e4a2ffa02ccb0a ROLLBACK_ROOT=/var/lib/homelab-change-sets/mail-cloud-edge-vm-remaining-runtime-contract-fix/20260713T181300Z
PVE1_HEALTH_FILE=mail-cloud-edge-vm-service-convergence.txt|STATUS=OK TS=2026-07-13T18:40:21Z TYPE=mail-cloud-edge-vm-service-convergence BAD=0 SERVICE_RESULT=success EXEC_MAIN_STATUS=0 CURRENT_SHA_FOUND=YES FOUND_DIR=20260713T183122Z ALREADY_UPLOADED_BRANCH=YES UPLOAD_PARTS_RUN=NO RETENTION_RUN=NO CLOUD_PURGE_RUN=NO MAIN_HEALTH_SOURCE=EXECUTED_CONTROLLED_NO_RETENTION EVIDENCE_ROOT=/var/lib/homelab-change-sets/mail-cloud-edge-vm-controlled-upload/20260713T183056Z
PVE1_HEALTH_FILE=mail-cloud-edge-vm.txt|STATUS=OK TS=20260719T063229Z TYPE=mail-cloud-edge-vm MODE=recurring-chunked REQUEST_MODE=run RETENTION_MODE=enabled REMOTE=pve03-mail-03-crypt DEST=full/edge-vm-vzdump/20260719T063229Z ARCHIVE=vzdump-qemu-130-2026_07_19-03_30_03.vma.zst SIZE=11013723404 SHA256=3d47e8d9c8137131fd96a3538e4284a2df60d7941c5e68ad1926ad9f89b15253 PART_SIZE=512M PARTS=21 ALREADY_UPLOADED=no DOWNLOAD_VERIFY=yes RETENTION_KEEP=4 SECRET_PRINTED=no NOTES=none
PVE1_HEALTH_FILE=mail-cloud-restore-drill.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=mealie-backup.txt|STATUS=OK TS=2026-07-09T16:17:31Z TYPE=mealie-backup JOB=homelab-pve01-app-mealie-20260709T051713Z SOURCE=latest-appbackup SIZE=912487 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-mealie.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=memos-backup.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=minio-backup.txt|STATUS=OK TS=2026-07-09T12:13:14Z TYPE=minio-backup JOB=homelab-pve01-app-minio-20260709T051713Z SOURCE=latest-appbackup SIZE=4585 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-minio.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=minio-console-backup.txt|STATUS=OK TS=2026-07-09T12:13:14Z TYPE=minio-backup JOB=homelab-pve01-app-minio-20260709T051713Z SOURCE=latest-appbackup SIZE=4585 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-minio.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=n8n-backup.txt|STATUS=OK TS=2026-07-09T19:39:23Z TYPE=n8n-backup BAD=0 APP=n8n JOB=homelab-pve01-app-n8n-20260709T051713Z SIZE=92667 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-n8n.json
PVE1_HEALTH_FILE=netbird-old-usa-poweroff-observation.txt|STATUS=OBSERVING || START_EPOCH=1787365981 || START_UTC=2026-08-22T02:33:01Z
PVE1_HEALTH_FILE=netbird-peers.txt|STATUS=OK || SERVICE=netbird-peers || TIMESTAMP_UTC=20260918T223532Z
PVE1_HEALTH_FILE=netbird-vps-backup.txt|STATUS=OK || SERVICE=netbird-vps-backup || TIMESTAMP_UTC=20260701T013054Z
PVE1_HEALTH_FILE=netbird-vps-offhost.txt|STATUS=OK || SERVICE=netbird-vps-offhost || TIMESTAMP_UTC=20260717T015941Z
PVE1_HEALTH_FILE=netbird-vps-restore.txt|STATUS=OK || SERVICE=netbird-vps-restore-validation || TIMESTAMP_UTC=20260617T075428Z
PVE1_HEALTH_FILE=netbird-vps-trust-clean-seal.txt|STATUS=OK TS=2026-07-07T15:55:43Z TYPE=netbird-vps-trust-clean-seal BAD=0 WARN=0 MODE=CLEAN_SEAL_NO_AWK_TAIL TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP=100.100.60.182 PUBLIC_VPS_IP=188.127.235.6 PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH INDEPENDENT_RUNNER_AVAILABLE=no DEPLOY_DECISION=NO_DEPLOY KNOWN_HOSTS_CHANGED=no PRIVATE_KEYS_READ=no PROBE_DEPLOYED=no REPORT=/var/lib/homelab-netbird-vps-trust/clean-seal-latest.md
PVE1_HEALTH_FILE=netbird-vps-trust-closure.txt|STATUS=OK TS=2026-07-07T15:29:49Z TYPE=netbird-vps-trust-closure BAD=0 WARN=0 MODE=TRUST_CLOSURE_NO_KNOWN_HOSTS_CHANGE TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP=100.100.60.182 PUBLIC_VPS_IP=188.127.235.6 PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH PUBLIC_IP_STRICT_BLOCKED=YES PUBLIC_IP_KEYSCAN_MATCH=NO EDGE_REF_OK=1 NETBIRD_PEER_OK=1 REPORT=/var/lib/homelab-netbird-vps-trust/closure-latest.md
PVE1_HEALTH_FILE=netbird-vps-trust.txt|STATUS=OK || TS=2026-07-17T04:52:15Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=netbox-backup.txt|STATUS=OK TS=2026-07-09T10:23:50Z TYPE=netbox-backup JOB=homelab-pve01-app-netbox-20260709T072101Z SOURCE=latest-appbackup SIZE=25860637 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-netbox.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=nextcloud-restore-proof-offhost.txt|STATUS=OK TS=20260624T190330Z TYPE=nextcloud-restore-proof-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge/nextcloud-aio/nextcloud-aio-restore-proof/20260624T120344Z ITEMS=1 OK=1 ERROR=0 BYTES=2135684 FILES=7 SECRET_PRINTED=NO
PVE1_HEALTH_FILE=node-exporter-backup.txt|STATUS=OK TS=2026-07-09T08:13:40Z TYPE=node-exporter-backup JOB=homelab-pve01-app-node-exporter-20260709T081340Z SOURCE=multi_host_node_exporter_config_textfile SIZE=19018 SHA256=bb7db78e6e79c104f89c7735922e2a2e311fba242d34728e64cd60cefe69e647 LOCAL_BACKUP_DELETED=true PVE01_TEXTFILE_REPAIR=OK NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=node-red-backup.txt|STATUS=OK TS=2026-07-09T19:46:42Z TYPE=node-red-backup BAD=0 APP=node-red JOB=homelab-pve01-app-node-red-20260709T051713Z SIZE=10521 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-node-red.json
PVE1_HEALTH_FILE=npmplus-backup.txt|STATUS=OK TS=2026-07-09T10:02:52Z TYPE=npmplus-edge-backup JOB=homelab-pve01-app-npmplus-edge-20260709T071928Z SOURCE=latest-appbackup SIZE=50316 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-npmplus-edge.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=npmplus-edge-backup.txt|STATUS=OK TS=2026-07-09T10:02:52Z TYPE=npmplus-edge-backup JOB=homelab-pve01-app-npmplus-edge-20260709T071928Z SOURCE=latest-appbackup SIZE=50316 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-npmplus-edge.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=npmplus-kuma-config-backup.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=ntfy-backup.txt|STATUS=OK TS=2026-07-09T20:39:29Z TYPE=ntfy-backup BAD=0 APP=ntfy JOB=homelab-pve01-app-ntfy-20260709T051713Z SIZE=13516 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-ntfy.json
PVE1_HEALTH_FILE=p1-safe-data-backup.txt|STATUS=OK TS=2026-07-06T22:07:44Z TYPE=p1-safe-data-backup BAD=0 OK=7 EXPECTED=7 REPORT=/var/lib/homelab-p1-safe/latest.md ENV_FILES_READ=NO SECRET_PRINTED=NO CONSISTENCY=emergency_data_dir_archive
PVE1_HEALTH_FILE=p2-media-backup.txt|STATUS=OK TS=2026-07-07T00:56:48Z TYPE=p2-media-backup BAD=0 OK=3 EXPECTED=3 REPORT=/var/lib/homelab-p2-media/latest.md ENV_FILES_READ=NO SECRET_PRINTED=NO CONSISTENCY=emergency_config_data_archive_corrected_var_lib_paths BULK_MEDIA_POLICY=excluded_selected_bulk_media
PVE1_HEALTH_FILE=paperless-backup.txt|STATUS=OK || TS=2026-07-17T22:52:29Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=proxmox-pve01-backup.txt|STATUS=OK TS=2026-07-09T12:29:38Z TYPE=proxmox-pve01-backup JOB=homelab-pve01-app-proxmox-pve01-20260709T122938Z SOURCE=safe_pve01_config_facts SIZE=4314 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-proxmox-pve01.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=proxmox-pve01-offhost.txt|STATUS=OK TS=2026-07-09T12:29:38Z TYPE=proxmox-pve01-offhost JOB=homelab-pve01-app-proxmox-pve01-20260709T122938Z HOST=pve02 DIR=/mnt/staging/offhost/proxmox-pve01-backups-from-pve01 SHA256_CHECK=OK NOTE=service-readiness-offhost-proof
PVE1_HEALTH_FILE=proxmox-pve01-restore.txt|STATUS=OK TS=2026-07-09T12:29:38Z TYPE=proxmox-pve01-restore JOB=homelab-pve01-app-proxmox-pve01-20260709T122938Z SOURCE=local-tar-restore RESTORECHECK=OK_SAFE_FILES NOTE=service-readiness-restore-proof
PVE1_HEALTH_FILE=pve01-backup.txt|STATUS=OK TS=2026-07-09T12:29:38Z TYPE=proxmox-pve01-backup JOB=homelab-pve01-app-proxmox-pve01-20260709T122938Z SOURCE=safe_pve01_config_facts SIZE=4314 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-proxmox-pve01.json NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=pve01-nextcloud-dns-restore-drill.txt|STATUS=OK TS=2026-07-08T15:20:12Z TYPE=pve01-nextcloud-dns-restore-drill BAD=0 MODE=CLOUD_VM_RESTORE_QGA_AND_DNS_SECONDARY_PROOF SOURCE_REMOTE=pve01-mail-01-crypt CLOUD_JOB=20260708T043700Z VMID_SOURCE=150 VMID_TEST=9150 DOWNLOAD_OK=YES SHA_OK=YES ZSTD_OK=YES VMA_VERIFY_OK=YES QMRESTORE_OK=YES QGA_OK=YES LINK_DOWN=YES DNS_SECONDARY_OK=YES PRODUCTION_OK=YES TEST_VM_CLEANED=YES STAGING_CLEANED=YES REPORT=/var/lib/homelab-node-loss/pve01-nextcloud-dns-drill/latest.md
PVE1_HEALTH_FILE=router-running-config-mail-cloud.txt|STATUS=OK TS=20260724T021528Z TYPE=router-running-config-mail-cloud MODE=recurring REMOTE=pve01-mail-02-crypt DEST=router/running-config/20260724T021528Z COMMAND=show_running_config CONFIG_SIZE_BYTES=18951 LOOKS_CONFIG=yes ARCHIVE=/var/lib/homelab-private/backups/router-running-config/router-running-config-20260724T021528Z.tar.gz ARCHIVE_SIZE_BYTES=3899 ARCHIVE_SHA256=7a4d86c0977007eea8de1098163a03b443b5066cc31024c610960068dac8ec8a SSH_RC=0 TAR_CREATE_OK=yes SHA_OK=yes UPLOAD_OK=yes DOWNLOAD_VERIFY=yes RETENTION_KEEP=14 DELETE_COUNT=1 SECRET_FILE=/var/lib/homelab-private/secrets/routerbackup.pass SECRET_PRINTED=no NOTES=none
PVE1_HEALTH_FILE=router-startup-config-mail-cloud.txt|STATUS=OK TS=2026-07-09T07:19:28Z TYPE=router-startup-config-mail-cloud MODE=manual SOURCE=existing_archive_and_remote_verified LOCAL_ARCHIVE_PRESENT=1 REMOTE_OBJECTS_PRESENT=1 NOTE=legacy-health-refresh-for-prometheus-readiness
PVE1_HEALTH_FILE=searxng-backup.txt|STATUS=OK TS=2026-07-09T09:39:15Z TYPE=searxng-backup JOB=homelab-pve01-app-searxng-20260709T093915Z SOURCE=searxng_stack_config_cache SIZE=738 SHA256=352c9c54437b8ceabc491452d1d7150624eda8c4bf99c28dc78fd488ec06f28c LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
PVE1_HEALTH_FILE=service-readiness-backup-health.txt|STATUS=OK TS=2026-07-09T06:41:40Z TYPE=service-readiness-final BAD=0 SERVICE=backup-health HOMEPAGE=OK NPMPLUS=OK KUMA=OK ENDPOINTS=LOCAL_LAN_PUBLIC_OK CORE_GATES=OK LISTENERS=OK URL=https://backup.gram1.ru/backup.txt OVERALL=https://backup.gram1.ru/overall.txt
PVE1_HEALTH_FILE=service-readiness-proxmox-pve01.txt|STATUS=OK TS=2026-07-09T12:33:46Z TYPE=service-readiness-final BAD=0 SERVICE=proxmox-pve01 HOMEPAGE=OK NPMPLUS=OK KUMA=OK HEALTH=OK QUORUM=OK CLUSTER_NODES=OK STORAGE=OK GUESTS=OK UI=OK API_401=OK PUBLIC=OK CERTS=OK BACKUP=OK OFFHOST=OK RESTORE=OK BACKUP_SLA=OK ALERTING=OK RUNBOOK=OK LOGS=OK URL=https://proxmox.vpn.gram1.ru STATE=pveproxy:active,pvedaemon:active,pvestatd:active,cluster:quorate
PVE1_HEALTH_FILE=stirling-pdf-backup.txt|STATUS=OK TS=2026-07-09T18:08:38Z TYPE=stirling-pdf-backup BAD=0 APP=stirling-pdf JOB=homelab-pve01-app-stirling-pdf-20260709T180838Z SIZE=40071 LOCAL_BACKUP_DELETED=true ENGINE=files JSON=/var/lib/homelab-backup/apps/latest-stirling-pdf.json
PVE1_HEALTH_FILE=syncthing-backup.txt|STATUS=OK TS=2026-07-09T19:55:12Z TYPE=syncthing-backup BAD=0 APP=syncthing JOB=homelab-pve01-app-syncthing-20260709T051713Z SIZE=6732 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-syncthing.json
PVE1_HEALTH_FILE=uptime-kuma-backup.txt|STATUS=OK TS=2026-07-09T20:39:29Z TYPE=uptime-kuma-backup BAD=0 APP=uptime-kuma JOB=homelab-pve01-app-uptime-kuma-20260709T072138Z SIZE=99851064 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-uptime-kuma.json
PVE1_HEALTH_FILE=vaultwarden-backup.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=vikunja-backup.txt|STATUS=OK TS=2026-07-09T17:51:14Z TYPE=vikunja-backup BAD=0 APP=vikunja JOB=homelab-pve01-app-vikunja-20260709T051713Z SIZE=34831 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-vikunja.json
PVE1_HEALTH_FILE=vm-backup-policy.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
PVE1_HEALTH_FILE=vm-backup.txt|STATUS=OK TS=2026-07-24T04:43:04Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active
PVE1_HEALTH_FILE=vm160-remote-backup.txt|STATUS=FAIL || SERVICE=homelab-vm160-remote-backup || TIMESTAMP_UTC=20260720T233913Z
PVE1_HEALTH_FILE=vm170-vm171-full-backup.txt|STATUS=FAIL || TS=2026-08-30T18:08:08Z || TYPE=backup-matrix-generated
CORE_VM170_RC=0
CORE_VM170_BEGIN
{
   "exitcode" : 0,
   "exited" : 1,
   "out-data" : "CORE_HOST=core-apps\nEDGE_HEALTH_BASE=http://[PRIVATE_IP]:18091\nCORE_CONTAINER_PROBE=PVE01_BACKUP_LOCAL|URL=http://[PRIVATE_IP]:9101/backup.txt|RC=2|RESULT=|ERR=TimeoutError:The operation was aborted due to timeout \nCORE_CONTAINER_PROBE=PVE01_BACKUP_DASH|URL=http://[PRIVATE_IP]:9101/backup-restore-dashboard.txt|RC=2|RESULT=|ERR=TimeoutError:The operation was aborted due to timeout \nCORE_CONTAINER_PROBE=PVE01_PROXMOX_READINESS|URL=http://[PRIVATE_IP]:9101/service-readiness-proxmox-pve01.txt|RC=2|RESULT=|ERR=TimeoutError:The operation was aborted due to timeout \nCORE_CONTAINER_PROBE=NEWFILYA_PUBLIC|URL=https://newfilya.ru/|RC=0|RESULT=200|https://newfilya.ru/|ERR=\nCORE_CONTAINER_PROBE=EDGE_HEALTH_ROUTER_MOSCOW|URL=http://[PRIVATE_IP]:18091/router-moscow|RC=0|RESULT=200|http://[PRIVATE_IP]:18091/router-moscow|ERR=\nCORE_CONTAINER_PROBE=EDGE_HEALTH_NPMPLUS|URL=http://[PRIVATE_IP]:18091/npmplus|RC=0|RESULT=200|http://[PRIVATE_IP]:18091/npmplus|ERR=\nCORE_CONTAINER_PROBE=EDGE_HEALTH_NETBIRD|URL=http://[PRIVATE_IP]:18091/netbird|RC=0|RESULT=404|http://[PRIVATE_IP]:18091/netbird|ERR=\nCORE_CONTAINER_PROBE=EDGE_HEALTH_NEXTCLOUD|URL=http://[PRIVATE_IP]:18091/nextcloud|RC=0|RESULT=404|http://[PRIVATE_IP]:18091/nextcloud|ERR=\nCORE_CONTAINER_PROBE=EDGE_HEALTH_PROXMOX|URL=http://[PRIVATE_IP]:18091/proxmox|RC=0|RESULT=404|http://[PRIVATE_IP]:18091/proxmox|ERR=\nCORE_CONTAINER_PROBE=EDGE_HEALTH_BACKUP_HEALTH|URL=http://[PRIVATE_IP]:18091/backup-health|RC=0|RESULT=404|http://[PRIVATE_IP]:18091/backup-health|ERR=\nCORE571_END\n",
   "out-truncated" : 0
}

CORE_VM170_END
EDGE_VM130_RC=0
EDGE_VM130_BEGIN
{
   "exitcode" : 0,
   "exited" : 1,
   "out-data" : "EDGE_HOST=edge-vm\nEDGE_HEALTH_SCRIPT_EXISTS=True\nEDGE_HEALTH_SCRIPT_SHA256=ccf25de98494625cd88015411c82c9be2d4349970d6d4c6c353c45f787225225\nEDGE_HEALTH_SCRIPT_SIZE=6704\nEDGE_HEALTH_SCRIPT_SAFE_BEGIN\n#!/usr/bin/env python3\nimport json\nimport socket\nimport ssl\nimport subprocess\nimport time\nimport urllib.error\nimport urllib.request\nfrom http.server import BaseHTTPRequestHandler, HTTPServer\n\nHOST = \"[PRIVATE_IP]\"\nPORT = 18091\n\ndef tcp(host, port, timeout=3):\n    try:\n        with socket.create_connection((host, int(port)), timeout=timeout):\n            return True\n    except Exception:\n        return False\n\ndef docker_running(name):\n    try:\n        out = subprocess.check_output([\"docker\", \"inspect\", \"-f\", \"{{.State.Running}}\", name], text=True, timeout=5)\n        return out.strip() == \"true\"\n    except Exception:\n        return False\n\ndef http_probe(url, timeout=5, ok_statuses=None, accept_2xx_4xx=False):\n    result = {\"url\": url, \"ok\": False, \"reachable\": False, \"status\": None, \"error\": None}\n    try:\n        ctx = ssl._create_unverified_context()\n        req = urllib.request.Request(url, headers={\"User-Agent\": \"homelab-health/1.0\"})\n        try:\n            with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:\n                result[\"status\"] = int(getattr(r, \"status\", 0) or 0)\n                result[\"reachable\"] = True\n        except urllib.error.HTTPError as e:\n            result[\"status\"] = int(e.code)\n            result[\"reachable\"] = True\n\n        status = int(result[\"status\"] or 0)\n        if ok_statuses is not None:\n            result[\"ok\"] = status in set(ok_statuses)\n        elif accept_2xx_4xx:\n            result[\"ok\"] = 200 <= status < 500\n        else:\n            result[\"ok\"] = 200 <= status < 400\n    except Exception as e:\n        result[\"error\"] = type(e).__name__ + \": \" + str(e)[:160]\n    return result\n\ndef response(service, ok, checks, note=\"\"):\n    return {\"service\": service, \"ok\": bool(ok), \"checks\": checks, \"note\": note, \"ts\": int(time.time())}\n\ndef router_status():\n    checks = {\n        \"lan_http_5080\": http_probe(\"http://[PRIVATE_IP]:5080/\", accept_2xx_4xx=True),\n        \"lan_https_5083\": http_probe(\"https://[PRIVATE_IP]:5083/\", accept_2xx_4xx=True),\n        \"public_https_5083\": http_probe(\"https://aleisaevn.netcraze.pro:5083/\", accept_2xx_4xx=True),\n        \"tcp_egress_1_1_1_1_443\": {\"ok\": tcp(\"1.1.1.1\", 443), \"host\": \"1.1.1.1\", \"port\": 443},\n    }\n    ok = checks[\"lan_http_5080\"][\"ok\"] or checks[\"lan_https_5083\"][\"ok\"] or checks[\"public_https_5083\"][\"ok\"]\n    return response(\"router-moscow-health\", ok, checks, \"HTTP 401/403 from router UI is accepted as reachable.\")\n\ndef npmplus_status():\n    checks = {\n        \"container_running\": {\"ok\": docker_running(\"npmplus\")},\n        \"tcp_127_0_0_1_80\": {\"ok\": tcp(\"127.0.0.1\", 80)},\n        \"tcp_127_0_0_1_443\": {\"ok\": tcp(\"127.0.0.1\", 443)},\n        \"tcp_127_0_0_1_81\": {\"ok\": tcp(\"127.0.0.1\", 81)},\n    }\n    return response(\"npmplus-health\", all(v.get(\"ok\") for v in checks.values()), checks)\n\ndef loki_status():\n    checks = {\n        \"container_running\": {\"ok\": docker_running(\"loki\")},\n        \"ready\": http_probe(\"http://127.0.0.1:3100/ready\", ok_statuses={200}),\n    }\n    return response(\"loki-health\", checks[\"container_running\"][\"ok\"] and checks[\"ready\"][\"ok\"], checks)\n\ndef alloy_status():\n    checks = {\n        \"container_running\": {\"ok\": docker_running(\"alloy\")},\n        \"tcp_12345\": {\"ok\": tcp(\"127.0.0.1\", 12345)},\n        \"ready\": http_probe(\"http://127.0.0.1:12345/-/ready\", ok_statuses={200}),\n    }\n    ok = checks[\"container_running\"][\"ok\"] and (checks[\"ready\"][\"ok\"] or checks[\"tcp_12345\"][\"ok\"])\n    return response(\"alloy-health\", ok, checks)\n\ndef cadvisor_status():\n    checks = {\n        \"container_running\": {\"ok\": docker_running(\"cadvisor\")},\n        \"tcp_8081\": {\"ok\": tcp(\"127.0.0.1\", 8081)},\n    }\n    return response(\"cadvisor-health\", all(v.get(\"ok\") for v in checks.values()), checks)\n\ndef registry_status():\n    checks = {\n        \"container_running\": {\"ok\": docker_running(\"homelab-registry-cache\")},\n        \"v2\": http_probe(\"http://[PRIVATE_IP]:5000/v2/\", ok_statuses={200, 401}),\n    }\n    return response(\"registry-cache-health\", checks[\"container_running\"][\"ok\"] and checks[\"v2\"][\"ok\"], checks)\n\ndef socket_proxy_status():\n    checks = {\n        \"container_running\": {\"ok\": docker_running(\"socket-proxy\")},\n        \"ping\": http_probe(\"http://127.0.0.1:2375/_ping\", ok_statuses={200}),\n    }\n    return response(\"socket-proxy-health\", checks[\"container_running\"][\"ok\"] and checks[\"ping\"][\"ok\"], checks)\n\ndef diun_status():\n    checks = {\"container_running\": {\"ok\": docker_running(\"diun\")}}\n    return response(\"diun-health\", checks[\"container_running\"][\"ok\"], checks)\n\ndef kopia_status():\n    checks = {\"container_running\": {\"ok\": docker_running(\"kopia\")}}\n    return response(\"kopia-health\", checks[\"container_running\"][\"ok\"], checks)\n\ndef crowdsec_status():\n    checks = {\n        \"container_running\": {\"ok\": docker_running(\"crowdsec\")},\n        \"lapi_tcp_8088\": {\"ok\": tcp(\"127.0.0.1\", 8088)},\n        \"metrics_tcp_7422\": {\"ok\": tcp(\"127.0.0.1\", 7422)},\n    }\n    return response(\"crowdsec-local-health\", checks[\"container_running\"][\"ok\"] and checks[\"lapi_tcp_8088\"][\"ok\"], checks)\n\nROUTES = {\n    \"/\": router_status,\n    \"/router-moscow\": router_status,\n    \"/router-moscow/\": router_status,\n    \"/npmplus\": npmplus_status,\n    \"/npmplus/\": npmplus_status,\n    \"/loki\": loki_status,\n    \"/loki/\": loki_status,\n    \"/alloy\": alloy_status,\n    \"/alloy/\": alloy_status,\n    \"/cadvisor\": cadvisor_status,\n    \"/cadvisor/\": cadvisor_status,\n    \"/registry-cache\": registry_status,\n    \"/registry-cache/\": registry_status,\n    \"/socket-proxy\": socket_proxy_status,\n    \"/socket-proxy/\": socket_proxy_status,\n    \"/diun\": diun_status,\n    \"/diun/\": diun_status,\n    \"/kopia\": kopia_status,\n    \"/kopia/\": kopia_status,\n    \"/crowdsec\": crowdsec_status,\n    \"/crowdsec/\": crowdsec_status,\n}\n\nclass Handler(BaseHTTPRequestHandler):\n    def log_message(self, fmt, *args):\n        return\n\n    def do_GET(self):\n        fn = ROUTES.get(self.path)\n        if not fn:\n            body = b\"not found\\n\"\n            self.send_response(404)\n            self.send_header(\"Content-Type\", \"text/plain\")\n            self.send_header(\"Content-Length\", str(len(body)))\n            self.end_headers()\n            self.wfile.write(body)\n            return\n\n        status = fn()\n        body = json.dumps(status, ensure_ascii=False, sort_keys=True).encode(\"utf-8\")\n        self.send_response(200 if status[\"ok\"] else 503)\n        self.send_header(\"Content-Type\", \"application/json; charset=utf-8\")\n        self.send_header(\"Cache-Control\", \"no-store\")\n        self.send_header(\"Content-Length\", str(len(body)))\n        self.end_headers()\n        self.wfile.write(body)\n\nif __name__ == \"__main__\":\n    HTTPServer((HOST, PORT), Handler).serve_forever()\nEDGE_HEALTH_SCRIPT_SAFE_END\nEDGE_PROBE=NEXTCLOUD_PUBLIC|URL=https://nc.gram1.ru/|RC=0|RESULT=200|[PRIVATE_IP]|https://nc.gram1.ru/|ERR=\nEDGE_PROBE=NETBIRD_PUBLIC|URL=https://nb.pvepro.ru/|RC=0|RESULT=200|46.16.34.129|https://nb.pvepro.ru/|ERR=\nEDGE_PROBE=NEWFILYA_PUBLIC|URL=https://newfilya.ru/|RC=0|RESULT=200|185.225.35.6|https://newfilya.ru/|ERR=\nEDGE_PROBE=PVE01_API|URL=https://[PRIVATE_IP]:8006/api2/json/version|RC=28|RESULT=000||https://[PRIVATE_IP]:8006/api2/json/version|ERR=curl: (28) Connection timed out after 10002 milliseconds \nEDGE_PROBE=BACKUP_PUBLIC|URL=https://backup.gram1.ru/backup.txt|RC=28|RESULT=000|[PRIVATE_IP]|https://backup.gram1.ru/backup.txt|ERR=curl: (28) Operation timed out after 10001 milliseconds with 0 bytes received \nEDGE_PROBE=PVE01_BACKUP_LOCAL|URL=http://[PRIVATE_IP]:9101/backup.txt|RC=28|RESULT=000||http://[PRIVATE_IP]:9101/backup.txt|ERR=curl: (28) Connection timed out after 10002 milliseconds \nEDGE_PROBE=PVE01_BACKUP_DASH|URL=http://[PRIVATE_IP]:9101/backup-restore-dashboard.txt|RC=28|RESULT=000||http://[PRIVATE_IP]:9101/backup-restore-dashboard.txt|ERR=curl: (28) Connection timed out after 10002 milliseconds \nEDGE_PROBE=PVE01_PROXMOX_READINESS|URL=http://[PRIVATE_IP]:9101/service-readiness-proxmox-pve01.txt|RC=28|RESULT=000||http://[PRIVATE_IP]:9101/service-readiness-proxmox-pve01.txt|ERR=curl: (28) Connection timed out after 10002 milliseconds \nEDGE571_END\n",
   "out-truncated" : 0
}

EDGE_VM130_END
HOME571R1_DECISION=MONITOR_ENDPOINT_RCA_COMPLETE
HOME571R1_NEXT_GATE=APPLY_MINIMAL_MONITOR_AND_CARD_REPAIR
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260919-HOME-GRAM1-MONITOR-ENDPOINT-RCA-READONLY-571R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}

OUTPUT_END
CHAT_OUTPUT_END
