runtime: publish FORUM-260923-REPAIR-ERROR-ALERT6F2BR1
This commit is contained in:
homelab-runtime-publisher
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"channel": "homelab-runtime",
|
||||
"command_id": "FORUM-260923-REPAIR-ERROR-ALERT6F2BR1",
|
||||
"status": "FAIL",
|
||||
"rc": 3,
|
||||
"host": "pve01",
|
||||
"mode": "change",
|
||||
"component": "forum-error-alert",
|
||||
"started_at_utc": "2026-09-23T21:10:56Z",
|
||||
"finished_at_utc": "2026-09-23T21:11:02Z",
|
||||
"reference_register_checked": true,
|
||||
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
|
||||
"error_register_checked": true,
|
||||
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
|
||||
"command_sha256": "75e2d85aade37899d9174d3a1e02df965f4a07157d05e9c6f8b6b75dfdfa9828",
|
||||
"duplicate_failed_command_blocked": false,
|
||||
"block_reason": null,
|
||||
"execution_started": true,
|
||||
"change_declared": true,
|
||||
"result_contract_valid": true,
|
||||
"result_contract_status": "FAIL",
|
||||
"result_contract_error": null,
|
||||
"command_rc": 3,
|
||||
"changes_made": true,
|
||||
"rollback_started": true,
|
||||
"rollback_restored": true,
|
||||
"mutation_outcome": "ROLLED_BACK",
|
||||
"sanitized": true,
|
||||
"secrets_included": false,
|
||||
"private_addresses_included": false,
|
||||
"raw_evidence_retained_locally": true,
|
||||
"raw_evidence_sha256": "a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2",
|
||||
"sanitized_output_sha256": "a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2",
|
||||
"output_truncated_in_json": false,
|
||||
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
||||
"output": "CID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1\nMODE=change\nCOMPONENT=forum-error-alert\nMUTATION_BOUNDARY=PVE01_FORUM_HEALTH_ADD_KINGOFWOLK;VM160_NEW_XF_ERROR_ALERT_SCRIPT_SERVICE_TIMER_STATE;ONE_CANARY_EMAIL;NO_XF_DB_WRITE;NO_XF_ERROR_DELETE;NO_MAIL_TRANSPORT_RECONFIG;NO_CLOUD;NO_VM_CONFIG\nEXECUTION_STARTED=true\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nHEALTH_PRE_SHA256=d7cee34b7d0e93c578f9203ba6e3634d304fee3ea5f3629a576dd683f1b50b29\nPVE02_TRANSPORT=PASS\nCHANGES_MADE=true\nPVE01_HEALTH_PATCHED=true\nHEALTH_POST_SHA256=4ee0c432fd906ba2d73198c965d0e57e4e7001781d61161b64f3c51625f034bd\nPVE02_VM160_STATE=running\nPVE02_QGA_TRANSPORT_RC=0\nVM160_GUEST_EXITCODE=90\nGUEST_CHANGE_BEGIN=true\nGUEST_FAILURE=MONITOR_PREFLIGHT\nGUEST_ROLLBACK_STARTED=true\nGUEST_ROLLBACK_RESTORED=true\nVM160_GUEST_STDERR_PRESENT=true\nVM160_GUEST_STDERR_SHA256=fd6627b9e2d9e0eca04003d62a4416ea377f7de34a5afb1800512ed9cf9094bd\nVM160_GUEST_STDERR_BYTES=94\nFAILURE=VM160_ALERT_INSTALL_OR_CANARY\nROLLBACK_STARTED=true\nROLLBACK_RESTORED=true\nDECISION=FAIL_REPAIR_ROLLED_BACK\nNEXT_GATE=REVIEW_FAILURE\nTASK_RESULT=FAIL_FORUM_ERROR_ALERT_REPAIR\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"FORUM-260923-REPAIR-ERROR-ALERT6F2BR1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":true,\"rollback_restored\":true}\n"
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1
|
||||
STATUS=FAIL
|
||||
RC=3
|
||||
HOST=pve01
|
||||
MODE=change
|
||||
COMPONENT=forum-error-alert
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=75e2d85aade37899d9174d3a1e02df965f4a07157d05e9c6f8b6b75dfdfa9828
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=true
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=FAIL
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=3
|
||||
CHANGES_MADE=true
|
||||
ROLLBACK_STARTED=true
|
||||
ROLLBACK_RESTORED=true
|
||||
MUTATION_OUTCOME=ROLLED_BACK
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2
|
||||
SANITIZED_OUTPUT_SHA256=a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2
|
||||
OUTPUT_BEGIN
|
||||
CID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1
|
||||
MODE=change
|
||||
COMPONENT=forum-error-alert
|
||||
MUTATION_BOUNDARY=PVE01_FORUM_HEALTH_ADD_KINGOFWOLK;VM160_NEW_XF_ERROR_ALERT_SCRIPT_SERVICE_TIMER_STATE;ONE_CANARY_EMAIL;NO_XF_DB_WRITE;NO_XF_ERROR_DELETE;NO_MAIL_TRANSPORT_RECONFIG;NO_CLOUD;NO_VM_CONFIG
|
||||
EXECUTION_STARTED=true
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
HEALTH_PRE_SHA256=d7cee34b7d0e93c578f9203ba6e3634d304fee3ea5f3629a576dd683f1b50b29
|
||||
PVE02_TRANSPORT=PASS
|
||||
CHANGES_MADE=true
|
||||
PVE01_HEALTH_PATCHED=true
|
||||
HEALTH_POST_SHA256=4ee0c432fd906ba2d73198c965d0e57e4e7001781d61161b64f3c51625f034bd
|
||||
PVE02_VM160_STATE=running
|
||||
PVE02_QGA_TRANSPORT_RC=0
|
||||
VM160_GUEST_EXITCODE=90
|
||||
GUEST_CHANGE_BEGIN=true
|
||||
GUEST_FAILURE=MONITOR_PREFLIGHT
|
||||
GUEST_ROLLBACK_STARTED=true
|
||||
GUEST_ROLLBACK_RESTORED=true
|
||||
VM160_GUEST_STDERR_PRESENT=true
|
||||
VM160_GUEST_STDERR_SHA256=fd6627b9e2d9e0eca04003d62a4416ea377f7de34a5afb1800512ed9cf9094bd
|
||||
VM160_GUEST_STDERR_BYTES=94
|
||||
FAILURE=VM160_ALERT_INSTALL_OR_CANARY
|
||||
ROLLBACK_STARTED=true
|
||||
ROLLBACK_RESTORED=true
|
||||
DECISION=FAIL_REPAIR_ROLLED_BACK
|
||||
NEXT_GATE=REVIEW_FAILURE
|
||||
TASK_RESULT=FAIL_FORUM_ERROR_ALERT_REPAIR
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"FORUM-260923-REPAIR-ERROR-ALERT6F2BR1","status":"FAIL","changes_made":true,"rollback_started":true,"rollback_restored":true}
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+18
-18
File diff suppressed because one or more lines are too long
+45
-205
@@ -1,222 +1,62 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-ACTIVE-DEPS-RECONCILE-1132R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
COMMAND_ID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1
|
||||
STATUS=FAIL
|
||||
RC=3
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-moldova-active-deps-reconcile
|
||||
MODE=change
|
||||
COMPONENT=forum-error-alert
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=8d956c461cae8f6370656cb989564264780767a2e0ae02073cc7210302312eca
|
||||
COMMAND_SHA256=75e2d85aade37899d9174d3a1e02df965f4a07157d05e9c6f8b6b75dfdfa9828
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
CHANGE_DECLARED=true
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_STATUS=FAIL
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=0
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
COMMAND_RC=3
|
||||
CHANGES_MADE=true
|
||||
ROLLBACK_STARTED=true
|
||||
ROLLBACK_RESTORED=true
|
||||
MUTATION_OUTCOME=ROLLED_BACK
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=85e3392765d3a95b9a1be4ad77e1d2d9c767b493fedfff9b4e785d1177c95daf
|
||||
SANITIZED_OUTPUT_SHA256=85e3392765d3a95b9a1be4ad77e1d2d9c767b493fedfff9b4e785d1177c95daf
|
||||
RAW_EVIDENCE_SHA256=a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2
|
||||
SANITIZED_OUTPUT_SHA256=a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_MOLDOVA_ACTIVE_DEPS_RECONCILE_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-ACTIVE-DEPS-RECONCILE-1132R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-moldova-active-deps-reconcile
|
||||
MUTATION_BOUNDARY=NONE;READ_ONLY_RECONCILE_ONLY;NO_PROVIDER_CALL;NO_MOLDOVA_SSH;NO_NETWORK_MUTATION;NO_SYSTEMD_ACTION;NO_DNS_CHANGE;NO_BACKUP_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT
|
||||
PURPOSE=CLASSIFY_MD_RELAY_ACTIVE_POLICY_MONITORING_AND_BACKUP_AUDIT_BLOCKERS_BEFORE_ANY_DETACH
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||
AUTHORITY_GATE=PASS
|
||||
FRESH_CONTEXT_BEGIN=1
|
||||
NOW_UTC=2026-09-23T21:07:51Z
|
||||
HOSTNAME=pve01
|
||||
FRESH_CONTEXT_END=1
|
||||
MD_RELAY_TARGET_POLICY_BEGIN=1
|
||||
TARGETS_SHA256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b
|
||||
TARGETS_PARSE=true
|
||||
MATCH_COUNT=2
|
||||
TARGET_SAFE={"due_seconds":86400,"enabled":true,"green1":true,"host_present":true,"host_sha256":"731a154cebb02c619b63295a7316e8484883299059110ff4200455a92aa396fd","hostkey_present":true,"hostkey_sha256":"21e3a92d98f47c5f79187d8fb6a55d04160b5f0460952d2e332dfeff78bcbcc6","logical_id":"md-relay","port":22,"scope":"vps","type":"vps_md","user":"ops"}
|
||||
TARGET_SAFE={"due_seconds":86400,"enabled":false,"green1":true,"host_present":true,"host_sha256":"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16","hostkey_present":true,"hostkey_sha256":"b67f1e81ebf030d7fd26ca5e2dc238c158bc4ad6e603cd6cc27588bee1d0628f","logical_id":"us-netbird","port":22,"scope":"vps","type":"vps_us","user":"root"}
|
||||
MD_RELAY_TARGET_POLICY_END=1
|
||||
MD_RELAY_RUNTIME_STATE_BEGIN=1
|
||||
STATE_REF|path=/var/lib/homelab-backup-v2/state/md-relay.json|mtime=2026-09-23 03:38:55.856821284 +0300|sha256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76
|
||||
1:{"consecutive_failures":0,"failure_backoff_seconds":0,"generation":"v2-md-relay-20260923T003832Z-8051ad720cce","green_verified":false,"last_attempt_epoch":1790123935,"last_success_epoch":1790123935,"logical_id":"md-relay","manifest_sha256":"b57ca8ee55d8622f6aeb044d3063fd0bb7cb6ada83f50511c87894d083a2a428","payload_sha256":"0d65b5ba587ec9f393e7a54af1df2e86b54332509da617d6bfe594b42d5fa3f8","replicas":[{"crypt":"pve03-mail-01-crypt","physical":"pve03-mail-01"},{"crypt":"pve01-mail-04-crypt","physical":"pve01-mail-04"}],"result":"SUCCESS","retry_after_epoch":0,"schema":1,"updated_at_utc":"2026-09-23T00:38:55Z"}
|
||||
STATE_REF|path=/var/lib/homelab-health/crowdsec-capi-netbird-final.txt|mtime=2026-07-02 22:30:48.260143650 +0300|sha256=20b03407e972e39c3a8b7e8ca008bf4a4b6928ed3e0e9a822be886c3cd63fae0
|
||||
1:STATUS=OK
|
||||
8:PRIMARY_EXIT=relay.netbird.selfhosted
|
||||
STATE_REF|path=/var/lib/homelab-health/homelab-backup-baseline.txt|mtime=2026-06-17 23:02:13.952559036 +0300|sha256=4959757d29bb503315c60468dc0eb080b43cefde23d48e8ab192f6bd93b564d3
|
||||
1:STATUS=OK
|
||||
10:ENDPOINT_SUMMARY=disk-space.txt:OK;immich-media-backup.txt:OK;immich-media-offhost.txt:OK;memos-backup.txt:OK;memos-offhost.txt:OK;paperless-backup.txt:OK;paperless-offhost.txt:OK;netbird-vps-backup.txt:OK;netbird-vps-offhost.txt:OK;netbird-vps-restore.txt:OK;netbird-peers.txt:OK;filebrowser-backup.txt:OK;filebrowser-offhost.txt:OK;filebrowser-restore.txt:OK;app-stacks-batch-backup.txt:OK;app-stacks-batch-offhost.txt:OK;app-stacks-batch-restore.txt:OK;
|
||||
11:KUMA_MONITORS=1|Homelab Backup Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/backup.txt;24|Immich Media Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-backup.txt;25|Immich Media Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-offhost.txt;26|Disk Space Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/disk-space.txt;27|Memos Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-backup.txt;28|Memos Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-offhost.txt;29|Paperless Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-backup.txt;30|Paperless Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-offhost.txt;31|NetBird VPS Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-backup.txt;32|NetBird VPS Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-offhost.txt;33|NetBird VPS Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-restore.txt;34|NetBird Homelab Peers|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-peers.txt;35|File Browser Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-backup.txt;36|File Browser Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-offhost.txt;37|File Browser Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-restore.txt;38|App Stacks Batch Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-backup.txt;39|App Stacks Batch Offhost|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-offhost.txt;40|App Stacks Batch Restore|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-restore.txt;
|
||||
STATE_REF|path=/var/lib/homelab-health/homepage-homelab-router-audit/homepage-homelab-router-audit-core-apps-live-homepage.txt|mtime=2026-07-02 23:09:46.546385105 +0300|sha256=a19529d0a5ce2242ce362bd02bcb0be586dd82964203f7cba38396106a6b886e
|
||||
1:REMOTE_STATUS=REVIEW
|
||||
47: 7 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z
|
||||
119:SCRIPT_CANDIDATE=/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z
|
||||
120:-rw-r--r-- 1 root root 8819 Jun 30 23:35 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z
|
||||
121:76506f002c640c7d6460fd16961df4ee1b884238a8cea7a945f6e8a2c5ec6dd4 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z
|
||||
276:REMOTE_STATUS=OK_772D_DONE
|
||||
STATE_REF|path=/var/lib/homelab-health/homepage-homelab-router-audit/homepage-homelab-router-audit-edge-vm.txt|mtime=2026-07-02 23:09:35.781222820 +0300|sha256=40c927fbe0d53452a205a4cdc97e6a52272e6430c0f4a0217ac188ef9393a399
|
||||
1:REMOTE_STATUS=REVIEW
|
||||
48: 11 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z
|
||||
348:SCRIPT_CANDIDATE=/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z
|
||||
349:-rw-r--r-- 1 root root 8819 Jun 30 23:35 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z
|
||||
350:76506f002c640c7d6460fd16961df4ee1b884238a8cea7a945f6e8a2c5ec6dd4 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z
|
||||
499:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083</p></div>
|
||||
565:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083</p></div>
|
||||
631:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083</p></div>
|
||||
697:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083</p></div>
|
||||
784:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083</p></div>
|
||||
1006:homelab-router-moscow-health-http.service enabled enabled
|
||||
1007:router-moscow-health.service enabled enabled
|
||||
1023:NPMPLUS_ROUTE id=8 domain_names=["home.gram1.ru"] forward_scheme=http forward_host=[IP] forward_port=3000 enabled=1 ssl_forced=1 access_list_id=1 advanced_config= locations=[]
|
||||
1024:NPMPLUS_ROUTE id=17 domain_names=["homepage.vpn.gram1.ru"] forward_scheme=http forward_host=[IP] forward_port=3000 enabled=1 ssl_forced=1 access_list_id=1 advanced_config= locations=[]
|
||||
1026:REMOTE_STATUS=OK_772D_DONE
|
||||
STATE_REF|path=/var/lib/homelab-health/netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e
|
||||
1:STATUS=OK
|
||||
2:SERVICE=netbird-vps-backup
|
||||
5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z
|
||||
6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest
|
||||
STATE_REF|path=/var/lib/homelab-health/netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b
|
||||
1:STATUS=OK
|
||||
5:DST=/mnt/staging/offhost/netbird-vps-backups-from-pve01
|
||||
STATE_REF|path=/var/lib/homelab-health/netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb
|
||||
1:STATUS=OK
|
||||
5:ARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz
|
||||
STATE_REF|path=/var/lib/homelab-health/netbird-vps-trust.txt|mtime=2026-07-18 01:54:49.485000918 +0300|sha256=643c858c4b1cfa77a7914349ccb05434a94e27a358320b687658684665d5c1ac
|
||||
1:STATUS=OK
|
||||
5:JOB_NAME=netbird-vps-backup
|
||||
7:TIMER=netbird-vps-backup.timer
|
||||
8:SERVICE=netbird-vps-backup.service
|
||||
9:TIMER_ENABLED=enabled
|
||||
14:SERVICE_EXEC_MAIN_STATUS=0
|
||||
16:SOURCE_STATUS=OK
|
||||
20:FRESHNESS=FRESH
|
||||
23:CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
||||
MD_RELAY_RUNTIME_STATE_END=1
|
||||
BACKUP_AUDIT_RECONCILE_BEGIN=1
|
||||
UNIT|name=homelab-backup-v2.timer|load=loaded|active=active|sub=waiting|result=success
|
||||
UNIT|name=homelab-backup-v2.service|load=loaded|active=inactive|sub=dead|result=success
|
||||
UNIT|name=homelab-backup-audit.timer|load=loaded|active=active|sub=waiting|result=success
|
||||
UNIT|name=homelab-backup-audit.service|load=loaded|active=failed|sub=failed|result=exit-code
|
||||
BACKUP_HEAVY_LOCK_NOW=FREE
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[1770869]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[1770869]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[1874934]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[1874934]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[1968089]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[1968089]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2071373]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2071373]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2164434]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2164434]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2655306]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2655306]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
Sep 23 [IPV6] pve01 homelab-backup-audit[2655306]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
Sep 24 [IPV6] pve01 homelab-backup-audit[2759058]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
Sep 24 [IPV6] pve01 homelab-backup-audit[2759058]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
Sep 24 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
||||
Sep 24 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
||||
BACKUP_AUDIT_MD_RELAY_DIRECT_ISSUE=false
|
||||
BACKUP_AUDIT_RECONCILE_END=1
|
||||
NETBIRD_LOCAL_RECONCILE_BEGIN=1
|
||||
NETBIRD_STATUS_JSON_RC=0
|
||||
NETBIRD_JSON_PARSE=true
|
||||
NETBIRD_DAEMON_STATUS=Connected
|
||||
NETBIRD_PEER|fqdn=relay.netbird.selfhosted|status=Connecting|connection=-
|
||||
NETBIRD_PEER|fqdn=edge-vm.netbird.selfhosted|status=Connecting|connection=-
|
||||
NETBIRD_PEER|fqdn=mail.netbird.selfhosted|status=Connecting|connection=-
|
||||
NETBIRD_NETWORKS_LIST_RC=0
|
||||
NETBIRD_ROUTES_LIST_RC=0
|
||||
NETBIRD_LOCAL_RECONCILE_END=1
|
||||
MONITORING_RECONCILE_BEGIN=1
|
||||
KUMA_POLICY_SHA256=87e1f14bde3e5eb2d0512c949e1e32d7fe3214f38797559b6e3e1a9d855f059a
|
||||
KUMA_LIVE_QUERY_ATTEMPT=true
|
||||
KUMA_LIVE_MONITOR|31|NetBird VPS Backup|1|keyword|http://[IP]:9101/netbird-vps-backup.txt
|
||||
KUMA_LIVE_MONITOR|32|NetBird VPS Offhost Backup|1|keyword|http://[IP]:9101/netbird-vps-offhost.txt
|
||||
KUMA_LIVE_MONITOR|33|NetBird VPS Restore Validation|1|keyword|http://[IP]:9101/netbird-vps-restore.txt
|
||||
KUMA_LIVE_MONITOR|34|NetBird Homelab Peers|1|keyword|http://[IP]:9101/netbird-peers.txt
|
||||
KUMA_LIVE_MONITOR|77|NetBird Dashboard|1|http|https://nb.pvepro.ru/
|
||||
MONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e
|
||||
1:STATUS=OK
|
||||
2:SERVICE=netbird-vps-backup
|
||||
4:VPS_IP=[IP]
|
||||
5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z
|
||||
6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest
|
||||
MONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b
|
||||
1:STATUS=OK
|
||||
2:SERVICE=netbird-vps-offhost
|
||||
5:DST=/mnt/staging/offhost/netbird-vps-backups-from-pve01
|
||||
MONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb
|
||||
1:STATUS=OK
|
||||
2:SERVICE=netbird-vps-restore-validation
|
||||
5:ARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz
|
||||
9:ROOT_NETBIRD_ENTRIES=7
|
||||
MONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-peers.txt|mtime=2026-09-24 00:03:22.325368652 +0300|sha256=dd1843335b572b28ec3b1c081478853841d6d8acf7209ab09522860d2cefd8cc
|
||||
1:STATUS=ERROR
|
||||
2:SERVICE=netbird-peers
|
||||
5:pve01 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected
|
||||
6:pve02 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected
|
||||
7:pve03 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected
|
||||
8:edge-vm service=active/enabled wt0_ip=[IP]/16 management=yes signal=yes peers=2/11 Connected
|
||||
MONITOR_HEALTH_REF|path=/var/lib/homelab-health/vps-identity-audit.txt|mtime=2026-09-23 22:36:10.160214214 +0300|sha256=2b969313bfdbe51ced1af4d8a7f78ab62dab7bd5e66fa419b9a47208ae426db6
|
||||
1:STATUS=OK TS=20260923T193559Z TYPE=vps-identity-audit VPS_IP=[IP] ALIAS=netbird.vpn.gram1.ru ALIAS_IP=[IP] SSH_TRUST=BLOCKED_NO_BACKED_UP_HOST_KEYS BACKUP_HOSTKEYS=NO KNOWN_HOSTS_UNCHANGED=YES SECRET_PRINTED=REDACTED
|
||||
MONITORING_RECONCILE_END=1
|
||||
STATIC_ACTIVE_REFERENCE_RECONCILE_BEGIN=1
|
||||
ACTIVE_REF|path=/etc/homelab-backup-v2/targets.json|mtime=2026-09-22 12:19:09.941039348 +0300|sha256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b
|
||||
169: "logical_id": "md-relay",
|
||||
ACTIVE_REF|path=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf|mtime=2026-07-08 08:25:22.050610124 +0300|sha256=4cf3f2825b770b32447624a611bce605d97987bd2c6e8862c84d35b166f9102a
|
||||
3:ExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof
|
||||
ACTIVE_REF|path=/usr/local/sbin/homelab-vps-identity-audit|mtime=2026-06-24 16:00:07.135818676 +0300|sha256=b709691c52ae48432b70e197eb4810b6c374a05911cee3b9fdba4ecc67c76c3a
|
||||
23:backup="$(find /mnt/staging/netbird-vps-backups/snapshots -maxdepth 2 -type f -name 'netbird-vps-backup.tgz' 2>/dev/null | sort | tail -n1 || true)"
|
||||
STATIC_ACTIVE_REFERENCE_RECONCILE_END=1
|
||||
MD_RELAY_ENABLED_CLASS=true
|
||||
DECISION=HOLD_DELETE_MD_RELAY_ACTIVE_BACKUP_POLICY
|
||||
NEXT_GATE=DETACH_MD_RELAY_FROM_BACKUPV2_POLICY_WITH_ROLLBACK
|
||||
TASK_RESULT=PASS_MOLDOVA_ACTIVE_DEPS_RECONCILE_EVIDENCE_COLLECTED
|
||||
WORKERS2_MOLDOVA_ACTIVE_DEPS_RECONCILE_END=1
|
||||
CID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1
|
||||
MODE=change
|
||||
COMPONENT=forum-error-alert
|
||||
MUTATION_BOUNDARY=PVE01_FORUM_HEALTH_ADD_KINGOFWOLK;VM160_NEW_XF_ERROR_ALERT_SCRIPT_SERVICE_TIMER_STATE;ONE_CANARY_EMAIL;NO_XF_DB_WRITE;NO_XF_ERROR_DELETE;NO_MAIL_TRANSPORT_RECONFIG;NO_CLOUD;NO_VM_CONFIG
|
||||
EXECUTION_STARTED=true
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
HEALTH_PRE_SHA256=d7cee34b7d0e93c578f9203ba6e3634d304fee3ea5f3629a576dd683f1b50b29
|
||||
PVE02_TRANSPORT=PASS
|
||||
CHANGES_MADE=true
|
||||
PVE01_HEALTH_PATCHED=true
|
||||
HEALTH_POST_SHA256=4ee0c432fd906ba2d73198c965d0e57e4e7001781d61161b64f3c51625f034bd
|
||||
PVE02_VM160_STATE=running
|
||||
PVE02_QGA_TRANSPORT_RC=0
|
||||
VM160_GUEST_EXITCODE=90
|
||||
GUEST_CHANGE_BEGIN=true
|
||||
GUEST_FAILURE=MONITOR_PREFLIGHT
|
||||
GUEST_ROLLBACK_STARTED=true
|
||||
GUEST_ROLLBACK_RESTORED=true
|
||||
VM160_GUEST_STDERR_PRESENT=true
|
||||
VM160_GUEST_STDERR_SHA256=fd6627b9e2d9e0eca04003d62a4416ea377f7de34a5afb1800512ed9cf9094bd
|
||||
VM160_GUEST_STDERR_BYTES=94
|
||||
FAILURE=VM160_ALERT_INSTALL_OR_CANARY
|
||||
ROLLBACK_STARTED=true
|
||||
ROLLBACK_RESTORED=true
|
||||
DECISION=FAIL_REPAIR_ROLLED_BACK
|
||||
NEXT_GATE=REVIEW_FAILURE
|
||||
TASK_RESULT=FAIL_FORUM_ERROR_ALERT_REPAIR
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"FORUM-260923-REPAIR-ERROR-ALERT6F2BR1","status":"FAIL","changes_made":true,"rollback_started":true,"rollback_restored":true}
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user